From 5c36eb467d851969bb2058395efb66aa76c01840 Mon Sep 17 00:00:00 2001 From: hulkoba Date: Thu, 30 May 2024 10:49:29 +0200 Subject: [PATCH 1/8] refactor(test conf feedstock) use ruamel --- tests/test_configure_feedstock.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/test_configure_feedstock.py b/tests/test_configure_feedstock.py index 4d81864b1..5c5892f81 100644 --- a/tests/test_configure_feedstock.py +++ b/tests/test_configure_feedstock.py @@ -7,10 +7,11 @@ from pathlib import Path import pytest -import yaml +from ruamel.yaml import YAML from conda_smithy import configure_feedstock +yaml=YAML(typ='safe') def test_noarch_skips_appveyor(noarch_recipe, jinja_env): noarch_recipe.config["provider"]["win"] = "appveyor" @@ -293,7 +294,7 @@ def test_upload_on_branch_azure(upload_on_branch_recipe, jinja_env): "azure-pipelines-osx.yml", ) ) as fp: - content_osx = yaml.safe_load(fp) + content_osx = yaml.load(fp) assert ( 'UPLOAD_ON_BRANCH="foo-branch"' in content_osx["jobs"][0]["steps"][0]["script"] @@ -310,7 +311,7 @@ def test_upload_on_branch_azure(upload_on_branch_recipe, jinja_env): "azure-pipelines-win.yml", ) ) as fp: - content_win = yaml.safe_load(fp) + content_win = yaml.load(fp) win_build_step = next( step for step in content_win["jobs"][0]["steps"] @@ -334,7 +335,7 @@ def test_upload_on_branch_azure(upload_on_branch_recipe, jinja_env): "azure-pipelines-linux.yml", ) ) as fp: - content_lin = yaml.safe_load(fp) + content_lin = yaml.load(fp) assert ( 'UPLOAD_ON_BRANCH="foo-branch"' in content_lin["jobs"][0]["steps"][1]["script"] @@ -360,7 +361,7 @@ def test_upload_on_branch_appveyor(upload_on_branch_recipe, jinja_env): with open( os.path.join(upload_on_branch_recipe.recipe, ".appveyor.yml") ) as fp: - content = yaml.safe_load(fp) + content = yaml.load(fp) assert "%APPVEYOR_REPO_BRANCH%" in content["deploy_script"][0] assert "UPLOAD_ON_BRANCH=foo-branch" in content["deploy_script"][-2] @@ -553,7 +554,7 @@ def test_secrets(py_recipe, jinja_env): ): if config_yaml.endswith(".yaml"): with open(config_yaml) as fo: - config = yaml.safe_load(fo) + config = yaml.load(fo) if "jobs" in config: assert any( any( @@ -572,7 +573,7 @@ def test_secrets(py_recipe, jinja_env): ) with open(os.path.join(py_recipe.recipe, ".drone.yml")) as fo: - config = list(yaml.safe_load_all(fo))[-1] + config = list(yaml.load_all(fo))[-1] assert any( step.get("environment", {}) .get("BINSTAR_TOKEN", {}) @@ -596,7 +597,7 @@ def test_migrator_recipe(recipe_migration_cfep9, jinja_env): "linux_64_python2.7.yaml", ) ) as fo: - variant = yaml.safe_load(fo) + variant = yaml.load(fo) assert variant["zlib"] == ["1000"] @@ -629,7 +630,7 @@ def test_migrator_cfp_override(recipe_migration_cfep9, jinja_env): "linux_64_python2.7.yaml", ) ) as fo: - variant = yaml.safe_load(fo) + variant = yaml.load(fo) assert variant["zlib"] == ["1001"] @@ -693,7 +694,7 @@ def test_migrator_downgrade_recipe( "linux_64_python2.7.yaml", ) ) as fo: - variant = yaml.safe_load(fo) + variant = yaml.load(fo) assert variant["zlib"] == ["1000"] @@ -788,7 +789,7 @@ def test_webservices_action_exists(py_recipe, jinja_env): with open( os.path.join(py_recipe.recipe, ".github/workflows/webservices.yml") ) as f: - action_config = yaml.safe_load(f) + action_config = yaml.load(f) assert "jobs" in action_config assert "webservices" in action_config["jobs"] @@ -805,7 +806,7 @@ def test_automerge_action_exists(py_recipe, jinja_env): with open( os.path.join(py_recipe.recipe, ".github/workflows/automerge.yml") ) as f: - action_config = yaml.safe_load(f) + action_config = yaml.load(f) assert "jobs" in action_config assert "automerge-action" in action_config["jobs"] From 198f719cf0f84a5a06c0adf287092a46d6dc0813 Mon Sep 17 00:00:00 2001 From: hulkoba Date: Thu, 30 May 2024 10:53:48 +0200 Subject: [PATCH 2/8] chore(test condaforge conf schema): remove unused yaml import --- tests/test_condaforge_config_schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_condaforge_config_schema.py b/tests/test_condaforge_config_schema.py index c987b6979..f90a95deb 100644 --- a/tests/test_condaforge_config_schema.py +++ b/tests/test_condaforge_config_schema.py @@ -1,6 +1,5 @@ import pytest from pydantic import ValidationError -import yaml from conda_smithy.schema import ConfigModel From df61193634c6fb34d62883684991ae6df95a27af Mon Sep 17 00:00:00 2001 From: hulkoba Date: Thu, 30 May 2024 10:54:32 +0200 Subject: [PATCH 3/8] refactor(test cli): use ruamel --- tests/test_cli.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index a4d09d3e0..cd37de559 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,7 @@ import subprocess from textwrap import dedent -import yaml +from ruamel.yaml import YAML import pytest import shutil @@ -12,6 +12,8 @@ _thisdir = os.path.abspath(os.path.dirname(__file__)) +yaml = YAML(typ='safe') + InitArgs = collections.namedtuple( "ArgsObject", ("recipe_directory", "feedstock_directory", "temporary_directory"), @@ -79,7 +81,7 @@ def test_init_with_custom_config(py_recipe): init_obj(args) destination = os.path.join(recipe, "py-test-feedstock") assert os.path.isdir(destination) - data = yaml.safe_load( + data = yaml.load( open(os.path.join(destination, "conda-forge.yml"), "r").read() ) assert data.get("bot") != None @@ -125,7 +127,7 @@ def test_init_multiple_output_matrix(testing_workdir): ) assert os.path.isfile(linux_libpng16) with open(linux_libpng16) as f: - config = yaml.safe_load(f) + config = yaml.load(f) assert config["libpng"] == ["1.6"] assert config["libpq"] == ["9.5"] # this is a zipped key, but it's not used, so it shouldn't show up @@ -240,7 +242,7 @@ def test_init_cuda_docker_images(testing_workdir): ) assert os.path.isfile(fn) with open(fn) as fh: - config = yaml.safe_load(fh) + config = yaml.load(fh) assert config["cuda_compiler"] == ["nvcc"] assert config["cuda_compiler_version"] == [f"{v}"] if v is None: @@ -291,7 +293,7 @@ def test_init_multiple_docker_images(testing_workdir): fn = os.path.join(matrix_dir, "linux_64_.yaml") assert os.path.isfile(fn) with open(fn) as fh: - config = yaml.safe_load(fh) + config = yaml.load(fh) assert config["docker_image"] == ["pickme_a"] assert config["cdt_name"] == ["pickme_1"] @@ -368,5 +370,5 @@ def test_render_variant_mismatches(testing_workdir): continue cfg = os.path.join(matrix_dir, _cfg) with open(cfg, "r") as f: - data = yaml.safe_load(f) + data = yaml.load(f) assert data["a"] == data["b"] From 6cf25ece3aed76723c74bf7f9695686886df2f6c Mon Sep 17 00:00:00 2001 From: hulkoba Date: Thu, 30 May 2024 10:56:19 +0200 Subject: [PATCH 4/8] refactor(conftest): use ruamel --- tests/conftest.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9dbc620c9..cd35ef9d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ from textwrap import dedent import pytest -import yaml +from ruamel.yaml import YAML from jinja2 import FileSystemLoader from jinja2.sandbox import SandboxedEnvironment @@ -55,6 +55,8 @@ def recipe_dirname(): @pytest.fixture(scope="function") def config_yaml(testing_workdir, recipe_dirname): + yaml = YAML() + yaml.default_flow_style = False config = {"python": ["2.7", "3.5"], "r_base": ["3.3.2", "3.4.2"]} os.makedirs(os.path.join(testing_workdir, recipe_dirname)) with open(os.path.join(testing_workdir, "config.yaml"), "w") as f: @@ -65,7 +67,7 @@ def config_yaml(testing_workdir, recipe_dirname): os.path.join(testing_workdir, recipe_dirname, "default_config.yaml"), "w", ) as f: - yaml.dump(config, f, default_flow_style=False) + yaml.dump(config, f) # need selectors, so write these more manually f.write( dedent( @@ -94,18 +96,18 @@ def config_yaml(testing_workdir, recipe_dirname): os.path.join(testing_workdir, recipe_dirname, "short_config.yaml"), "w" ) as f: config = {"python": ["2.7"]} - yaml.dump(config, f, default_flow_style=False) + yaml.dump(config, f) with open( os.path.join(testing_workdir, recipe_dirname, "long_config.yaml"), "w" ) as f: config = {"python": ["2.7", "3.5", "3.6"]} - yaml.dump(config, f, default_flow_style=False) + yaml.dump(config, f) with open(os.path.join(testing_workdir, "conda-forge.yml"), "w") as f: config = { "upload_on_branch": "foo-branch", "recipe_dir": recipe_dirname, } - yaml.dump(config, f, default_flow_style=False) + yaml.dump(config, f) return testing_workdir From 9b1f9c16ac6f308275ce37f7f408a77dad784c75 Mon Sep 17 00:00:00 2001 From: hulkoba Date: Thu, 30 May 2024 11:00:24 +0200 Subject: [PATCH 5/8] refactor(schema): use ruamel --- conda_smithy/schema.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conda_smithy/schema.py b/conda_smithy/schema.py index 3f2ccc798..bb08e9464 100644 --- a/conda_smithy/schema.py +++ b/conda_smithy/schema.py @@ -6,7 +6,7 @@ from inspect import cleandoc from typing import Any, Dict, List, Literal, Optional, Union -import yaml +from ruamel.yaml import YAML from pydantic import BaseModel, Field, create_model, ConfigDict from conda.base.constants import KNOWN_SUBDIRS @@ -1257,7 +1257,8 @@ class ConfigModel(BaseModel): if __name__ == "__main__": # This is used to generate the model dump for conda-smithy internal use # and for documentation purposes. - + yaml = YAML() + yaml.indent(mapping=2, sequence=4, offset=2) model = ConfigModel() with CONDA_FORGE_YAML_SCHEMA_FILE.open(mode="w+") as f: @@ -1266,4 +1267,4 @@ class ConfigModel(BaseModel): f.write("\n") with CONDA_FORGE_YAML_DEFAULTS_FILE.open(mode="w+") as f: - f.write(yaml.dump(model.model_dump(), indent=2)) + f.write(yaml.dump(model.model_dump())) From 42d45b1cd340a3fc932d6973b880cb98e93542cc Mon Sep 17 00:00:00 2001 From: hulkoba Date: Fri, 31 May 2024 08:58:21 +0200 Subject: [PATCH 6/8] refactor(configure feedstock): use ruamel --- conda_smithy/configure_feedstock.py | 45 +++++++++++++++-------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index b0f46cd92..234909b9e 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1,6 +1,7 @@ import copy import glob import hashlib +import io import logging import os import re @@ -18,6 +19,8 @@ from os import fspath from pathlib import Path, PurePath import requests +from ruamel.yaml import YAML as ruamel_yaml +from collections import OrderedDict try: from builtins import ExceptionGroup @@ -676,14 +679,6 @@ def _collapse_subpackage_variants( ) -def _yaml_represent_ordereddict(yaml_representer, data): - # represent_dict processes dict-likes with a .sort() method or plain iterables of key-value - # pairs. Only for the latter it never sorts and retains the order of the OrderedDict. - return yaml.representer.SafeRepresenter.represent_dict( - yaml_representer, data.items() - ) - - def _santize_remote_ci_setup(remote_ci_setup): remote_ci_setup_ = conda_build.utils.ensure_list(remote_ci_setup) remote_ci_setup = [] @@ -717,6 +712,10 @@ def finalize_config(config, platform, arch, forge_config): return config +def represent_ordereddict(dumper, data): + return dumper.represent_mapping(u'tag:yaml.org,2002:map', data) + + def dump_subspace_config_files( metas, root_path, platform, arch, upload, forge_config ): @@ -739,12 +738,9 @@ def dump_subspace_config_files( "collapsed subspace config files: {}".format(pprint.pformat(configs)) ) - # get rid of the special object notation in the yaml file for objects that we dump - yaml.add_representer(set, yaml.representer.SafeRepresenter.represent_list) - yaml.add_representer( - tuple, yaml.representer.SafeRepresenter.represent_list - ) - yaml.add_representer(OrderedDict, _yaml_represent_ordereddict) + ruamel = ruamel_yaml(typ='safe') + ruamel.default_flow_style = False + ruamel.Representer.add_representer(OrderedDict, represent_ordereddict) platform_arch = "{}-{}".format(platform, arch) @@ -773,7 +769,7 @@ def dump_subspace_config_files( ) with write_file(out_path) as f: - yaml.dump(config, f, default_flow_style=False) + ruamel.dump(config, f) target_platform = config.get("target_platform", [platform_arch])[0] result.append( @@ -1836,7 +1832,9 @@ def _azure_specific_setup(jinja_env, forge_config, forge_dir, platform): azure_settings["strategy"]["matrix"][data["config_name"]] = config_rendered # fmt: on - forge_config["azure_yaml"] = yaml.dump(azure_settings) + buffer = io.StringIO() + ruamel_yaml(typ='safe').dump(azure_settings, buffer) + forge_config["azure_yaml"] = buffer.getvalue() _render_template_exe_files( forge_config=forge_config, jinja_env=jinja_env, @@ -2041,7 +2039,7 @@ def render_README(jinja_env, forge_config, forge_dir, render_info=None): variant_name, _ = os.path.splitext(filename) variants.append(variant_name) with open(os.path.join(ci_support_path, filename)) as fh: - data = yaml.safe_load(fh) + data = ruamel_yaml(typ='safe').load(fh) channel_targets.append( data.get("channel_targets", ["conda-forge main"])[0] ) @@ -2098,7 +2096,9 @@ def render_README(jinja_env, forge_config, forge_dir, render_info=None): azure_build_id_from_token(forge_config) logger.debug("README") - logger.debug(yaml.dump(forge_config)) + buffer = io.StringIO() + ruamel_yaml(typ='safe').dump(forge_config, buffer) + logger.debug(buffer.getvalue()) with write_file(target_fname) as fh: fh.write(template.render(**forge_config)) @@ -2160,7 +2160,7 @@ def _update_dict_within_dict(items, config): def _read_forge_config(forge_dir, forge_yml=None): # Load default values from the conda-forge.yml file with open(CONDA_FORGE_YAML_DEFAULTS_FILE, "r") as fh: - default_config = yaml.safe_load(fh.read()) + default_config = ruamel_yaml(typ='safe').load(fh.read()) if forge_yml is None: forge_yml = os.path.join(forge_dir, "conda-forge.yml") @@ -2175,7 +2175,7 @@ def _read_forge_config(forge_dir, forge_yml=None): ) with open(forge_yml, "r") as fh: - documents = list(yaml.safe_load_all(fh)) + documents = list(ruamel_yaml(typ='safe').load_all(fh)) file_config = (documents or [None])[0] or {} # Validate loaded configuration against a JSON schema. @@ -2326,9 +2326,10 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): # Set some more azure defaults config["azure"].setdefault("user_or_org", config["github"]["user_or_org"]) - log = yaml.safe_dump(config) logger.debug("## CONFIGURATION USED\n") - logger.debug(log) + buffer = io.StringIO() + ruamel_yaml().dump(config, buffer) + logger.debug(buffer.getvalue()) logger.debug("## END CONFIGURATION\n") if config["provider"]["linux_aarch64"] == "default": From 370b7886c6f63a583282b76895f04e84098cd09e Mon Sep 17 00:00:00 2001 From: hulkoba Date: Tue, 4 Jun 2024 10:55:47 +0200 Subject: [PATCH 7/8] add ruamel refactor news entry --- news/ruamel-refactor.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 news/ruamel-refactor.rst diff --git a/news/ruamel-refactor.rst b/news/ruamel-refactor.rst new file mode 100644 index 000000000..20db4b5da --- /dev/null +++ b/news/ruamel-refactor.rst @@ -0,0 +1,23 @@ +**Added:** + +* + +**Changed:** + +* Use ruamel library and remove usage of yaml almost everywhere - except at conda-smithy/variant_algebra.py + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* From 367846dcf3c6bac367a6fa04007e5a9330df99e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alba=20Herrer=C3=ADas?= Date: Fri, 21 Jun 2024 12:53:50 +0100 Subject: [PATCH 8/8] fix: linting --- conda_smithy/configure_feedstock.py | 14 +++++++------- tests/test_cli.py | 2 +- tests/test_configure_feedstock.py | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 234909b9e..8f5e97caa 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -713,7 +713,7 @@ def finalize_config(config, platform, arch, forge_config): def represent_ordereddict(dumper, data): - return dumper.represent_mapping(u'tag:yaml.org,2002:map', data) + return dumper.represent_mapping("tag:yaml.org,2002:map", data) def dump_subspace_config_files( @@ -738,7 +738,7 @@ def dump_subspace_config_files( "collapsed subspace config files: {}".format(pprint.pformat(configs)) ) - ruamel = ruamel_yaml(typ='safe') + ruamel = ruamel_yaml(typ="safe") ruamel.default_flow_style = False ruamel.Representer.add_representer(OrderedDict, represent_ordereddict) @@ -1833,7 +1833,7 @@ def _azure_specific_setup(jinja_env, forge_config, forge_dir, platform): # fmt: on buffer = io.StringIO() - ruamel_yaml(typ='safe').dump(azure_settings, buffer) + ruamel_yaml(typ="safe").dump(azure_settings, buffer) forge_config["azure_yaml"] = buffer.getvalue() _render_template_exe_files( forge_config=forge_config, @@ -2039,7 +2039,7 @@ def render_README(jinja_env, forge_config, forge_dir, render_info=None): variant_name, _ = os.path.splitext(filename) variants.append(variant_name) with open(os.path.join(ci_support_path, filename)) as fh: - data = ruamel_yaml(typ='safe').load(fh) + data = ruamel_yaml(typ="safe").load(fh) channel_targets.append( data.get("channel_targets", ["conda-forge main"])[0] ) @@ -2097,7 +2097,7 @@ def render_README(jinja_env, forge_config, forge_dir, render_info=None): logger.debug("README") buffer = io.StringIO() - ruamel_yaml(typ='safe').dump(forge_config, buffer) + ruamel_yaml(typ="safe").dump(forge_config, buffer) logger.debug(buffer.getvalue()) with write_file(target_fname) as fh: @@ -2160,7 +2160,7 @@ def _update_dict_within_dict(items, config): def _read_forge_config(forge_dir, forge_yml=None): # Load default values from the conda-forge.yml file with open(CONDA_FORGE_YAML_DEFAULTS_FILE, "r") as fh: - default_config = ruamel_yaml(typ='safe').load(fh.read()) + default_config = ruamel_yaml(typ="safe").load(fh.read()) if forge_yml is None: forge_yml = os.path.join(forge_dir, "conda-forge.yml") @@ -2175,7 +2175,7 @@ def _read_forge_config(forge_dir, forge_yml=None): ) with open(forge_yml, "r") as fh: - documents = list(ruamel_yaml(typ='safe').load_all(fh)) + documents = list(ruamel_yaml(typ="safe").load_all(fh)) file_config = (documents or [None])[0] or {} # Validate loaded configuration against a JSON schema. diff --git a/tests/test_cli.py b/tests/test_cli.py index cd37de559..171b202b2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,7 +12,7 @@ _thisdir = os.path.abspath(os.path.dirname(__file__)) -yaml = YAML(typ='safe') +yaml = YAML(typ="safe") InitArgs = collections.namedtuple( "ArgsObject", diff --git a/tests/test_configure_feedstock.py b/tests/test_configure_feedstock.py index 5c5892f81..8974be8bb 100644 --- a/tests/test_configure_feedstock.py +++ b/tests/test_configure_feedstock.py @@ -11,7 +11,8 @@ from conda_smithy import configure_feedstock -yaml=YAML(typ='safe') +yaml = YAML(typ="safe") + def test_noarch_skips_appveyor(noarch_recipe, jinja_env): noarch_recipe.config["provider"]["win"] = "appveyor"