Merge pull request #159 from pelson/custom_matrix

Added ability to define a custom matrix item.

conda_smithy/ci_register.py

@@ -53,7 +53,7 @@ def add_project_to_circle(user, project):
     # It is a strange response code, but is doing what was asked...
     if response.status_code != 400:
         response.raise_for_status()
-    print(' * {}/{} enabled on CircleCI')
+    print(' * {}/{} enabled on CircleCI'.format(user, project))
 
 
 def add_project_to_appveyor(user, project):

conda_smithy/configure_feedstock.py

@@ -24,12 +24,14 @@ def render_run_docker_build(jinja_env, forge_config, forge_dir):
     with fudge_subdir('linux-64'):
         meta = forge_config['package']
         meta.parse_again()
-        matrix = compute_build_matrix(meta)
+        matrix = compute_build_matrix(meta, forge_config.get('matrix'))
         cases_not_skipped = []
         for case in matrix:
-            if not ResolvedDistribution(meta, case).skip():
-                cases_not_skipped.append(case)
-        matrix = cases_not_skipped
+            pkgs, vars = split_case(case)
+            with enable_vars(vars):
+                if not ResolvedDistribution(meta, pkgs).skip():
+                    cases_not_skipped.append(vars + sorted(pkgs))
+        matrix = sorted(cases_not_skipped)
 
     target_fname = os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh')
     if not matrix:
@@ -38,8 +40,8 @@ def render_run_docker_build(jinja_env, forge_config, forge_dir):
         # be skipped anyway).
         matrix = [()]
 
-    forge_config = forge_config.copy()
-    forge_config['matrix'] = matrix
+    matrix = prepare_matrix_for_env_vars(matrix)
+    forge_config = update_matrix(forge_config, matrix)
 
     # If there is a "yum_requirements.txt" file in the recipe, we honour it.
     yum_requirements_fpath = os.path.join(forge_dir, 'recipe',
@@ -87,13 +89,15 @@ def render_travis(jinja_env, forge_config, forge_dir):
     with fudge_subdir('osx-64'):
         meta = forge_config['package']
         meta.parse_again()
-        matrix = compute_build_matrix(meta)
+        matrix = compute_build_matrix(meta, forge_config.get('matrix'))
 
         cases_not_skipped = []
         for case in matrix:
-            if not ResolvedDistribution(meta, case).skip():
-                cases_not_skipped.append(case)
-        matrix = cases_not_skipped
+            pkgs, vars = split_case(case)
+            with enable_vars(vars):
+                if not ResolvedDistribution(meta, pkgs).skip():
+                    cases_not_skipped.append(vars + sorted(pkgs))
+        matrix = sorted(cases_not_skipped)
 
     target_fname = os.path.join(forge_dir, '.travis.yml')
 
@@ -103,8 +107,8 @@ def render_travis(jinja_env, forge_config, forge_dir):
         # be skipped anyway).
         matrix = [()]
 
-    forge_config = forge_config.copy()
-    forge_config['matrix'] = matrix
+    matrix = prepare_matrix_for_env_vars(matrix)
+    forge_config = update_matrix(forge_config, matrix)
 
     template = jinja_env.get_template('travis.yml.tmpl')
     with open(target_fname, 'w') as fh:
@@ -118,17 +122,84 @@ def render_README(jinja_env, forge_config, forge_dir):
         fh.write(template.render(**forge_config))
 
 
-def render_appveyor(jinja_env, forge_config, forge_dir):
-    with fudge_subdir('win-64'):
-        meta = forge_config['package']
-        meta.parse_again()
-        matrix = compute_build_matrix(meta)
+class MatrixCaseEnvVar(object):
+    def __init__(self, name, value):
+        self.name = name
+        self.value = value
 
-        cases_not_skipped = []
-        for case in matrix:
-            if not ResolvedDistribution(meta, case).skip():
-                cases_not_skipped.append(case)
-        matrix = cases_not_skipped
+    def __iter__(self):
+        # We make the Var iterable so that loops like
+        # ``for name, value in cases`` can be used.
+        return iter([self.name, self.value])
+
+    def __cmp__(self, other):
+        # Implement ordering so that sorting functions as expected.
+        if not isinstance(other, type(self)):
+            return -3
+        elif other.name != self.name:
+            return -2
+        else:
+            return cmp(self.value, other.value)
+
+
+@contextmanager
+def enable_vars(vars):
+    existing = {}
+    for var in vars:
+        if var.name in os.environ:
+            existing[var.name] = os.environ[var.name]
+        os.environ[var.name] = str(var.value)
+    yield
+    for var in vars:
+        if var.name in existing:
+            os.environ[var.name] = existing[var.name]
+        else:
+            os.environ.pop(var.name)
+
+
+def split_case(case):
+    vars = [item for item in case
+            if isinstance(item, MatrixCaseEnvVar)]
+    pkgs = [item for item in case
+            if not isinstance(item, MatrixCaseEnvVar)]
+    return pkgs, vars
+
+
+def render_appveyor(jinja_env, forge_config, forge_dir):
+    full_matrix = []
+    for platform, arch in [['win-32', 'x86'], ['win-64', 'x64']]:
+        with fudge_subdir(platform):
+            meta = forge_config['package']
+            meta.parse_again()
+            matrix = compute_build_matrix(meta, forge_config.get('matrix'))
+
+            cases_not_skipped = []
+            for case in matrix:
+                pkgs, vars = split_case(case)
+                with enable_vars(vars):
+                    if not ResolvedDistribution(meta, pkgs).skip():
+                        cases_not_skipped.append(vars + sorted(pkgs))
+            if cases_not_skipped:
+                arch_env = MatrixCaseEnvVar('TARGET_ARCH', arch)
+                full_matrix.extend([arch_env] + list(case)
+                                   for case in cases_not_skipped)
+
+    matrix = full_matrix
+
+    def sort_without_target_arch(case):
+        arch = None
+        python = None
+        cmp_case = []
+        for name, val in case:
+            if name == 'TARGET_ARCH':
+                arch_order = {'x86': 1, 'x64': 2}.get(val, 0)
+            elif name == 'python':
+                # We group all pythons together.
+                python = val
+            else:
+                cmp_case.append([name, val])
+        return [python, cmp_case, arch_order]
+    matrix = sorted(matrix, key=sort_without_target_arch)
 
     target_fname = os.path.join(forge_dir, 'appveyor.yml')
     target_fname_disabled = os.path.join(forge_dir, 'disabled_appveyor.yml')
@@ -144,14 +215,50 @@ def render_appveyor(jinja_env, forge_config, forge_dir):
         if os.path.exists(target_fname_disabled):
             os.remove(target_fname_disabled)
 
-        forge_config = forge_config.copy()
-        forge_config['matrix'] = matrix
-
+        matrix = prepare_matrix_for_env_vars(matrix)
+        forge_config = update_matrix(forge_config, matrix)
         template = jinja_env.get_template('appveyor.yml.tmpl')
         with open(target_fname, 'w') as fh:
             fh.write(template.render(**forge_config))
 
 
+def update_matrix(forge_config, new_matrix):
+    """
+    Return a new config with the build matrix updated.
+
+    """
+    forge_config = forge_config.copy()
+    forge_config['matrix'] = new_matrix
+    return forge_config
+
+
+def prepare_matrix_for_env_vars(matrix):
+    """
+    Turns a matrix with environment variables and pakages into a matrix of
+    just environment variables. The package variables are prefixed with CONDA,
+    and special cases such as Python and Numpy are handled.
+
+    """
+    special_conda_vars = {'python': 'CONDA_PY', 'numpy': 'CONDA_NPY'}
+    env_matrix = []
+    for case in matrix:
+        new_case = []
+        for item in case:
+            if isinstance(item, MatrixCaseEnvVar):
+                new_case.append((item.name, item.value))
+            else:
+                # We have a package, so transform it into something conda understands.
+                name, value = item
+                if name in special_conda_vars:
+                    name = special_conda_vars[name]
+                    value = str(value).replace('.', '')
+                else:
+                    name = 'CONDA_' + name.upper()
+                new_case.append((name, value))
+        env_matrix.append(new_case)
+    return env_matrix
+
+
 def copytree(src, dst, ignore=(), root_dst=None):
     if root_dst is None:
         root_dst = dst
@@ -183,10 +290,14 @@ def meta_of_feedstock(forge_dir):
     return meta
 
 
-def compute_build_matrix(meta):
+def compute_build_matrix(meta, existing_matrix=None):
     index = conda.api.get_index()
     mtx = special_case_version_matrix(meta, index)
     mtx = list(filter_cases(mtx, ['python >=2.7,<3|>=3.4', 'numpy >=1.10']))
+    if existing_matrix:
+        mtx = [tuple(mtx_case) + tuple(MatrixCaseEnvVar(*item) for item in case)
+               for case in sorted(existing_matrix)
+               for mtx_case in mtx]
     return mtx
 
 

conda_smithy/github.py

@@ -34,7 +34,7 @@ def create_github_repo(args):
         # Use the organization provided.
         user_or_org = gh.get_organization(args.organization)
 
-    repo_name = os.path.basename(os.path.abspath(args.feedstock_directory))
+    repo_name = meta.name()
     try:
         gh_repo = user_or_org.create_repo(repo_name, has_wiki=False,
                                           description='A conda-smithy repository for {}.'.format(meta.name()))

conda_smithy/templates/README.md.tmpl

@@ -69,9 +69,10 @@ Terminology
 
 Current build status
 ====================
+{% set appveyor_name = package.name().replace('_', '-').replace('.', '-') %}
 Linux: [![Circle CI](https://circleci.com/gh/conda-forge/{{package.name()}}-feedstock.svg?style=svg)](https://circleci.com/gh/conda-forge/{{package.name()}}-feedstock)
 OSX: [![TravisCI](https://travis-ci.org/conda-forge/{{package.name()}}-feedstock.svg?branch=master)](https://travis-ci.org/conda-forge/{{package.name()}}-feedstock) 
-Windows: [![AppVeyor](https://ci.appveyor.com/api/projects/status/github/conda-forge/{{package.name()}}-feedstock?svg=True)](https://ci.appveyor.com/project/conda-forge/{{package.name()}}-feedstock/branch/master)
+Windows: [![AppVeyor](https://ci.appveyor.com/api/projects/status/github/conda-forge/{{appveyor_name}}-feedstock?svg=True)](https://ci.appveyor.com/project/conda-forge/{{appveyor_name}}-feedstock/branch/master)
 
 Current release info
 ====================

conda_smithy/templates/appveyor.yml.tmpl

@@ -1,9 +1,3 @@
-{%- set conda_vars = {'python': 'CONDA_PY', 'numpy': 'CONDA_NPY'} -%}
-{% macro matrix_env(arch, matrix_item) -%}
-  - TARGET_ARCH: {{ arch }}{% for dep_name, version in matrix_item | sort %}
-      {%- if dep_name in conda_vars %}{% set version = version|replace('.', '') %}{% endif %}
-      {{ conda_vars.get(dep_name, 'CONDA_' + dep_name|upper) }}: {{ version }}{% endfor %}
-{%- endmacro %}
 # This file was automatically generated by conda-smithy. To update a component of this
 # file, make changes to conda-forge.yaml and/or recipe/meta.yaml, and run
 # "conda-smithy regenerate".
@@ -27,9 +21,11 @@ environment:
     secure: {{ hashed_secure }}
   {%- endfor %}
 
-  matrix:{% for case in matrix | sort %}
-    {{ matrix_env("x86", case) }}
-    {{ matrix_env("x64", case) }}
+  matrix:
+  {%- for case in matrix %}
+    {%- for dep_name, version in case %}
+    {% if loop.first %}- {% else %}  {% endif %}{{ dep_name }}: {{version }}
+    {%- endfor %}
 {% endfor %}
 
 # We always use a 64-bit machine, but can build x86 distributions

conda_smithy/templates/run_docker_build_matrix.tmpl

@@ -1,22 +1,21 @@
 {%- extends "run_docker_build.tmpl" -%}
-{%- set conda_vars = {'python': 'CONDA_PY', 'numpy': 'CONDA_NPY'} -%}
 {% macro matrix_env(matrix_item) -%}
-    {% for dep_name, version in matrix_item | sort %}
-        {%- if dep_name in conda_vars %}{% set version = version|replace('.', '') %}{% endif %}
-    export {{ conda_vars.get(dep_name, 'CONDA_' + dep_name|upper) }}={{ version }}{% endfor %}
+  {%- for dep_name, version in matrix_item %}
+    export {{ dep_name }}={{version }}
+  {%- endfor %}
 {%- endmacro -%}
 
 {%- block build -%}
   # Embarking on {{ matrix|length }} case(s).
-{%- for case in matrix | sort %}
+{%- for case in matrix %}
   {%- if case %}
     set -x
     {{- matrix_env(case) }}
     set +x
   {%- endif -%}
   {{- super()|indent(4) }}
 {% endfor %}{% endblock -%}
-{%- block test_and_upload -%}{% for case in matrix | sort -%}
+{%- block test_and_upload -%}{% for case in matrix -%}
     {{ matrix_env(case) }}
     {{- super()|indent(4) }}
 {%- endfor -%}{% endblock -%}

conda_smithy/templates/travis.yml.tmpl

@@ -1,7 +1,7 @@
-{%- set conda_vars = {'python': 'CONDA_PY', 'numpy': 'CONDA_NPY'} -%}
 {%- macro matrix_env(matrix_item) -%}
-    {% for dep_name, version in matrix_item | sort %}{%- if dep_name in conda_vars %}{% set version = version|replace('.', '') %}{% endif -%}
-        {{ conda_vars.get(dep_name, 'CONDA_' + dep_name|upper) }}={{ version }}{% if not loop.last %}  {% endif %}{% endfor %}
+    {% for dep_name, version in matrix_item -%}
+        {{ dep_name }}={{ version }}{% if not loop.last %}  {% endif %}
+    {%- endfor %}
 {%- endmacro -%}
 # This file was generated automatically from conda-smithy. To update this configuration,
 # update the conda-forge.yaml and/or the recipe/meta.yaml.
@@ -13,7 +13,7 @@ language: objective-c
 env:
   {%- if matrix[0] %}
   matrix:
-    {% for case in matrix | sort %}
+    {% for case in matrix %}
     - {{ matrix_env(case) }}
     {%- endfor %}
   {%- endif %}