Merge branch 'master' into interpreter_python

.ci/README.md

@@ -1,8 +1,8 @@
 
 # `.ci`
 
-This directory contains scripts for Travis CI and (more or less) Azure
-Pipelines, but they will also happily run on any Debian-like machine.
+This directory contains scripts for Continuous Integration platforms. Currently
+GitHub Actions, but ideally they will also run on any Debian-like machine.
 
 The scripts are usually split into `_install` and `_test` steps. The `_install`
 step will damage your machine, the `_test` step will just run the tests the way
@@ -28,17 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `VER`: Ansible version the `_install` script should install. Default changes
-  over time.
-* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
-* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
-  name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
-* `DISTROS`: the `ansible_` tests can run against multiple targets
-  simultaneously, which speeds things up. This is a space-separated list of
-  DISTRO names, but additionally, supports:
+* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run
+  the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines
+  the Linux distribution, target Python interepreter & number of instances.
+  Only distributions with a pre-built Linux container image can be used.
     * `debian-py3`: when generating Ansible inventory file, set
       `ansible_python_interpreter` to `python3`, i.e. run a test where the
       target interpreter is Python 3.
     * `debian*16`: generate 16 Docker containers running Debian. Also works
       with -py3.
 
+* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name,
+  and hence the container registry used for test targets.

.ci/ansible_tests.py

@@ -1,15 +1,18 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
+import collections
 import glob
 import os
 import signal
 import sys
 
+import jinja2
+
 import ci_lib
-from ci_lib import run
 
 
+TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
 TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
 HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
 
@@ -32,45 +35,51 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers()
+    containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split())
     ci_lib.start_containers(containers)
 
 
 with ci_lib.Fold('job_setup'):
     os.chdir(TESTS_DIR)
-    os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
+    os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 8))
 
-    run("mkdir %s", HOSTS_DIR)
+    ci_lib.run("mkdir %s", HOSTS_DIR)
     for path in glob.glob(TESTS_DIR + '/hosts/*'):
         if not path.endswith('default.hosts'):
-            run("ln -s %s %s", path, HOSTS_DIR)
-
+            ci_lib.run("ln -s %s %s", path, HOSTS_DIR)
+
+    distros = collections.defaultdict(list)
+    families = collections.defaultdict(list)
+    for container in containers:
+        distros[container['distro']].append(container['name'])
+        families[container['family']].append(container['name'])
+
+    jinja_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
+        lstrip_blocks=True,  # Remove spaces and tabs from before a block
+        trim_blocks=True,  # Remove first newline after a block
+    )
+    inventory_template = jinja_env.get_template('test-targets.j2')
     inventory_path = os.path.join(HOSTS_DIR, 'target')
+
     with open(inventory_path, 'w') as fp:
-        fp.write('[test-targets]\n')
-        fp.writelines(
-            "%(name)s "
-            "ansible_host=%(hostname)s "
-            "ansible_port=%(port)s "
-            "ansible_python_interpreter=%(python_path)s "
-            "ansible_user=mitogen__has_sudo_nopw "
-            "ansible_password=has_sudo_nopw_password"
-            "\n"
-            % container
-            for container in containers
-        )
+        fp.write(inventory_template.render(
+            containers=containers,
+            distros=distros,
+            families=families,
+        ))
 
     ci_lib.dump_file(inventory_path)
 
     if not ci_lib.exists_in_path('sshpass'):
-        run("sudo apt-get update")
-        run("sudo apt-get install -y sshpass")
+        ci_lib.run("sudo apt-get update")
+        ci_lib.run("sudo apt-get install -y sshpass")
 
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     try:
-        run('./run_ansible_playbook.py %s -i "%s" -vvv %s',
+        ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s',
             playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
     except:
         pause_if_interactive()