diff --git a/.ci/README.md b/.ci/README.md index 1b9f9dfa9..17b7d2dd8 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -1,8 +1,8 @@ # `.ci` -This directory contains scripts for Travis CI and (more or less) Azure -Pipelines, but they will also happily run on any Debian-like machine. +This directory contains scripts for Continuous Integration platforms. Currently +GitHub Actions, but ideally they will also run on any Debian-like machine. The scripts are usually split into `_install` and `_test` steps. The `_install` step will damage your machine, the `_test` step will just run the tests the way @@ -28,17 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `VER`: Ansible version the `_install` script should install. Default changes - over time. -* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. -* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This - name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` -* `DISTROS`: the `ansible_` tests can run against multiple targets - simultaneously, which speeds things up. This is a space-separated list of - DISTRO names, but additionally, supports: +* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run + the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines + the Linux distribution, target Python interepreter & number of instances. + Only distributions with a pre-built Linux container image can be used. * `debian-py3`: when generating Ansible inventory file, set `ansible_python_interpreter` to `python3`, i.e. run a test where the target interpreter is Python 3. * `debian*16`: generate 16 Docker containers running Debian. Also works with -py3. +* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name, + and hence the container registry used for test targets. diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py deleted file mode 100755 index 08f043563..000000000 --- a/.ci/ansible_install.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ - [ - # Must be installed separately, as PyNACL indirect requirement causes - # newer version to be installed if done in a single pip run. - # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml - 'pip install "pycparser<2.19" "idna<2.7"', - 'pip install ' - '-r tests/requirements.txt ' - '-r tests/ansible/requirements.txt', - # encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error - 'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install -q ansible=={0}'.format(ci_lib.ANSIBLE_VERSION) - ] -] - -batches.append(ci_lib.throttle( - 'docker pull %s' % (ci_lib.image_for_distro(distro),) - for distro in ci_lib.DISTROS -)) - -ci_lib.run_batches(batches) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index b2aa31991..4a7bedaeb 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -1,15 +1,18 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen +import collections import glob import os import signal import sys +import jinja2 + import ci_lib -from ci_lib import run +TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates') TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') @@ -32,45 +35,51 @@ def pause_if_interactive(): with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers() + containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split()) ci_lib.start_containers(containers) with ci_lib.Fold('job_setup'): os.chdir(TESTS_DIR) - os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) + os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 8)) - run("mkdir %s", HOSTS_DIR) + ci_lib.run("mkdir %s", HOSTS_DIR) for path in glob.glob(TESTS_DIR + '/hosts/*'): if not path.endswith('default.hosts'): - run("ln -s %s %s", path, HOSTS_DIR) - + ci_lib.run("ln -s %s %s", path, HOSTS_DIR) + + distros = collections.defaultdict(list) + families = collections.defaultdict(list) + for container in containers: + distros[container['distro']].append(container['name']) + families[container['family']].append(container['name']) + + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR), + lstrip_blocks=True, # Remove spaces and tabs from before a block + trim_blocks=True, # Remove first newline after a block + ) + inventory_template = jinja_env.get_template('test-targets.j2') inventory_path = os.path.join(HOSTS_DIR, 'target') + with open(inventory_path, 'w') as fp: - fp.write('[test-targets]\n') - fp.writelines( - "%(name)s " - "ansible_host=%(hostname)s " - "ansible_port=%(port)s " - "ansible_python_interpreter=%(python_path)s " - "ansible_user=mitogen__has_sudo_nopw " - "ansible_password=has_sudo_nopw_password" - "\n" - % container - for container in containers - ) + fp.write(inventory_template.render( + containers=containers, + distros=distros, + families=families, + )) ci_lib.dump_file(inventory_path) if not ci_lib.exists_in_path('sshpass'): - run("sudo apt-get update") - run("sudo apt-get install -y sshpass") + ci_lib.run("sudo apt-get update") + ci_lib.run("sudo apt-get install -y sshpass") with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') try: - run('./run_ansible_playbook.py %s -i "%s" -vvv %s', + ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s', playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) except: pause_if_interactive() diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml deleted file mode 100644 index 41b6a8369..000000000 --- a/.ci/azure-pipelines-steps.yml +++ /dev/null @@ -1,23 +0,0 @@ - -parameters: - name: '' - pool: '' - sign: false - -steps: -- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py" - displayName: "Run prep_azure.py" - -- script: | - echo "##vso[task.prependpath]/tmp/venv/bin" - - displayName: activate venv - -- script: .ci/spawn_reverse_shell.py - displayName: "Spawn reverse shell" - -- script: .ci/$(MODE)_install.py - displayName: "Run $(MODE)_install.py" - -- script: .ci/$(MODE)_tests.py - displayName: "Run $(MODE)_tests.py" diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml deleted file mode 100644 index c22dcf6c8..000000000 --- a/.ci/azure-pipelines.yml +++ /dev/null @@ -1,126 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -jobs: - -- job: Mac - # vanilla Ansible is really slow - timeoutInMinutes: 120 - steps: - - template: azure-pipelines-steps.yml - pool: - vmImage: macOS-10.15 - strategy: - matrix: - Mito27_27: - python.version: '2.7' - MODE: mitogen - # TODO: test python3, python3 tests are broken - Ans210_27: - python.version: '2.7' - MODE: localhost_ansible - VER: 2.10.0 - - # NOTE: this hangs when ran in Ubuntu 18.04 - Vanilla_210_27: - python.version: '2.7' - MODE: localhost_ansible - VER: 2.10.0 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - - -- job: Linux - pool: - vmImage: "Ubuntu 18.04" - steps: - - template: azure-pipelines-steps.yml - strategy: - matrix: - # - # Confirmed working - # - Mito27Debian_27: - python.version: '2.7' - MODE: mitogen - DISTRO: debian9 - - #MitoPy27CentOS6_26: - #python.version: '2.7' - #MODE: mitogen - #DISTRO: centos6 - - Mito36CentOS6_26: - python.version: '3.6' - MODE: mitogen - DISTRO: centos6 - - Mito37Debian_27: - python.version: '3.7' - MODE: mitogen - DISTRO: debian9 - - Mito39Debian_27: - python.version: '3.9' - MODE: mitogen - DISTRO: debian9 - VER: 2.10.0 - - #Py26CentOS7: - #python.version: '2.7' - #MODE: mitogen - #DISTRO: centos6 - - #DebOps_2460_27_27: - #python.version: '2.7' - #MODE: debops_common - #VER: 2.4.6.0 - - #DebOps_262_36_27: - #python.version: '3.6' - #MODE: debops_common - #VER: 2.6.2 - - #Ansible_2460_26: - #python.version: '2.7' - #MODE: ansible - #VER: 2.4.6.0 - - #Ansible_262_26: - #python.version: '2.7' - #MODE: ansible - #VER: 2.6.2 - - #Ansible_2460_36: - #python.version: '3.6' - #MODE: ansible - #VER: 2.4.6.0 - - #Ansible_262_36: - #python.version: '3.6' - #MODE: ansible - #VER: 2.6.2 - - #Vanilla_262_27: - #python.version: '2.7' - #MODE: ansible - #VER: 2.6.2 - #DISTROS: debian - #STRATEGY: linear - - Ansible_210_27: - python.version: '2.7' - MODE: ansible - VER: 2.10.0 - - Ansible_210_35: - python.version: '3.5' - MODE: ansible - VER: 2.10.0 - - Ansible_210_39: - python.version: '3.9' - MODE: ansible - VER: 2.10.0 diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 524337a95..afb62e023 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -1,15 +1,20 @@ - from __future__ import absolute_import from __future__ import print_function import atexit +import errno import os +import re import shlex import shutil -import subprocess import sys import tempfile +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess + try: import urlparse except ImportError: @@ -23,66 +28,51 @@ ) -# -# check_output() monkeypatch cutpasted from testlib.py -# +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) -def subprocess__check_output(*popenargs, **kwargs): - # Missing from 2.6. - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, _ = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output -if not hasattr(subprocess, 'check_output'): - subprocess.check_output = subprocess__check_output +_print = print +def print(*args, **kwargs): + file = kwargs.get('file', sys.stdout) + flush = kwargs.pop('flush', False) + _print(*args, **kwargs) + if flush: + file.flush() -# ------------------ +def _have_cmd(args): + try: + subprocess.run( + args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + except OSError as exc: + if exc.errno == errno.ENOENT: + return False + raise + except subprocess.CallProcessError: + return False + return True + def have_apt(): - proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True) - return proc.wait() == 0 + return _have_cmd(['apt', '--help']) + def have_brew(): - proc = subprocess.Popen('brew help >/dev/null 2>/dev/null', shell=True) - return proc.wait() == 0 + return _have_cmd(['brew', 'help']) def have_docker(): - proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True) - return proc.wait() == 0 + return _have_cmd(['docker', 'info']) -# ----------------- - -# Force line buffering on stdout. -sys.stdout = os.fdopen(1, 'w', 1) - -# Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars. -if 'TRAVIS_HOME' in os.environ: - proc = subprocess.Popen( - args=['stdbuf', '-oL', 'cat'], - stdin=subprocess.PIPE - ) - - os.dup2(proc.stdin.fileno(), 1) - os.dup2(proc.stdin.fileno(), 2) - - def cleanup_travis_junk(stdout=sys.stdout, stderr=sys.stderr, proc=proc): - stdout.close() - stderr.close() - proc.terminate() - - atexit.register(cleanup_travis_junk) - -# ----------------- - def _argv(s, *args): """Interpolate a command line using *args, return an argv style list. @@ -95,24 +85,22 @@ def _argv(s, *args): def run(s, *args, **kwargs): - """ Run a command, with arguments, and print timing information + """ Run a command, with arguments >>> rc = run('echo "%s %s"', 'foo', 'bar') - Running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + Running: ['echo', 'foo bar'] foo bar - 0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k - 0inputs+0outputs (0major+71minor)pagefaults 0swaps - Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + Finished running: ['echo', 'foo bar'] >>> rc 0 """ - argv = ['/usr/bin/time', '--'] + _argv(s, *args) - print('Running: %s' % (argv,)) + argv = _argv(s, *args) + print('Running: %s' % (argv,), flush=True) try: ret = subprocess.check_call(argv, **kwargs) - print('Finished running: %s' % (argv,)) + print('Finished running: %s' % (argv,), flush=True) except Exception: - print('Exception occurred while running: %s' % (argv,)) + print('Exception occurred while running: %s' % (argv,), file=sys.stderr, flush=True) raise return ret @@ -179,13 +167,13 @@ def get_output(s, *args, **kwargs): 'foo bar\n' """ argv = _argv(s, *args) - print('Running: %s' % (argv,)) + print('Running: %s' % (argv,), flush=True) return subprocess.check_output(argv, **kwargs) def exists_in_path(progname): """ - Return True if proganme exists in $PATH. + Return True if progname exists in $PATH. >>> exists_in_path('echo') True @@ -206,40 +194,12 @@ def destroy(self, rmtree=shutil.rmtree): class Fold(object): - """ - Bracket a section of stdout with travis_fold markers. - - This allows the section to be collapsed or expanded in Travis CI web UI. - - >>> with Fold('stage 1'): - ... print('Frobnicate the frobnitz') - ... - travis_fold:start:stage 1 - Frobnicate the frobnitz - travis_fold:end:stage 1 - """ - def __init__(self, name): - self.name = name - - def __enter__(self): - print('travis_fold:start:%s' % (self.name)) + def __init__(self, name): pass + def __enter__(self): pass + def __exit__(self, _1, _2, _3): pass - def __exit__(self, _1, _2, _3): - print('') - print('travis_fold:end:%s' % (self.name)) - -os.environ.setdefault('ANSIBLE_STRATEGY', - os.environ.get('STRATEGY', 'mitogen_linear')) -# Ignoreed when MODE=mitogen -ANSIBLE_VERSION = os.environ.get('VER', '2.6.2') GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -# Used only when MODE=mitogen -DISTRO = os.environ.get('DISTRO', 'debian') -# Used only when MODE=ansible -DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() -TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) -BASE_PORT = 2200 TMP = TempDir().path @@ -262,6 +222,7 @@ def __exit__(self, _1, _2, _3): def get_docker_hostname(): """Return the hostname where the docker daemon is running. """ + # Duplicated in testlib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -270,61 +231,65 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def image_for_distro(distro): - """Return the container image name or path for a test distro name. - - The returned value is suitable for use with `docker pull`. - - >>> image_for_distro('centos5') - 'public.ecr.aws/n5z0e8q9/centos5-test' - >>> image_for_distro('centos5-something_custom') - 'public.ecr.aws/n5z0e8q9/centos5-test' - """ - return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],) - - -def make_containers(name_prefix='', port_offset=0): +def container_specs( + distros, + base_port=2200, + image_template=IMAGE_TEMPLATE, + name_template='target-%(distro)s-%(index)d', +): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian', 'centos6'] - >>> pprint.pprint(make_containers()) - [{'distro': 'debian', + >>> pprint.pprint(container_specs(['debian11-py3', 'centos6'])) + [{'distro': 'debian11', + 'family': 'debian', 'hostname': 'localhost', - 'name': 'target-debian-1', + 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'index': 1, + 'name': 'target-debian11-1', 'port': 2201, - 'python_path': '/usr/bin/python'}, + 'python_path': '/usr/bin/python3'}, {'distro': 'centos6', + 'family': 'centos', 'hostname': 'localhost', + 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'index': 2, 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() - firstbit = lambda s: (s+'-').split('-')[0] - secondbit = lambda s: (s+'-').split('-')[1] - + # Code duplicated in testlib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) i = 1 lst = [] - for distro in DISTROS: - distro, star, count = distro.partition('*') - if star: - count = int(count) + for distro in distros: + # Code duplicated in testlib.py, both should be updated together + d = distro_pattern.match(distro).groupdict(default=None) + + if d.pop('py') == 'py3': + python_path = '/usr/bin/python3' else: - count = 1 + python_path = '/usr/bin/python' + + count = int(d.pop('count') or '1', 10) for x in range(count): - lst.append({ - "distro": firstbit(distro), - "name": name_prefix + ("target-%s-%s" % (distro, i)), + d['index'] = i + d.update({ + 'image': image_template % d, + 'name': name_template % d, "hostname": docker_hostname, - "port": BASE_PORT + i + port_offset, - "python_path": ( - '/usr/bin/python3' - if secondbit(distro) == 'py3' - else '/usr/bin/python' - ) + 'port': base_port + i, + "python_path": python_path, }) + lst.append(d) i += 1 return lst @@ -348,18 +313,24 @@ def proc_is_docker(pid): def get_interesting_procs(container_name=None): + """ + Return a list of (pid, line) tuples for processes considered interesting. + """ args = ['ps', 'ax', '-oppid=', '-opid=', '-ocomm=', '-ocommand='] if container_name is not None: args = ['docker', 'exec', container_name] + args out = [] - for line in subprocess__check_output(args).decode().splitlines(): + for line in subprocess.check_output(args).decode().splitlines(): ppid, pid, comm, rest = line.split(None, 3) if ( ( any(comm.startswith(s) for s in INTERESTING_COMMS) or 'mitogen:' in rest ) and + ( + 'WALinuxAgent' not in rest + ) and ( container_name is not None or (not proc_is_docker(pid)) @@ -394,7 +365,7 @@ def start_containers(containers): "--publish 0.0.0.0:%(port)s:22/tcp " "--hostname=%(name)s " "--name=%(name)s " - "mitogen/%(distro)s-test " + "%(image)s" % container ] for container in containers @@ -409,12 +380,10 @@ def start_containers(containers): def verify_procs(hostname, old, new): oldpids = set(pid for pid, _ in old) if any(pid not in oldpids for pid, _ in new): - print('%r had stray processes running:' % (hostname,)) + print('%r had stray processes running:' % (hostname,), file=sys.stderr, flush=True) for pid, line in new: if pid not in oldpids: - print('New process:', line) - - print() + print('New process:', line, flush=True) return False return True @@ -438,13 +407,10 @@ def check_stray_processes(old, containers=None): def dump_file(path): - print() - print('--- %s ---' % (path,)) - print() + print('--- %s ---' % (path,), flush=True) with open(path, 'r') as fp: - print(fp.read().rstrip()) - print('---') - print() + print(fp.read().rstrip(), flush=True) + print('---', flush=True) # SSH passes these through to the container when run interactively, causing diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 0217c6845..825126c7d 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -2,18 +2,9 @@ import ci_lib -# Naturally DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] - ci_lib.run_batches([ [ - # Must be installed separately, as PyNACL indirect requirement causes - # newer version to be installed if done in a single pip run. - 'pip install "pycparser<2.19"', - 'pip install -qqq debops[ansible]==2.1.2 ansible==%s' % ci_lib.ANSIBLE_VERSION, - ], - [ - 'docker pull %s' % (ci_lib.image_for_distro('debian'),), + 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', ], ]) diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 976317044..b065486f8 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -1,16 +1,11 @@ #!/usr/bin/env python -from __future__ import print_function import os -import shutil import sys import ci_lib -# DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT - project_dir = os.path.join(ci_lib.TMP, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' @@ -18,7 +13,11 @@ with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') + containers = ci_lib.container_specs( + ['debian*2'], + base_port=2700, + name_template='debops-target-%(distro)s-%(index)d', + ) ci_lib.start_containers(containers) @@ -60,11 +59,7 @@ for container in containers ) - print() - print(' echo --- ansible/inventory/hosts: ---') - ci_lib.run('cat ansible/inventory/hosts') - print('---') - print() + ci_lib.dump_file('ansible/inventory/hosts') # Now we have real host key checking, we need to turn it off os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False' diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py deleted file mode 100755 index ddeb2ae18..000000000 --- a/.ci/localhost_ansible_install.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ - [ - # Must be installed separately, as PyNACL indirect requirement causes - # newer version to be installed if done in a single pip run. - # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml - # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. - 'pip install "pycparser<2.19" "idna<2.7" virtualenv', - 'pip install ' - '-r tests/requirements.txt ' - '-r tests/ansible/requirements.txt', - 'pip install -q ansible=={}'.format(ci_lib.ANSIBLE_VERSION) - ] -] - -ci_lib.run_batches(batches) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 6d7bef0d0..e4b8329ba 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -1,11 +1,13 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen +from __future__ import print_function + import os +import subprocess import sys import ci_lib -from ci_lib import run TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') @@ -24,33 +26,61 @@ # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", # there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually if not ci_lib.exists_in_path('sshpass'): - os.system("curl -O -L https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \ + subprocess.check_call( + "curl -O -L https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \ tar xvf sshpass-1.05.tar.gz && \ cd sshpass-1.05 && \ ./configure && \ - sudo make install") + sudo make install", + shell=True, + ) with ci_lib.Fold('machine_prep'): # generate a new ssh key for localhost ssh - os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa") - os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys") + if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")): + subprocess.check_call("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa", shell=True) + subprocess.check_call("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys", shell=True) + os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8)) + os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8)) + # also generate it for the sudo user - os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa") - os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys") - os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8)) - os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8)) - # run chmod through sudo since it's owned by root - os.system('sudo chmod 600 /var/root/.ssh') - os.system('sudo chmod 600 /var/root/.ssh/authorized_keys') + if os.system("sudo [ -f ~root/.ssh/id_rsa ]") != 0: + subprocess.check_call("sudo ssh-keygen -P '' -m pem -f ~root/.ssh/id_rsa", shell=True) + subprocess.check_call("sudo cat ~root/.ssh/id_rsa.pub | sudo tee -a ~root/.ssh/authorized_keys", shell=True) + subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True) + subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True) + + os.chdir(IMAGE_PREP_DIR) + ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml") if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': os.chdir(IMAGE_PREP_DIR) - run("ansible-playbook -c local -i localhost, _user_accounts.yml -vvv") + ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") + + cmd = ';'.join([ + 'from __future__ import print_function', + 'import os, sys', + 'print(sys.executable, os.path.realpath(sys.executable))', + ]) + for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']: + print(interpreter) + try: + subprocess.call([interpreter, '-c', cmd]) + except OSError as exc: + print(exc) + + print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1') + environ = os.environ.copy() + environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1' + try: + subprocess.call([interpreter, '-c', cmd], env=environ) + except OSError as exc: + print(exc) with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') - run('./run_ansible_playbook.py %s -l target %s -vvv', + ci_lib.run('./run_ansible_playbook.py %s %s', playbook, ' '.join(sys.argv[1:])) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py deleted file mode 100755 index b8862f89c..000000000 --- a/.ci/mitogen_install.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ - [ - 'pip install "pycparser<2.19" "idna<2.7"', - 'pip install -r tests/requirements.txt', - ] -] - -if ci_lib.have_docker(): - batches.append([ - 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), - ]) - - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index 868ae4e4a..85ea013c2 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -3,9 +3,6 @@ import ci_lib batches = [ - [ - 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), - ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', ] diff --git a/.ci/mitogen_py24_tests.py b/.ci/mitogen_py24_tests.py index 228e79bdd..96b144eb3 100755 --- a/.ci/mitogen_py24_tests.py +++ b/.ci/mitogen_py24_tests.py @@ -8,8 +8,6 @@ os.environ.update({ 'NOCOVERAGE': '1', 'UNIT2': '/usr/local/python2.4.6/bin/unit2', - - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 4de94b4c3..47aa2444f 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -6,7 +6,6 @@ import ci_lib os.environ.update({ - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py deleted file mode 100755 index 80dbf4853..000000000 --- a/.ci/prep_azure.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -import os -import sys - -import ci_lib - -batches = [] - -if 0 and os.uname()[0] == 'Linux': - batches += [ - [ - "sudo chown `whoami`: ~", - "chmod u=rwx,g=rx,o= ~", - - "sudo mkdir /var/run/sshd", - "sudo /etc/init.d/ssh start", - - "mkdir -p ~/.ssh", - "chmod u=rwx,go= ~/.ssh", - - "ssh-keyscan -H localhost >> ~/.ssh/known_hosts", - "chmod u=rw,go= ~/.ssh/known_hosts", - - "cat tests/data/docker/mitogen__has_sudo_pubkey.key > ~/.ssh/id_rsa", - "chmod u=rw,go= ~/.ssh/id_rsa", - - "cat tests/data/docker/mitogen__has_sudo_pubkey.key.pub > ~/.ssh/authorized_keys", - "chmod u=rw,go=r ~/.ssh/authorized_keys", - ] - ] - -# setup venv, need all python commands in 1 list to be subprocessed at the same time -venv_steps = [] - -need_to_fix_psycopg2 = False - -is_python3 = os.environ['PYTHONVERSION'].startswith('3') - -# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage, -# broken symlinks, incorrect permissions and missing codecs. So we use the -# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our -# stuff into. The virtualenv can probably be removed again, but this was a -# hard-fought battle and for now I am tired of this crap. -if ci_lib.have_apt(): - venv_steps.extend([ - 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', - 'sudo add-apt-repository ppa:deadsnakes/ppa', - 'sudo apt-get update', - 'sudo apt-get -y install ' - 'python{pv} ' - 'python{pv}-dev ' - 'libsasl2-dev ' - 'libldap2-dev ' - .format(pv=os.environ['PYTHONVERSION']), - 'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}' - .format(pv=os.environ['PYTHONVERSION']) - ]) - if is_python3: - venv_steps.append('sudo apt-get -y install python{pv}-venv'.format(pv=os.environ['PYTHONVERSION'])) -# TODO: somehow `Mito36CentOS6_26` has both brew and apt installed https://dev.azure.com/dw-mitogen/Mitogen/_build/results?buildId=1031&view=logs&j=7bdbcdc6-3d3e-568d-ccf8-9ddca1a9623a&t=73d379b6-4eea-540f-c97e-046a2f620483 -elif is_python3 and ci_lib.have_brew(): - # Mac's System Integrity Protection prevents symlinking /usr/bin - # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html - # so we'll use /usr/local/bin/python for everything - # /usr/local/bin/python2.7 already exists! - need_to_fix_psycopg2 = True - venv_steps.append( - 'brew install python@{pv} postgresql' - .format(pv=os.environ['PYTHONVERSION']) - ) - -# need wheel before building virtualenv because of bdist_wheel and setuptools deps -venv_steps.append('/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools'.format(pv=os.environ['PYTHONVERSION'])) - -if os.environ['PYTHONVERSION'].startswith('2'): - venv_steps.extend([ - '/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(pv=os.environ['PYTHONVERSION']), - '/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(pv=os.environ['PYTHONVERSION']) - ]) -else: - venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(pv=os.environ['PYTHONVERSION'])) -# fixes https://stackoverflow.com/questions/59595649/can-not-install-psycopg2-on-macos-catalina https://github.com/Azure/azure-cli/issues/12854#issuecomment-619213863 -if need_to_fix_psycopg2: - venv_steps.append('/tmp/venv/bin/pip3 install psycopg2==2.8.5 psycopg2-binary') - -batches.append(venv_steps) - -ci_lib.run_batches(batches) diff --git a/.ci/spawn_reverse_shell.py b/.ci/spawn_reverse_shell.py deleted file mode 100755 index 8a6b95004..000000000 --- a/.ci/spawn_reverse_shell.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -""" -Allow poking around Azure while the job is running. -""" - -import os -import pty -import socket -import subprocess -import sys -import time - - -if os.fork(): - sys.exit(0) - - -def try_once(): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(("k3.botanicus.net", 9494)) - open('/tmp/interactive', 'w').close() - - os.dup2(s.fileno(), 0) - os.dup2(s.fileno(), 1) - os.dup2(s.fileno(), 2) - p = pty.spawn("/bin/sh") - - -while True: - try: - try_once() - except: - time.sleep(5) - continue - diff --git a/.ci/travis.sh b/.ci/travis.sh deleted file mode 100755 index 8bab72876..000000000 --- a/.ci/travis.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# workaround from https://stackoverflow.com/a/26082445 to handle Travis 4MB log limit -set -e - -export PING_SLEEP=30s -export WORKDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export BUILD_OUTPUT=$WORKDIR/build.out - -touch $BUILD_OUTPUT - -dump_output() { - echo Tailing the last 1000 lines of output: - tail -1000 $BUILD_OUTPUT -} -error_handler() { - echo ERROR: An error was encountered with the build. - dump_output - kill $PING_LOOP_PID - exit 1 -} -# If an error occurs, run our error handler to output a tail of the build -trap 'error_handler' ERR - -# Set up a repeating loop to send some output to Travis. - -bash -c "while true; do echo \$(date) - building ...; sleep $PING_SLEEP; done" & -PING_LOOP_PID=$! - -.ci/${MODE}_tests.py >> $BUILD_OUTPUT 2>&1 - -# The build finished without returning an error so dump a tail of the output -dump_output - -# nicely terminate the ping output loop -kill $PING_LOOP_PID diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..9a275a00c --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,270 @@ +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions + +name: Tests + +on: + pull_request: + push: + branches-ignore: + - docs-master + +env: + #ANSIBLE_VERBOSITY: 3 + #MITOGEN_LOG_LEVEL: DEBUG + MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" + +# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners +# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support +jobs: + linux: + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + include: + - name: Ans_27_210 + tox_env: py27-mode_ansible-ansible2.10 + - name: Ans_27_4 + tox_env: py27-mode_ansible-ansible4 + + - name: Ans_36_210 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible2.10 + - name: Ans_36_4 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible4 + + - name: Ans_311_210 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible2.10 + - name: Ans_311_3 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible3 + - name: Ans_311_4 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible4 + - name: Ans_311_5 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible5 + - name: Ans_313_6 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible6 + - name: Ans_313_7 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible7 + - name: Ans_313_8 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible8 + - name: Ans_313_9 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible9 + - name: Ans_313_10 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible10 + - name: Ans_313_11 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible11 + + - name: Van_313_11 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible11-strategy_linear + + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_36 + python_version: '3.6' + tox_env: py36-mode_mitogen + - name: Mito_313 + python_version: '3.13' + tox_env: py313-mode_mitogen + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install build deps + run: | + set -o errexit -o nounset -o pipefail + + sudo apt-get update + sudo apt-get install -y python2-dev python3-pip virtualenv + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + macos: + # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md + runs-on: macos-13 + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + include: + - name: Mito_313 + tox_env: py313-mode_mitogen + + - name: Loc_313_11 + tox_env: py313-mode_localhost-ansible11 + + - name: Van_313_11 + tox_env: py313-mode_localhost-ansible11-strategy_linear + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + # https://github.com/marketplace/actions/alls-green + check: + if: always() + needs: + - linux + - macos + runs-on: ubuntu-latest + steps: + - uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.gitignore b/.gitignore index aa75f6917..7297d7204 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ venvs/** *.pyc *.pyd *.pyo +*.retry MANIFEST build/ dist/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a7a9422da..000000000 --- a/.travis.yml +++ /dev/null @@ -1,86 +0,0 @@ -sudo: required -dist: xenial # Ubuntu 16.04 LTS - -notifications: - email: false - irc: "chat.freenode.net#mitogen-builds" - -language: python - -branches: - except: - - docs-master - -cache: -- pip -- directories: - - /home/travis/virtualenv - -install: -- pip install -U pip==20.2.1 -- .ci/${MODE}_install.py - -# Travis has a 4MB log limit (https://github.com/travis-ci/travis-ci/issues/1382), but verbose Mitogen logs run larger than that -# in order to keep verbosity to debug a build failure, will run with this workaround: https://stackoverflow.com/a/26082445 -script: -- .ci/spawn_reverse_shell.py -- MODE=${MODE} .ci/travis.sh - -# To avoid matrix explosion, just test against oldest->newest and -# newest->oldest in various configuartions. - -matrix: - include: - # Debops tests. - # NOTE: debops tests turned off for Ansible 2.10: https://github.com/debops/debops/issues/1521 - # 2.10; 3.6 -> 2.7 - # - python: "3.6" - # env: MODE=debops_common VER=2.10.0 - # 2.10; 2.7 -> 2.7 - # - python: "2.7" - # env: MODE=debops_common VER=2.10.0 - - # Sanity check against vanilla Ansible. One job suffices. - # https://github.com/dw/mitogen/pull/715#issuecomment-719266420 migrating to Azure for now due to Travis 50 min time limit cap - # azure lets us adjust the cap, and the current STRATEGY=linear tests take up to 1.5 hours to finish - # - python: "2.7" - # env: MODE=ansible VER=2.10.0 DISTROS=debian STRATEGY=linear - - # ansible_mitogen tests. - - # 2.10 -> {debian, centos6, centos7} - - python: "3.6" - env: MODE=ansible VER=2.10.0 - # 2.10 -> {debian, centos6, centos7} - - python: "3.9" - env: MODE=ansible VER=2.10.0 - # 2.10 -> {debian, centos6, centos7} - - python: "2.7" - env: MODE=ansible VER=2.10.0 - # 2.10 -> {debian, centos6, centos7} - # - python: "2.6" - # env: MODE=ansible VER=2.10.0 - - # 2.10 -> {centos5} - # - python: "2.6" - # env: MODE=ansible DISTROS=centos5 VER=2.10.0 - - # Mitogen tests. - # 2.4 -> 2.4 - # - language: c - # env: MODE=mitogen_py24 DISTROS=centos5 VER=2.10.0 - # 2.7 -> 2.7 -- moved to Azure - # 2.7 -> 2.6 - #- python: "2.7" - #env: MODE=mitogen DISTRO=centos6 - - python: "3.6" - env: MODE=mitogen DISTRO=centos7 - - python: "3.9" - env: MODE=mitogen DISTRO=centos7 - # 2.6 -> 2.7 - # - python: "2.6" - # env: MODE=mitogen DISTRO=centos7 - # 2.6 -> 3.5 - # - python: "2.6" - # env: MODE=mitogen DISTRO=debian-py3 - # 3.6 -> 2.6 -- moved to Azure diff --git a/README.md b/README.md index 85beacb48..c3cd9a87d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # Mitogen - +[![PyPI - Version](https://img.shields.io/pypi/v/mitogen)](https://pypi.org/project/mitogen/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mitogen)](https://pypi.org/project/mitogen/) +[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster) + Please see the documentation. ![](https://i.imgur.com/eBM6LhJ.gif) [![Total alerts](https://img.shields.io/lgtm/alerts/g/mitogen-hq/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mitogen-hq/mitogen/alerts/) - -[![Build Status](https://api.travis-ci.com/mitogen-hq/mitogen.svg?branch=master)](https://api.travis-ci.com/mitogen-hq/mitogen) - -[![Pipelines Status](https://dev.azure.com/mitogen-hq/mitogen/_apis/build/status/mitogen-hq.mitogen?branchName=master)](https://dev.azure.com/mitogen-hq/mitogen/_build/latest?definitionId=1&branchName=master) diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index 7f4c8db56..223794abb 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -73,7 +73,9 @@ decisions. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import ctypes import logging import mmap @@ -81,7 +83,6 @@ import os import struct -import mitogen.core import mitogen.parent @@ -263,7 +264,7 @@ def _mask_to_bytes(self, mask): for x in range(16): chunks.append(struct.pack('>= 64 - return mitogen.core.b('').join(chunks) + return b''.join(chunks) def _get_thread_ids(self): try: diff --git a/ansible_mitogen/compat/simplejson/__init__.py b/ansible_mitogen/compat/simplejson/__init__.py deleted file mode 100644 index d5b4d3991..000000000 --- a/ansible_mitogen/compat/simplejson/__init__.py +++ /dev/null @@ -1,318 +0,0 @@ -r"""JSON (JavaScript Object Notation) is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print json.dumps("\"foo\bar") - "\"foo\bar" - >>> print json.dumps(u'\u1234') - "\u1234" - >>> print json.dumps('\\') - "\\" - >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) - {"a": 0, "b": 0, "c": 0} - >>> from StringIO import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) - >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from StringIO import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> import decimal - >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) -""" -__version__ = '2.0.9' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONEncoder', -] - -__author__ = 'Bob Ippolito ' - -from decoder import JSONDecoder -from encoder import JSONEncoder - -_default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, -) - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the some chunks written to ``fp`` - may be ``unicode`` instances, subject to normal Python ``str`` to - ``unicode`` coercion rules. Unless ``fp.write()`` explicitly - understands ``unicode`` (as in ``codecs.getwriter()``) this is likely - to cause an error. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) - in strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a non-negative integer, then JSON array elements and object - members will be pretty-printed with that indent level. An indent level - of 0 will only insert newlines. ``None`` is the most compact representation. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not kw): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is false then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the return value will be a - ``unicode`` instance subject to normal Python ``str`` to ``unicode`` - coercion rules instead of being escaped to an ASCII ``str``. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in - strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a non-negative integer, then JSON array elements and - object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not kw): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - **kw).encode(obj) - - -_default_decoder = JSONDecoder(encoding=None, object_hook=None) - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document) to a Python object. - - If the contents of ``fp`` is encoded with an ASCII based encoding other - than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must - be specified. Encodings that are not ASCII based (such as UCS-2) are - not allowed, and should be wrapped with - ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode`` - object and passed to ``loads()`` - - ``object_hook`` is an optional function that will be called with the - result of any object literal decode (a ``dict``). The return value of - ``object_hook`` will be used instead of the ``dict``. This feature - can be used to implement custom decoders (e.g. JSON-RPC class hinting). - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding - other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name - must be specified. Encodings that are not ASCII based (such as UCS-2) - are not allowed and should be decoded to ``unicode`` first. - - ``object_hook`` is an optional function that will be called with the - result of any object literal decode (a ``dict``). The return value of - ``object_hook`` will be used instead of the ``dict``. This feature - can be used to implement custom decoders (e.g. JSON-RPC class hinting). - - ``parse_float``, if specified, will be called with the string - of every JSON float to be decoded. By default this is equivalent to - float(num_str). This can be used to use another datatype or parser - for JSON floats (e.g. decimal.Decimal). - - ``parse_int``, if specified, will be called with the string - of every JSON int to be decoded. By default this is equivalent to - int(num_str). This can be used to use another datatype or parser - for JSON integers (e.g. float). - - ``parse_constant``, if specified, will be called with one of the - following strings: -Infinity, Infinity, NaN, null, true, false. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - return cls(encoding=encoding, **kw).decode(s) diff --git a/ansible_mitogen/compat/simplejson/decoder.py b/ansible_mitogen/compat/simplejson/decoder.py deleted file mode 100644 index b769ea486..000000000 --- a/ansible_mitogen/compat/simplejson/decoder.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Implementation of JSONDecoder -""" -import re -import sys -import struct - -from simplejson.scanner import make_scanner -try: - from simplejson._speedups import scanstring as c_scanstring -except ImportError: - c_scanstring = None - -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - if sys.byteorder != 'big': - _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] - nan, inf = struct.unpack('dd', _BYTES) - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - # Note that this function is called from _speedups - lineno, colno = linecol(doc, pos) - if end is None: - #fmt = '{0}: line {1} column {2} (char {3})' - #return fmt.format(msg, lineno, colno, pos) - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' - #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise ValueError( - errmsg("Unterminated string starting at", s, begin)) - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" % (terminator,) - #msg = "Invalid control character {0!r} at".format(terminator) - raise ValueError(errmsg(msg, s, end)) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise ValueError( - errmsg("Unterminated string starting at", s, begin)) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\escape: " + repr(esc) - raise ValueError(errmsg(msg, s, end)) - end += 1 - else: - # Unicode escape sequence - esc = s[end + 1:end + 5] - next_end = end + 5 - if len(esc) != 4: - msg = "Invalid \\uXXXX escape" - raise ValueError(errmsg(msg, s, end)) - uni = int(esc, 16) - # Check for surrogate pair on UCS-4 systems - if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: - msg = "Invalid \\uXXXX\\uXXXX surrogate pair" - if not s[end + 5:end + 7] == '\\u': - raise ValueError(errmsg(msg, s, end)) - esc2 = s[end + 7:end + 11] - if len(esc2) != 4: - raise ValueError(errmsg(msg, s, end)) - uni2 = int(esc2, 16) - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) - next_end += 6 - char = unichr(uni) - end = next_end - # Append the unescaped character - _append(char) - return u''.join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - pairs = {} - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - return pairs, end + 1 - elif nextchar != '"': - raise ValueError(errmsg("Expecting property name", s, end)) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise ValueError(errmsg("Expecting : delimiter", s, end)) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - try: - value, end = scan_once(s, end) - except StopIteration: - raise ValueError(errmsg("Expecting object", s, end)) - pairs[key] = value - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise ValueError(errmsg("Expecting , delimiter", s, end - 1)) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise ValueError(errmsg("Expecting property name", s, end - 1)) - - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - _append = values.append - while True: - try: - value, end = scan_once(s, end) - except StopIteration: - raise ValueError(errmsg("Expecting object", s, end)) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise ValueError(errmsg("Expecting , delimiter", s, end)) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True): - """``encoding`` determines the encoding used to interpret any ``str`` - objects decoded by this instance (utf-8 by default). It has no - effect when decoding ``unicode`` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as ``unicode``. - - ``object_hook``, if specified, will be called with the result - of every JSON object decoded and its return value will be used in - place of the given ``dict``. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - ``parse_float``, if specified, will be called with the string - of every JSON float to be decoded. By default this is equivalent to - float(num_str). This can be used to use another datatype or parser - for JSON floats (e.g. decimal.Decimal). - - ``parse_int``, if specified, will be called with the string - of every JSON int to be decoded. By default this is equivalent to - int(num_str). This can be used to use another datatype or parser - for JSON integers (e.g. float). - - ``parse_constant``, if specified, will be called with one of the - following strings: -Infinity, Infinity, NaN. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - """ - self.encoding = encoding - self.object_hook = object_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or int - self.parse_constant = parse_constant or _CONSTANTS.__getitem__ - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - obj, end = self.raw_decode(s, idx=_w(s, 0).end()) - end = _w(s, end).end() - if end != len(s): - raise ValueError(errmsg("Extra data", s, end, len(s))) - return obj - - def raw_decode(self, s, idx=0): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning - with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - try: - obj, end = self.scan_once(s, idx) - except StopIteration: - raise ValueError("No JSON object could be decoded") - return obj, end diff --git a/ansible_mitogen/compat/simplejson/encoder.py b/ansible_mitogen/compat/simplejson/encoder.py deleted file mode 100644 index cf5829036..000000000 --- a/ansible_mitogen/compat/simplejson/encoder.py +++ /dev/null @@ -1,440 +0,0 @@ -"""Implementation of JSONEncoder -""" -import re - -try: - from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii -except ImportError: - c_encode_basestring_ascii = None -try: - from simplejson._speedups import make_encoder as c_make_encoder -except ImportError: - c_make_encoder = None - -ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) - -# Assume this produces an infinity on all machines (probably not guaranteed) -INFINITY = float('1e66666') -FLOAT_REPR = repr - -def encode_basestring(s): - """Return a JSON representation of a Python string - - """ - def replace(match): - return ESCAPE_DCT[match.group(0)] - return '"' + ESCAPE.sub(replace, s) + '"' - - -def py_encode_basestring_ascii(s): - """Return an ASCII-only JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii - -class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true, then NaN, Infinity, and -Infinity will be - encoded as such. This behavior is not JSON specification compliant, - but is consistent with most JavaScript based encoders and decoders. - Otherwise, it will be a ValueError to encode such floats. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a non-negative integer, then JSON array - elements and object members will be pretty-printed with that - indent level. An indent level of 0 will only insert newlines. - None is the most compact representation. - - If specified, separators should be a (item_separator, key_separator) - tuple. The default is (', ', ': '). To get the most compact JSON - representation you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError(repr(o) + " is not JSON serializable") - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, basestring): - if isinstance(o, str): - _encoding = self.encoding - if (_encoding is not None - and not (_encoding == 'utf-8')): - o = o.decode(_encoding) - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - return ''.join(chunks) - - def iterencode(self, o, _one_shot=False): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, str): - o = o.decode(_encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): - # Check for specials. Note that this type of test is processor- and/or - # platform-specific, so do tests which don't depend on the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - - if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys: - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot) - return _iterencode(o, 0) - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - ## HACK: hand-optimized bytecode; turn globals into locals - False=False, - True=True, - ValueError=ValueError, - basestring=basestring, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - long=long, - str=str, - tuple=tuple, - ): - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, basestring): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, (int, long)): - yield buf + str(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (' ' * (_indent * _current_indent_level)) - yield ']' - if markers is not None: - del markers[markerid] - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _sort_keys: - items = dct.items() - items.sort(key=lambda kv: kv[0]) - else: - items = dct.iteritems() - for key, value in items: - if isinstance(key, basestring): - pass - # JavaScript is weakly typed for these, so it makes sense to - # also allow them. Many encoders seem to do something like this. - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, (int, long)): - key = str(key) - elif _skipkeys: - continue - else: - raise TypeError("key " + repr(key) + " is not a string") - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, basestring): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, (int, long)): - yield str(value) - elif isinstance(value, float): - yield _floatstr(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (' ' * (_indent * _current_indent_level)) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, basestring): - yield _encoder(o) - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, (int, long)): - yield str(o) - elif isinstance(o, float): - yield _floatstr(o) - elif isinstance(o, (list, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - else: - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/ansible_mitogen/compat/simplejson/scanner.py b/ansible_mitogen/compat/simplejson/scanner.py deleted file mode 100644 index adbc6ec97..000000000 --- a/ansible_mitogen/compat/simplejson/scanner.py +++ /dev/null @@ -1,65 +0,0 @@ -"""JSON token scanner -""" -import re -try: - from simplejson._speedups import make_scanner as c_make_scanner -except ImportError: - c_make_scanner = None - -__all__ = ['make_scanner'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - - def _scan_once(string, idx): - try: - nextchar = string[idx] - except IndexError: - raise StopIteration - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise StopIteration - - return _scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index e4d259546..2a4e6c1f4 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -26,8 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function from __future__ import unicode_literals +__metaclass__ = type import errno import logging @@ -40,11 +41,8 @@ import ansible.constants as C import ansible.errors import ansible.plugins.connection -import ansible.utils.shlex import mitogen.core -import mitogen.fork -import mitogen.utils import ansible_mitogen.mixins import ansible_mitogen.parsing @@ -52,6 +50,7 @@ import ansible_mitogen.services import ansible_mitogen.target import ansible_mitogen.transport_config +import ansible_mitogen.utils.unsafe LOG = logging.getLogger(__name__) @@ -120,7 +119,7 @@ def _connect_ssh(spec): """ Return ContextService arguments for an SSH connection. """ - if C.HOST_KEY_CHECKING: + if spec.host_key_checking(): check_host_keys = 'enforce' else: check_host_keys = 'ignore' @@ -146,7 +145,7 @@ def _connect_ssh(spec): 'identity_file': private_key_file, 'identities_only': False, 'ssh_path': spec.ssh_executable(), - 'connect_timeout': spec.ansible_ssh_timeout(), + 'connect_timeout': spec.timeout(), 'ssh_args': spec.ssh_args(), 'ssh_debug_level': spec.mitogen_ssh_debug_level(), 'remote_name': get_remote_name(spec), @@ -159,6 +158,7 @@ def _connect_ssh(spec): } } + def _connect_buildah(spec): """ Return ContextService arguments for a Buildah connection. @@ -169,11 +169,12 @@ def _connect_buildah(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } + def _connect_docker(spec): """ Return ContextService arguments for a Docker connection. @@ -184,7 +185,7 @@ def _connect_docker(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(rediscover_python=True), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -199,7 +200,7 @@ def _connect_kubectl(spec): 'kwargs': { 'pod': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'kubectl_path': spec.mitogen_kubectl_path(), 'kubectl_args': spec.extra_args(), 'remote_name': get_remote_name(spec), @@ -217,7 +218,7 @@ def _connect_jail(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -233,7 +234,7 @@ def _connect_lxc(spec): 'container': spec.remote_addr(), 'python_path': spec.python_path(), 'lxc_attach_path': spec.mitogen_lxc_attach_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -249,7 +250,7 @@ def _connect_lxd(spec): 'container': spec.remote_addr(), 'python_path': spec.python_path(), 'lxc_path': spec.mitogen_lxc_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -262,6 +263,22 @@ def _connect_machinectl(spec): return _connect_setns(spec, kind='machinectl') +def _connect_podman(spec): + """ + Return ContextService arguments for a Docker connection. + """ + return { + 'method': 'podman', + 'kwargs': { + 'username': spec.remote_user(), + 'container': spec.remote_addr(), + 'python_path': spec.python_path(rediscover_python=True), + 'connect_timeout': spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + def _connect_setns(spec, kind=None): """ Return ContextService arguments for a mitogen_setns connection. @@ -400,6 +417,7 @@ def _connect_mitogen_doas(spec): 'lxc': _connect_lxc, 'lxd': _connect_lxd, 'machinectl': _connect_machinectl, + 'podman': _connect_podman, 'setns': _connect_setns, 'ssh': _connect_ssh, 'smart': _connect_ssh, # issue #548. @@ -469,6 +487,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): login_context = None #: Only sudo, su, and doas are supported for now. + # Ansible ConnectionBase attribute, removed in Ansible >= 2.8 become_methods = ['sudo', 'su', 'doas'] #: Dict containing init_child() return value as recorded at startup by @@ -506,15 +525,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): # set by `_get_task_vars()` for interpreter discovery _action = None - def __del__(self): - """ - Ansible cannot be trusted to always call close() e.g. the synchronize - action constructs a local connection like this. So provide a destructor - in the hopes of catching these cases. - """ - # https://github.com/dw/mitogen/issues/140 - self.close() - def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir): """ Invoked by ActionModuleMixin to indicate a new task is about to start @@ -670,6 +680,9 @@ def get_binding(self): @property def connected(self): + """ + Ansible connection plugin property. Used by ansible-connection command. + """ return self.context is not None def _spec_from_via(self, proxied_inventory_name, via_spec): @@ -788,7 +801,7 @@ def _connect_stack(self, stack): call_context=self.binding.get_service_context(), service_name='ansible_mitogen.services.ContextService', method_name='get', - stack=mitogen.utils.cast(list(stack)), + stack=ansible_mitogen.utils.unsafe.cast(list(stack)), ) except mitogen.core.CallError: LOG.warning('Connection failed; stack configuration was:\n%s', @@ -802,7 +815,7 @@ def _connect_stack(self, stack): self.context = dct['context'] self.chain = CallChain(self, self.context, pipelined=True) - if self._play_context.become: + if self.become: self.login_context = dct['via'] else: self.login_context = self.context @@ -828,14 +841,18 @@ def _connect(self): the _connect_*() service calls defined above to cause the master process to establish the real connection on our behalf, or return a reference to the existing one. + + Ansible connection plugin method. """ + # In some Ansible connection plugins this method returns self. + # However nothing I've found uses it, it's not even assigned. if self.connected: return inventory_name, stack = self._build_stack() worker_model = ansible_mitogen.process.get_worker_model() self.binding = worker_model.get_binding( - mitogen.utils.cast(inventory_name) + ansible_mitogen.utils.unsafe.cast(inventory_name) ) self._connect_stack(stack) @@ -866,12 +883,37 @@ def close(self): Arrange for the mitogen.master.Router running in the worker to gracefully shut down, and wait for shutdown to complete. Safe to call multiple times. + + Ansible connection plugin method. """ self._put_connection() if self.binding: self.binding.close() self.binding = None + def _mitogen_var_options(self, templar): + # Workaround for https://github.com/ansible/ansible/issues/84238 + var_names = C.config.get_plugin_vars('connection', self._load_name) + variables = templar.available_variables + var_options = { + var_name: templar.template(variables[var_name]) + for var_name in var_names + if var_name in variables + } + + if self.allow_extras: + extras_var_prefix = 'ansible_%s_' % self.extras_prefix + var_options['_extras'] = { + var_name: templar.template(variables[var_name]) + for var_name in variables + if var_name not in var_options + and var_name.startswith(extras_var_prefix) + } + else: + var_options['_extras'] = {} + + return var_options + reset_compat_msg = ( 'Mitogen only supports "reset_connection" on Ansible 2.5.6 or later' ) @@ -882,6 +924,8 @@ def reset(self): any local state we hold for the connection, returns the Connection to the 'disconnected' state, and informs ContextService the connection is bad somehow, and should be shut down and discarded. + + Ansible connection plugin method. """ if self._play_context.remote_addr is None: # <2.5.6 incorrectly populate PlayContext for reset_connection @@ -890,23 +934,44 @@ def reset(self): self.reset_compat_msg ) - # Strategy's _execute_meta doesn't have an action obj but we'll need one for - # running interpreter_discovery - # will create a new temporary action obj for this purpose - self._action = ansible_mitogen.mixins.ActionModuleMixin( - task=0, - connection=self, - play_context=self._play_context, - loader=0, - templar=0, - shared_loader_obj=0 - ) + # Handle templated connection variables during `meta: reset_connection`. + # Many bugs/implementation details of Mitogen & Ansible collide here. + # See #1079, #1096, #1132, ansible/ansible#84238, ... + try: + task, templar = self._play_context.vars.pop( + '_mitogen.smuggled.reset_connection', + ) + except KeyError: + self._action_monkey_patched_by_mitogen = False + else: + # LOG.info('%r.reset(): remote_addr=%r', self, self._play_context.remote_addr) + # ansible.plugins.strategy.StrategyBase._execute_meta() doesn't + # have an action object, which we need for interpreter_discovery. + # Create a temporary action object for this purpose. + self._action = ansible_mitogen.mixins.ActionModuleMixin( + task=task, + connection=self, + play_context=self._play_context, + loader=templar._loader, + templar=templar, + shared_loader_obj=0, + ) + self._action_monkey_patched_by_mitogen = True + + # Workaround for https://github.com/ansible/ansible/issues/84238 + self.set_options( + task_keys=task.dump_attrs(), + var_options=self._mitogen_var_options(templar), + ) + + del task + del templar # Clear out state in case we were ever connected. self.close() inventory_name, stack = self._build_stack() - if self._play_context.become: + if self.become: stack = stack[:-1] worker_model = ansible_mitogen.process.get_worker_model() @@ -916,11 +981,16 @@ def reset(self): call_context=binding.get_service_context(), service_name='ansible_mitogen.services.ContextService', method_name='reset', - stack=mitogen.utils.cast(list(stack)), + stack=ansible_mitogen.utils.unsafe.cast(list(stack)), ) finally: binding.close() + # Cleanup any monkey patching we did for `meta: reset_connection` + if self._action_monkey_patched_by_mitogen: + del self._action + del self._action_monkey_patched_by_mitogen + # Compatibility with Ansible 2.4 wait_for_connection plug-in. _reset = reset @@ -988,12 +1058,14 @@ def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None): Data to supply on ``stdin`` of the process. :returns: (return code, stdout bytes, stderr bytes) + + Ansible connection plugin method. """ emulate_tty = (not in_data and sudoable) rc, stdout, stderr = self.get_chain().call( ansible_mitogen.target.exec_command, - cmd=mitogen.utils.cast(cmd), - in_data=mitogen.utils.cast(in_data), + cmd=ansible_mitogen.utils.unsafe.cast(cmd), + in_data=ansible_mitogen.utils.unsafe.cast(in_data), chdir=mitogen_chdir or self.get_default_cwd(), emulate_tty=emulate_tty, ) @@ -1013,12 +1085,14 @@ def fetch_file(self, in_path, out_path): Remote filesystem path to read. :param str out_path: Local filesystem path to write. + + Ansible connection plugin method. """ self._connect() ansible_mitogen.target.transfer_file( context=self.context, # in_path may be AnsibleUnicode - in_path=mitogen.utils.cast(in_path), + in_path=ansible_mitogen.utils.unsafe.cast(in_path), out_path=out_path ) @@ -1036,7 +1110,7 @@ def put_data(self, out_path, data, mode=None, utimes=None): """ self.get_chain().call_no_reply( ansible_mitogen.target.write_path, - mitogen.utils.cast(out_path), + ansible_mitogen.utils.unsafe.cast(out_path), mitogen.core.Blob(data), mode=mode, utimes=utimes, @@ -1062,6 +1136,8 @@ def put_file(self, in_path, out_path): Local filesystem path to read. :param str out_path: Remote filesystem path to write. + + Ansible connection plugin method. """ try: st = os.stat(in_path) @@ -1082,7 +1158,7 @@ def put_file(self, in_path, out_path): s = fp.read(self.SMALL_FILE_LIMIT + 1) finally: fp.close() - except OSError: + except OSError as e: self._throw_io_error(e, in_path) raise @@ -1096,7 +1172,7 @@ def put_file(self, in_path, out_path): call_context=self.binding.get_service_context(), service_name='mitogen.service.FileService', method_name='register', - path=mitogen.utils.cast(in_path) + path=ansible_mitogen.utils.unsafe.cast(in_path) ) # For now this must remain synchronous, as the action plug-in may have @@ -1106,6 +1182,6 @@ def put_file(self, in_path, out_path): self.get_chain().call( ansible_mitogen.target.transfer_file, context=self.binding.get_child_service_context(), - in_path=in_path, - out_path=out_path + in_path=ansible_mitogen.utils.unsafe.cast(in_path), + out_path=ansible_mitogen.utils.unsafe.cast(out_path) ) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index c00915d52..632b11b14 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -30,11 +30,16 @@ Stable names for PluginLoader instances across Ansible versions. """ -from __future__ import absolute_import -import distutils.version +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import ansible.errors + +import ansible_mitogen.utils __all__ = [ 'action_loader', + 'become_loader', 'connection_loader', 'module_loader', 'module_utils_loader', @@ -42,10 +47,9 @@ 'strategy_loader', ] -import ansible ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 10) +ANSIBLE_VERSION_MAX = (2, 18) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" @@ -68,10 +72,7 @@ def assert_supported_release(): Throw AnsibleError with a descriptive message in case of being loaded into an unsupported Ansible release. """ - v = ansible.__version__ - if not isinstance(v, tuple): - v = tuple(distutils.version.LooseVersion(v).version) - + v = ansible_mitogen.utils.ansible_version if v[:2] < ANSIBLE_VERSION_MIN: raise ansible.errors.AnsibleError( OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN) @@ -79,7 +80,7 @@ def assert_supported_release(): if v[:2] > ANSIBLE_VERSION_MAX: raise ansible.errors.AnsibleError( - NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX) + NEW_VERSION_MSG % (v, ANSIBLE_VERSION_MAX) ) @@ -90,6 +91,7 @@ def assert_supported_release(): from ansible.plugins.loader import action_loader +from ansible.plugins.loader import become_loader from ansible.plugins.loader import connection_loader from ansible.plugins.loader import module_loader from ansible.plugins.loader import module_utils_loader diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 00a701842..4d5647a45 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -26,19 +26,19 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import logging import os +import ansible.utils.display + import mitogen.core import mitogen.utils -try: - from __main__ import display -except ImportError: - from ansible.utils.display import Display - display = Display() +display = ansible.utils.display.Display() #: The process name set via :func:`set_process_name`. _process_name = None diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 7e7a3ff09..3953eb52e 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -26,50 +26,32 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import logging import os import pwd import random import traceback -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - -from ansible.module_utils._text import to_bytes -from ansible.parsing.utils.jsonify import jsonify - import ansible -import ansible.constants -import ansible.plugins import ansible.plugins.action +import ansible.utils.unsafe_proxy +import ansible.vars.clean + +from ansible.module_utils.common.text.converters import to_bytes, to_text +from ansible.module_utils.six.moves import shlex_quote +from ansible.parsing.utils.jsonify import jsonify import mitogen.core import mitogen.select -import mitogen.utils import ansible_mitogen.connection import ansible_mitogen.planner import ansible_mitogen.target -from ansible.module_utils._text import to_text - -try: - from ansible.utils.unsafe_proxy import wrap_var -except ImportError: - from ansible.vars.unsafe_proxy import wrap_var - -try: - # ansible 2.8 moved remove_internal_keys to the clean module - from ansible.vars.clean import remove_internal_keys -except ImportError: - try: - from ansible.vars.manager import remove_internal_keys - except ImportError: - # ansible 2.3.3 has remove_internal_keys as a protected func on the action class - # we'll fallback to calling self._remove_internal_keys in this case - remove_internal_keys = lambda a: "Not found" +import ansible_mitogen.utils +import ansible_mitogen.utils.unsafe LOG = logging.getLogger(__name__) @@ -183,7 +165,7 @@ def _remote_file_exists(self, path): LOG.debug('_remote_file_exists(%r)', path) return self._connection.get_chain().call( ansible_mitogen.target.file_exists, - mitogen.utils.cast(path) + ansible_mitogen.utils.unsafe.cast(path) ) def _configure_module(self, module_name, module_args, task_vars=None): @@ -226,7 +208,7 @@ def _remove_tmp_path(self, tmp_path): with a pipelined call to :func:`ansible_mitogen.target.prune_tree`. """ LOG.debug('_remove_tmp_path(%r)', tmp_path) - if tmp_path is None and ansible.__version__ > '2.6': + if tmp_path is None and ansible_mitogen.utils.ansible_version[:2] >= (2, 6): tmp_path = self._connection._shell.tmpdir # 06f73ad578d if tmp_path is not None: self._connection.get_chain().call_no_reply( @@ -276,7 +258,9 @@ def _remote_chmod(self, paths, mode, sudoable=False): paths, mode, sudoable) return self.fake_shell(lambda: mitogen.select.Select.all( self._connection.get_chain().call_async( - ansible_mitogen.target.set_file_mode, path, mode + ansible_mitogen.target.set_file_mode, + ansible_mitogen.utils.unsafe.cast(path), + mode, ) for path in paths )) @@ -310,7 +294,7 @@ def _remote_expand_user(self, path, sudoable=True): if not path.startswith('~'): # /home/foo -> /home/foo return path - if sudoable or not self._play_context.become: + if sudoable or not self._connection.become: if path == '~': # ~ -> /home/dmw return self._connection.homedir @@ -320,7 +304,7 @@ def _remote_expand_user(self, path, sudoable=True): # ~root/.ansible -> /root/.ansible return self._connection.get_chain(use_login=(not sudoable)).call( os.path.expanduser, - mitogen.utils.cast(path), + ansible_mitogen.utils.unsafe.cast(path), ) def get_task_timeout_secs(self): @@ -335,7 +319,7 @@ def get_task_timeout_secs(self): def _set_temp_file_args(self, module_args, wrap_async): # Ansible>2.5 module_utils reuses the action's temporary directory if # one exists. Older versions error if this key is present. - if ansible.__version__ > '2.5': + if ansible_mitogen.utils.ansible_version[:2] >= (2, 5): if wrap_async: # Sharing is not possible with async tasks, as in that case, # the directory must outlive the action plug-in. @@ -346,14 +330,16 @@ def _set_temp_file_args(self, module_args, wrap_async): # If _ansible_tmpdir is unset, Ansible>2.6 module_utils will use # _ansible_remote_tmp as the location to create the module's temporary # directory. Older versions error if this key is present. - if ansible.__version__ > '2.6': + if ansible_mitogen.utils.ansible_version[:2] >= (2, 6): module_args['_ansible_remote_tmp'] = ( self._connection.get_good_temp_dir() ) def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, - delete_remote_tmp=True, wrap_async=False): + delete_remote_tmp=True, wrap_async=False, + ignore_unknown_opts=False, + ): """ Collect up a module's execution environment then use it to invoke target.run_module() or helpers.run_module_async() in the target @@ -366,7 +352,13 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None, if task_vars is None: task_vars = {} - self._update_module_args(module_name, module_args, task_vars) + if ansible_mitogen.utils.ansible_version[:2] >= (2, 17): + self._update_module_args( + module_name, module_args, task_vars, + ignore_unknown_opts=ignore_unknown_opts, + ) + else: + self._update_module_args(module_name, module_args, task_vars) env = {} self._compute_environment_string(env) self._set_temp_file_args(module_args, wrap_async) @@ -383,26 +375,23 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None, ansible_mitogen.planner.Invocation( action=self, connection=self._connection, - module_name=mitogen.core.to_text(module_name), - module_args=mitogen.utils.cast(module_args), + module_name=ansible_mitogen.utils.unsafe.cast(mitogen.core.to_text(module_name)), + module_args=ansible_mitogen.utils.unsafe.cast(module_args), task_vars=task_vars, templar=self._templar, - env=mitogen.utils.cast(env), + env=ansible_mitogen.utils.unsafe.cast(env), wrap_async=wrap_async, timeout_secs=self.get_task_timeout_secs(), ) ) - if tmp and ansible.__version__ < '2.5' and delete_remote_tmp: + if tmp and delete_remote_tmp and ansible_mitogen.utils.ansible_version[:2] < (2, 5): # Built-in actions expected tmpdir to be cleaned up automatically # on _execute_module(). self._remove_tmp_path(tmp) # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set - # handle ansible 2.3.3 that has remove_internal_keys in a different place - check = remove_internal_keys(result) - if check == 'Not found': - self._remove_internal_keys(result) + ansible.vars.clean.remove_internal_keys(result) # taken from _execute_module of ansible 2.8.6 # propagate interpreter discovery results back to the controller @@ -426,7 +415,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None, result['deprecations'] = [] result['deprecations'].extend(self._discovery_deprecation_warnings) - return wrap_var(result) + return ansible.utils.unsafe_proxy.wrap_var(result) def _postprocess_response(self, result): """ diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index 89aa2beba..a1870833d 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -26,19 +26,36 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function from __future__ import unicode_literals +__metaclass__ = type import collections -import imp +import logging import os +import re +import sys + +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp import mitogen.master +LOG = logging.getLogger(__name__) PREFIX = 'ansible.module_utils.' +# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`. +# name Unqualified name of the module. +# path Filesystem path of the module. +# kind One of the constants in `imp`, as returned in `imp.find_module()` +# parent `ansible_mitogen.module_finder.Module` of parent package (if any). Module = collections.namedtuple('Module', 'name path kind parent') @@ -118,14 +135,121 @@ def find_relative(parent, name, path=()): def scan_fromlist(code): + """Return an iterator of (level, name) for explicit imports in a code + object. + + Not all names identify a module. `from os import name, path` generates + `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string. + + >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n' + >>> code = compile(src, '', 'exec') + >>> list(scan_fromlist(code)) + [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')] + """ for level, modname_s, fromlist in mitogen.master.scan_code_imports(code): for name in fromlist: - yield level, '%s.%s' % (modname_s, name) + yield level, str('%s.%s' % (modname_s, name)) if not fromlist: yield level, modname_s +def walk_imports(code, prefix=None): + """Return an iterator of names for implicit parent imports & explicit + imports in a code object. + + If a prefix is provided, then only children of that prefix are included. + Not all names identify a module. `from os import name, path` generates + `'os', 'os.name', 'os.path'`, but `os.name` is usually a string. + + >>> source = 'import a; import b; import b.c; from b.d import e, f\\n' + >>> code = compile(source, '', 'exec') + >>> list(walk_imports(code)) + ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'] + >>> list(walk_imports(code, prefix='b')) + ['b.c', 'b.d', 'b.d.e', 'b.d.f'] + """ + if prefix is None: + prefix = '' + pattern = re.compile(r'(^|\.)(\w+)') + start = len(prefix) + for _, name, fromlist in mitogen.master.scan_code_imports(code): + if not name.startswith(prefix): + continue + for match in pattern.finditer(name, start): + yield name[:match.end()] + for leaf in fromlist: + yield str('%s.%s' % (name, leaf)) + + def scan(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + """Return a list of (name, path, is_package) for ansible.module_utils + imports used by an Ansible module. + """ + log = LOG.getChild('scan') + log.debug('%r, %r, %r', module_name, module_path, search_path) + + if sys.version_info >= (3, 4): + result = _scan_importlib_find_spec( + module_name, module_path, search_path, + ) + log.debug('_scan_importlib_find_spec %r', result) + else: + result = _scan_imp_find_module(module_name, module_path, search_path) + log.debug('_scan_imp_find_module %r', result) + return result + + +def _scan_importlib_find_spec(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + module = importlib.machinery.ModuleSpec( + module_name, loader=None, origin=module_path, + ) + prefix = importlib.machinery.ModuleSpec( + PREFIX.rstrip('.'), loader=None, + ) + prefix.submodule_search_locations = search_path + queue = collections.deque([module]) + specs = {prefix.name: prefix} + while queue: + spec = queue.popleft() + if spec.origin is None: + continue + try: + with open(spec.origin, 'rb') as f: + code = compile(f.read(), spec.name, 'exec') + except Exception as exc: + raise ValueError((exc, module, spec, specs)) + + for name in walk_imports(code, prefix.name): + if name in specs: + continue + + parent_name = name.rpartition('.')[0] + parent = specs[parent_name] + if parent is None or not parent.submodule_search_locations: + specs[name] = None + continue + + child = importlib.util._find_spec( + name, parent.submodule_search_locations, + ) + if child is None or child.origin is None: + specs[name] = None + continue + + specs[name] = child + queue.append(child) + + del specs[prefix.name] + return sorted( + (spec.name, spec.origin, spec.submodule_search_locations is not None) + for spec in specs.values() if spec is not None + ) + + +def _scan_imp_find_module(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] module = Module(module_name, module_path, imp.PY_SOURCE, None) stack = [module] seen = set() diff --git a/ansible_mitogen/parsing.py b/ansible_mitogen/parsing.py index 27fca7cd6..2ca50315d 100644 --- a/ansible_mitogen/parsing.py +++ b/ansible_mitogen/parsing.py @@ -26,8 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function from __future__ import unicode_literals +__metaclass__ = type import mitogen.core diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index c0913a3e0..4cdc0f206 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -34,25 +34,27 @@ [0] "Ansible Module Architecture", developing_program_flow_modules.html """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function from __future__ import unicode_literals +__metaclass__ = type import json import logging import os import random +import re -from ansible.executor import module_common -from ansible.collections.list import list_collection_dirs +import ansible.collections.list import ansible.errors -import ansible.module_utils -import ansible.release +import ansible.executor.module_common + import mitogen.core import mitogen.select import ansible_mitogen.loaders import ansible_mitogen.parsing import ansible_mitogen.target +import ansible_mitogen.utils.unsafe LOG = logging.getLogger(__name__) @@ -191,7 +193,7 @@ class BinaryPlanner(Planner): @classmethod def detect(cls, path, source): - return module_common._is_binary(source) + return ansible.executor.module_common._is_binary(source) def get_push_files(self): return [mitogen.core.to_text(self._inv.module_path)] @@ -214,12 +216,15 @@ class ScriptPlanner(BinaryPlanner): """ def _rewrite_interpreter(self, path): """ - Given the original interpreter binary extracted from the script's - interpreter line, look up the associated `ansible_*_interpreter` - variable, render it and return it. + Given the interpreter path (from the script's hashbang line), return + the desired interpreter path. This tries, in order + + 1. Look up & render the `ansible_*_interpreter` variable, if set + 2. Look up the `discovered_interpreter_*` fact, if present + 3. The unmodified path from the hashbang line. :param str path: - Absolute UNIX path to original interpreter. + Absolute path to original interpreter (e.g. '/usr/bin/python'). :returns: Shell fragment prefix used to execute the script via "/bin/sh -c". @@ -227,13 +232,25 @@ def _rewrite_interpreter(self, path): involved here, the vanilla implementation uses it and that use is exploited in common playbooks. """ - key = u'ansible_%s_interpreter' % os.path.basename(path).strip() + interpreter_name = os.path.basename(path).strip() + key = u'ansible_%s_interpreter' % interpreter_name try: template = self._inv.task_vars[key] except KeyError: - return path + pass + else: + configured_interpreter = self._inv.templar.template(template) + return ansible_mitogen.utils.unsafe.cast(configured_interpreter) + + key = u'discovered_interpreter_%s' % interpreter_name + try: + discovered_interpreter = self._inv.task_vars['ansible_facts'][key] + except KeyError: + pass + else: + return ansible_mitogen.utils.unsafe.cast(discovered_interpreter) - return mitogen.utils.cast(self._inv.templar.template(template)) + return path def _get_interpreter(self): path, arg = ansible_mitogen.parsing.parse_hashbang( @@ -248,7 +265,8 @@ def _get_interpreter(self): if arg: fragment += ' ' + arg - return fragment, path.startswith('python') + is_python = path.startswith('python') + return fragment, is_python def get_kwargs(self, **kwargs): interpreter_fragment, is_python = self._get_interpreter() @@ -268,7 +286,7 @@ class JsonArgsPlanner(ScriptPlanner): @classmethod def detect(cls, path, source): - return module_common.REPLACER_JSONARGS in source + return ansible.executor.module_common.REPLACER_JSONARGS in source class WantJsonPlanner(ScriptPlanner): @@ -297,11 +315,11 @@ class NewStylePlanner(ScriptPlanner): preprocessing the module. """ runner_name = 'NewStyleRunner' - marker = b'from ansible.module_utils.' + MARKER = re.compile(br'from ansible(?:_collections|\.module_utils)\.') @classmethod def detect(cls, path, source): - return cls.marker in source + return cls.MARKER.search(source) is not None def _get_interpreter(self): return None, None @@ -321,6 +339,8 @@ def get_module_deps(self): ALWAYS_FORK_MODULES = frozenset([ 'dnf', # issue #280; py-dnf/hawkey need therapy 'firewalld', # issue #570: ansible module_utils caches dbus conn + 'ansible.legacy.dnf', # issue #776 + 'ansible.builtin.dnf', # issue #832 ]) def should_fork(self): @@ -360,7 +380,7 @@ def get_module_map(self): module_name='ansible_module_%s' % (self._inv.module_name,), module_path=self._inv.module_path, search_path=self.get_search_path(), - builtin_path=module_common._MODULE_UTILS_PATH, + builtin_path=ansible.executor.module_common._MODULE_UTILS_PATH, context=self._inv.connection.context, ) return self._module_map @@ -403,7 +423,7 @@ class ReplacerPlanner(NewStylePlanner): @classmethod def detect(cls, path, source): - return module_common.REPLACER in source + return ansible.executor.module_common.REPLACER in source class OldStylePlanner(ScriptPlanner): @@ -425,12 +445,6 @@ def detect(cls, path, source): ] -try: - _get_ansible_module_fqn = module_common._get_ansible_module_fqn -except AttributeError: - _get_ansible_module_fqn = None - - def py_modname_from_path(name, path): """ Fetch the logical name of a new-style module as it might appear in @@ -440,11 +454,12 @@ def py_modname_from_path(name, path): package hierarchy approximated on the target, enabling relative imports to function correctly. For example, "ansible.modules.system.setup". """ - if _get_ansible_module_fqn: - try: - return _get_ansible_module_fqn(path) - except ValueError: - pass + try: + return ansible.executor.module_common._get_ansible_module_fqn(path) + except AttributeError: + pass + except ValueError: + pass return 'ansible.modules.' + name @@ -462,7 +477,7 @@ def read_file(path): finally: os.close(fd) - return mitogen.core.b('').join(bits) + return b''.join(bits) def _propagate_deps(invocation, planner, context): @@ -526,12 +541,15 @@ def _invoke_isolated_task(invocation, planner): context.shutdown() -def _get_planner(name, path, source): +def _get_planner(invocation, source): for klass in _planners: - if klass.detect(path, source): - LOG.debug('%r accepted %r (filename %r)', klass, name, path) + if klass.detect(invocation.module_path, source): + LOG.debug( + '%r accepted %r (filename %r)', + klass, invocation.module_name, invocation.module_path, + ) return klass - LOG.debug('%r rejected %r', klass, name) + LOG.debug('%r rejected %r', klass, invocation.module_name) raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation)) @@ -562,7 +580,7 @@ def _load_collections(invocation): Goes through all collection path possibilities and stores paths to installed collections Stores them on the current invocation to later be passed to the master service """ - for collection_path in list_collection_dirs(): + for collection_path in ansible.collections.list.list_collection_dirs(): invocation._extra_sys_paths.add(collection_path.decode('utf-8')) @@ -594,8 +612,7 @@ def invoke(invocation): module_source = invocation.get_module_source() _fix_py35(invocation, module_source) _planner_by_path[invocation.module_path] = _get_planner( - invocation.module_name, - invocation.module_path, + invocation, module_source ) diff --git a/ansible_mitogen/plugins/action/mitogen_fetch.py b/ansible_mitogen/plugins/action/mitogen_fetch.py index b9eece76a..c1ef19020 100644 --- a/ansible_mitogen/plugins/action/mitogen_fetch.py +++ b/ansible_mitogen/plugins/action/mitogen_fetch.py @@ -18,23 +18,17 @@ __metaclass__ = type import os - -from ansible.module_utils._text import to_bytes +import base64 +from ansible.errors import AnsibleError, AnsibleActionFail, AnsibleActionSkip +from ansible.module_utils.common.text.converters import to_bytes, to_text from ansible.module_utils.six import string_types from ansible.module_utils.parsing.convert_bool import boolean from ansible.plugins.action import ActionBase -from ansible.utils.hashing import checksum, md5, secure_hash -from ansible.utils.path import makedirs_safe - +from ansible.utils.display import Display +from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash +from ansible.utils.path import makedirs_safe, is_subpath -REMOTE_CHECKSUM_ERRORS = { - '0': "unable to calculate the checksum of the remote file", - '1': "the remote file does not exist", - '2': "no read permission on remote file", - '3': "remote file is a directory, fetch cannot work on directories", - '4': "python isn't present on the system. Unable to compute checksum", - '5': "stdlib json was not found on the remote machine. Only the raw module can work without those installed", -} +display = Display() class ActionModule(ActionBase): @@ -45,36 +39,94 @@ def run(self, tmp=None, task_vars=None): task_vars = dict() result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + try: if self._play_context.check_mode: - result['skipped'] = True - result['msg'] = 'check mode not (yet) supported for this module' - return result + raise AnsibleActionSkip('check mode not (yet) supported for this module') + source = self._task.args.get('src', None) + original_dest = dest = self._task.args.get('dest', None) flat = boolean(self._task.args.get('flat'), strict=False) fail_on_missing = boolean(self._task.args.get('fail_on_missing', True), strict=False) validate_checksum = boolean(self._task.args.get('validate_checksum', True), strict=False) + msg = '' # validate source and dest are strings FIXME: use basic.py and module specs - source = self._task.args.get('src') if not isinstance(source, string_types): - result['msg'] = "Invalid type supplied for source option, it must be a string" + msg = "Invalid type supplied for source option, it must be a string" - dest = self._task.args.get('dest') if not isinstance(dest, string_types): - result['msg'] = "Invalid type supplied for dest option, it must be a string" + msg = "Invalid type supplied for dest option, it must be a string" + + if source is None or dest is None: + msg = "src and dest are required" - if result.get('msg'): - result['failed'] = True - return result + if msg: + raise AnsibleActionFail(msg) source = self._connection._shell.join_path(source) source = self._remote_expand_user(source) - # calculate checksum for the remote file, don't bother if using - # become as slurp will be used Force remote_checksum to follow - # symlinks because fetch always follows symlinks - remote_checksum = self._remote_checksum(source, all_vars=task_vars, follow=True) + remote_stat = {} + remote_checksum = None + if True: + # Get checksum for the remote file even using become. Mitogen doesn't need slurp. + # Follow symlinks because fetch always follows symlinks + try: + remote_stat = self._execute_remote_stat(source, all_vars=task_vars, follow=True) + except AnsibleError as ae: + result['changed'] = False + result['file'] = source + if fail_on_missing: + result['failed'] = True + result['msg'] = to_text(ae) + else: + result['msg'] = "%s, ignored" % to_text(ae, errors='surrogate_or_replace') + + return result + + remote_checksum = remote_stat.get('checksum') + if remote_stat.get('exists'): + if remote_stat.get('isdir'): + result['failed'] = True + result['changed'] = False + result['msg'] = "remote file is a directory, fetch cannot work on directories" + + # Historically, these don't fail because you may want to transfer + # a log file that possibly MAY exist but keep going to fetch other + # log files. Today, this is better achieved by adding + # ignore_errors or failed_when to the task. Control the behaviour + # via fail_when_missing + if not fail_on_missing: + result['msg'] += ", not transferring, ignored" + del result['changed'] + del result['failed'] + + return result + + # use slurp if permissions are lacking or privilege escalation is needed + remote_data = None + if remote_checksum in (None, '1', ''): + slurpres = self._execute_module(module_name='ansible.legacy.slurp', module_args=dict(src=source), task_vars=task_vars) + if slurpres.get('failed'): + if not fail_on_missing: + result['file'] = source + result['changed'] = False + else: + result.update(slurpres) + + if 'not found' in slurpres.get('msg', ''): + result['msg'] = "the remote file does not exist, not transferring, ignored" + elif slurpres.get('msg', '').startswith('source is a directory'): + result['msg'] = "remote file is a directory, fetch cannot work on directories" + + return result + else: + if slurpres['encoding'] == 'base64': + remote_data = base64.b64decode(slurpres['content']) + if remote_data is not None: + remote_checksum = checksum_s(remote_data) # calculate the destination name if os.path.sep not in self._connection._shell.join_path('a', ''): @@ -83,13 +135,14 @@ def run(self, tmp=None, task_vars=None): else: source_local = source - dest = os.path.expanduser(dest) + # ensure we only use file name, avoid relative paths + if not is_subpath(dest, original_dest): + # TODO: ? dest = os.path.expanduser(dest.replace(('../',''))) + raise AnsibleActionFail("Detected directory traversal, expected to be contained in '%s' but got '%s'" % (original_dest, dest)) + if flat: if os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and not dest.endswith(os.sep): - result['msg'] = "dest is an existing directory, use a trailing slash if you want to fetch src into that directory" - result['file'] = dest - result['failed'] = True - return result + raise AnsibleActionFail("dest is an existing directory, use a trailing slash if you want to fetch src into that directory") if dest.endswith(os.sep): # if the path ends with "/", we'll use the source filename as the # destination filename @@ -106,23 +159,7 @@ def run(self, tmp=None, task_vars=None): target_name = self._play_context.remote_addr dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local) - dest = dest.replace("//", "/") - - if remote_checksum in REMOTE_CHECKSUM_ERRORS: - result['changed'] = False - result['file'] = source - result['msg'] = REMOTE_CHECKSUM_ERRORS[remote_checksum] - # Historically, these don't fail because you may want to transfer - # a log file that possibly MAY exist but keep going to fetch other - # log files. Today, this is better achieved by adding - # ignore_errors or failed_when to the task. Control the behaviour - # via fail_when_missing - if fail_on_missing: - result['failed'] = True - del result['changed'] - else: - result['msg'] += ", not transferring, ignored" - return result + dest = os.path.normpath(dest) # calculate checksum for the local file local_checksum = checksum(dest) @@ -132,7 +169,15 @@ def run(self, tmp=None, task_vars=None): makedirs_safe(os.path.dirname(dest)) # fetch the file and check for changes - self._connection.fetch_file(source, dest) + if remote_data is None: + self._connection.fetch_file(source, dest) + else: + try: + f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb') + f.write(remote_data) + f.close() + except (IOError, OSError) as e: + raise AnsibleActionFail("Failed to fetch the file: %s" % e) new_checksum = secure_hash(dest) # For backwards compatibility. We'll return None on FIPS enabled systems try: @@ -157,10 +202,6 @@ def run(self, tmp=None, task_vars=None): result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum)) finally: - try: - self._remove_tmp_path(self._connection._shell.tmpdir) - except AttributeError: - # .tmpdir was added to ShellModule in v2.6.0, so old versions don't have it - pass + self._remove_tmp_path(self._connection._shell.tmpdir) return result diff --git a/ansible_mitogen/plugins/action/mitogen_get_stack.py b/ansible_mitogen/plugins/action/mitogen_get_stack.py index 0d0afe863..a8634e574 100644 --- a/ansible_mitogen/plugins/action/mitogen_get_stack.py +++ b/ansible_mitogen/plugins/action/mitogen_get_stack.py @@ -26,14 +26,15 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import -from __future__ import unicode_literals - """ Fetch the connection configuration stack that would be used to connect to a target, without actually connecting to it. """ +from __future__ import absolute_import, division, print_function +from __future__ import unicode_literals +__metaclass__ = type + import ansible_mitogen.connection from ansible.plugins.action import ActionBase diff --git a/ansible_mitogen/plugins/connection/mitogen_buildah.py b/ansible_mitogen/plugins/connection/mitogen_buildah.py index 017214b24..10ab6b41e 100644 --- a/ansible_mitogen/plugins/connection/mitogen_buildah.py +++ b/ansible_mitogen/plugins/connection/mitogen_buildah.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_doas.py b/ansible_mitogen/plugins/connection/mitogen_doas.py index 1113d7c63..963ec5972 100644 --- a/ansible_mitogen/plugins/connection/mitogen_doas.py +++ b/ansible_mitogen/plugins/connection/mitogen_doas.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_docker.py b/ansible_mitogen/plugins/connection/mitogen_docker.py index b71ef5f11..e0dd21202 100644 --- a/ansible_mitogen/plugins/connection/mitogen_docker.py +++ b/ansible_mitogen/plugins/connection/mitogen_docker.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_jail.py b/ansible_mitogen/plugins/connection/mitogen_jail.py index c7475fb14..a432e17d5 100644 --- a/ansible_mitogen/plugins/connection/mitogen_jail.py +++ b/ansible_mitogen/plugins/connection/mitogen_jail.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_kubectl.py b/ansible_mitogen/plugins/connection/mitogen_kubectl.py index 44d3b50a2..bae41609c 100644 --- a/ansible_mitogen/plugins/connection/mitogen_kubectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_kubectl.py @@ -27,12 +27,13 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys -from ansible.errors import AnsibleConnectionFailure -from ansible.module_utils.six import iteritems +import ansible.errors try: import ansible_mitogen @@ -45,17 +46,11 @@ import ansible_mitogen.loaders -_class = ansible_mitogen.loaders.connection_loader__get( +_get_result = ansible_mitogen.loaders.connection_loader__get( 'kubectl', class_only=True, ) -if _class: - kubectl = sys.modules[_class.__module__] - del _class -else: - kubectl = None - class Connection(ansible_mitogen.connection.Connection): transport = 'kubectl' @@ -66,14 +61,22 @@ class Connection(ansible_mitogen.connection.Connection): ) def __init__(self, *args, **kwargs): - if kubectl is None: - raise AnsibleConnectionFailure(self.not_supported_msg) + if not _get_result: + raise ansible.errors.AnsibleConnectionFailure(self.not_supported_msg) super(Connection, self).__init__(*args, **kwargs) def get_extra_args(self): + try: + # Ansible < 2.10, _get_result is the connection class + connection_options = _get_result.connection_options + except AttributeError: + # Ansible >= 2.10, _get_result is a get_with_context_result + connection_options = _get_result.object.connection_options parameters = [] - for key, option in iteritems(kubectl.CONNECTION_OPTIONS): - if self.get_task_var('ansible_' + key) is not None: - parameters += [ option, self.get_task_var('ansible_' + key) ] + for key in connection_options: + task_var_name = 'ansible_%s' % key + task_var = self.get_task_var(task_var_name) + if task_var is not None: + parameters += [connection_options[key], task_var] return parameters diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index a98c834c5..2d1e7052b 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys @@ -40,13 +42,7 @@ import ansible_mitogen.connection import ansible_mitogen.process - -if sys.version_info > (3,): - viewkeys = dict.keys -elif sys.version_info > (2, 7): - viewkeys = dict.viewkeys -else: - viewkeys = lambda dct: set(dct) +viewkeys = getattr(dict, 'viewkeys', dict.keys) def dict_diff(old, new): diff --git a/ansible_mitogen/plugins/connection/mitogen_lxc.py b/ansible_mitogen/plugins/connection/mitogen_lxc.py index 696c9abd0..8850b3d77 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxc.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxc.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_lxd.py b/ansible_mitogen/plugins/connection/mitogen_lxd.py index 95e692a01..25370ef52 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxd.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxd.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/ansible_mitogen/plugins/connection/mitogen_machinectl.py index 0f5a0d282..1f538797c 100644 --- a/ansible_mitogen/plugins/connection/mitogen_machinectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_podman.py b/ansible_mitogen/plugins/connection/mitogen_podman.py new file mode 100644 index 000000000..e423aac82 --- /dev/null +++ b/ansible_mitogen/plugins/connection/mitogen_podman.py @@ -0,0 +1,46 @@ +# Copyright 2022, Mitogen contributers +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import os.path +import sys + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'podman' diff --git a/ansible_mitogen/plugins/connection/mitogen_setns.py b/ansible_mitogen/plugins/connection/mitogen_setns.py index 20c6f1370..4d70892f2 100644 --- a/ansible_mitogen/plugins/connection/mitogen_setns.py +++ b/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index 1c81dae52..f6a27a6e7 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -26,21 +26,26 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys +from ansible.plugins.connection.ssh import ( + DOCUMENTATION as _ansible_ssh_DOCUMENTATION, +) + DOCUMENTATION = """ + name: mitogen_ssh author: David Wilson - connection: mitogen_ssh short_description: Connect over SSH via Mitogen description: - This connects using an OpenSSH client controlled by the Mitogen for Ansible extension. It accepts every option the vanilla ssh plugin accepts. - version_added: "2.5" options: -""" +""" + _ansible_ssh_DOCUMENTATION.partition('options:\n')[2] try: import ansible_mitogen diff --git a/ansible_mitogen/plugins/connection/mitogen_su.py b/ansible_mitogen/plugins/connection/mitogen_su.py index 4ab2711e5..9395cc5f2 100644 --- a/ansible_mitogen/plugins/connection/mitogen_su.py +++ b/ansible_mitogen/plugins/connection/mitogen_su.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_sudo.py b/ansible_mitogen/plugins/connection/mitogen_sudo.py index 130f54454..ef5494283 100644 --- a/ansible_mitogen/plugins/connection/mitogen_sudo.py +++ b/ansible_mitogen/plugins/connection/mitogen_sudo.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index 66872663f..abbe76726 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index ffe2fbd94..4f4e1f818 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py b/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py index 23eccd369..c3396c5fa 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py +++ b/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index 1b198e61d..b1b03aef3 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os.path import sys diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 1fc7bf801..7ec70f2a0 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import atexit import logging import multiprocessing @@ -59,10 +61,9 @@ import ansible import ansible.constants as C import ansible.errors + import ansible_mitogen.logging import ansible_mitogen.services - -from mitogen.core import b import ansible_mitogen.affinity @@ -178,42 +179,6 @@ def setup_pool(pool): LOG.debug('Service pool configured: size=%d', pool.size) -def _setup_simplejson(responder): - """ - We support serving simplejson for Python 2.4 targets on Ansible 2.3, at - least so the package's own CI Docker scripts can run without external - help, however newer versions of simplejson no longer support Python - 2.4. Therefore override any installed/loaded version with a - 2.4-compatible version we ship in the compat/ directory. - """ - responder.whitelist_prefix('simplejson') - - # issue #536: must be at end of sys.path, in case existing newer - # version is already loaded. - compat_path = os.path.join(os.path.dirname(__file__), 'compat') - sys.path.append(compat_path) - - for fullname, is_pkg, suffix in ( - (u'simplejson', True, '__init__.py'), - (u'simplejson.decoder', False, 'decoder.py'), - (u'simplejson.encoder', False, 'encoder.py'), - (u'simplejson.scanner', False, 'scanner.py'), - ): - path = os.path.join(compat_path, 'simplejson', suffix) - fp = open(path, 'rb') - try: - source = fp.read() - finally: - fp.close() - - responder.add_source_override( - fullname=fullname, - path=path, - source=source, - is_pkg=is_pkg, - ) - - def _setup_responder(responder): """ Configure :class:`mitogen.master.ModuleResponder` to only permit @@ -221,7 +186,6 @@ def _setup_responder(responder): """ responder.whitelist_prefix('ansible') responder.whitelist_prefix('ansible_mitogen') - _setup_simplejson(responder) # Ansible 2.3 is compatible with Python 2.4 targets, however # ansible/__init__.py is not. Instead, executor/module_common.py writes @@ -316,11 +280,11 @@ def get_cpu_count(default=None): class Broker(mitogen.master.Broker): """ - WorkerProcess maintains at most 2 file descriptors, therefore does not need + WorkerProcess maintains fewer file descriptors, therefore does not need the exuberant syscall expense of EpollPoller, so override it and restore the poll() poller. """ - poller_class = mitogen.core.Poller + poller_class = mitogen.parent.POLLER_LIGHTWEIGHT class Binding(object): @@ -674,7 +638,7 @@ def worker_main(self): try: # Let the parent know our listening socket is ready. - mitogen.core.io_op(self.model.child_sock.send, b('1')) + mitogen.core.io_op(self.model.child_sock.send, b'1') # Block until the socket is closed, which happens on parent exit. mitogen.core.io_op(self.model.child_sock.recv, 1) finally: diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 064023442..b60e537ce 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -36,8 +36,13 @@ how to build arguments for it, preseed related data, etc. """ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import atexit -import imp +import ctypes +import json +import logging import os import re import shlex @@ -47,24 +52,19 @@ import traceback import types +from ansible.module_utils.six.moves import shlex_quote + import mitogen.core import ansible_mitogen.target # TODO: circular import -from mitogen.core import b -from mitogen.core import bytes_partition -from mitogen.core import str_rpartition from mitogen.core import to_text try: - import ctypes -except ImportError: - # Python 2.4 - ctypes = None - -try: - import json + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util except ImportError: - # Python 2.4 - import simplejson as json + # Python < 3.4, PEP 302 Import Hooks + import imp try: # Cannot use cStringIO as it does not support Unicode. @@ -72,15 +72,6 @@ except ImportError: from io import StringIO -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - -# Absolute imports for <2.5. -logging = __import__('logging') - - # Prevent accidental import of an Ansible module from hanging on stdin read. import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' @@ -90,15 +81,13 @@ # explicit call to res_init() on each task invocation. BSD-alikes export it # directly, Linux #defines it as "__res_init". libc__res_init = None -if ctypes: - libc = ctypes.CDLL(None) - for symbol in 'res_init', '__res_init': - try: - libc__res_init = getattr(libc, symbol) - except AttributeError: - pass +libc = ctypes.CDLL(None) +for symbol in 'res_init', '__res_init': + try: + libc__res_init = getattr(libc, symbol) + except AttributeError: + pass -iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) @@ -212,13 +201,13 @@ def _parse(self, fp): for line in fp: # ' #export foo=some var ' -> ['#export', 'foo=some var '] bits = shlex_split_b(line) - if (not bits) or bits[0].startswith(b('#')): + if (not bits) or bits[0].startswith(b'#'): continue - if bits[0] == b('export'): + if bits[0] == b'export': bits.pop(0) - key, sep, value = bytes_partition(b(' ').join(bits), b('=')) + key, sep, value = b' '.join(bits).partition(b'=') if key and sep: yield key, value @@ -516,10 +505,71 @@ def revert(self): sys.modules.pop(fullname, None) def find_module(self, fullname, path=None): + """ + Return a loader for the module with fullname, if we will load it. + + Implements importlib.abc.MetaPathFinder.find_module(). + Deprecrated in Python 3.4+, replaced by find_spec(). + Raises ImportWarning in Python 3.10+. Removed in Python 3.12. + """ if fullname in self._by_fullname: return self + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`. + + Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+. + """ + if fullname.endswith('.'): + return None + + try: + module_path, is_package = self._by_fullname[fullname] + except KeyError: + LOG.debug('Skipping %s: not present', fullname) + return None + + LOG.debug('Handling %s', fullname) + origin = 'master:%s' % (module_path,) + return importlib.machinery.ModuleSpec( + fullname, loader=self, origin=origin, is_package=is_package, + ) + + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + spec = module.__spec__ + path, _ = self._by_fullname[spec.name] + source = ansible_mitogen.target.get_small_file(self._context, path) + code = compile(source, path, 'exec', 0, 1) + exec(code, module.__dict__) + self._loaded.add(spec.name) + def load_module(self, fullname): + """ + Return the loaded module specified by fullname. + + Implements PEP 302 importlib.abc.Loader.load_module(). + Deprecated in Python 3.4+, replaced by create_module() & exec_module(). + """ path, is_pkg = self._by_fullname[fullname] source = ansible_mitogen.target.get_small_file(self._context, path) code = compile(source, path, 'exec', 0, 1) @@ -530,7 +580,7 @@ def load_module(self, fullname): mod.__path__ = [] mod.__package__ = str(fullname) else: - mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0]) + mod.__package__ = str(to_text(fullname).rpartition('.')[0]) exec(code, mod.__dict__) self._loaded.add(fullname) return mod @@ -545,7 +595,7 @@ class TemporaryEnvironment(object): def __init__(self, env=None): self.original = dict(os.environ) self.env = env or {} - for key, value in iteritems(self.env): + for key, value in mitogen.core.iteritems(self.env): key = mitogen.core.to_text(key) value = mitogen.core.to_text(value) if value is None: @@ -753,7 +803,7 @@ def __init__(self, interpreter_fragment, is_python, **kwargs): self.interpreter_fragment = interpreter_fragment self.is_python = is_python - b_ENCODING_STRING = b('# -*- coding: utf-8 -*-') + b_ENCODING_STRING = b'# -*- coding: utf-8 -*-' def _get_program(self): return self._rewrite_source( @@ -786,13 +836,13 @@ def _rewrite_source(self, s): # While Ansible rewrites the #! using ansible_*_interpreter, it is # never actually used to execute the script, instead it is a shell # fragment consumed by shell/__init__.py::build_module_command(). - new = [b('#!') + utf8(self.interpreter_fragment)] + new = [b'#!' + utf8(self.interpreter_fragment)] if self.is_python: new.append(self.b_ENCODING_STRING) - _, _, rest = bytes_partition(s, b('\n')) + _, _, rest = s.partition(b'\n') new.append(rest) - return b('\n').join(new) + return b'\n'.join(new) class NewStyleRunner(ScriptRunner): @@ -820,12 +870,17 @@ def _setup_imports(self): synchronization mechanism by importing everything the module will need prior to detaching. """ + # I think "custom" means "found in custom module_utils search path", + # e.g. playbook relative dir, ~/.ansible/..., Ansible collection. for fullname, _, _ in self.module_map['custom']: mitogen.core.import_module(fullname) + + # I think "builtin" means "part of ansible/ansible-base/ansible-core", + # as opposed to Python builtin modules such as sys. for fullname in self.module_map['builtin']: try: mitogen.core.import_module(fullname) - except ImportError: + except ImportError as exc: # #590: Ansible 2.8 module_utils.distro is a package that # replaces itself in sys.modules with a non-package during # import. Prior to replacement, it is a real package containing @@ -836,8 +891,18 @@ def _setup_imports(self): # loop progresses to the next entry and attempts to preload # 'distro._distro', the import mechanism will fail. So here we # silently ignore any failure for it. - if fullname != 'ansible.module_utils.distro._distro': - raise + if fullname == 'ansible.module_utils.distro._distro': + continue + + # ansible.module_utils.compat.selinux raises ImportError if it + # can't load libselinux.so. The importer would usually catch + # this & skip selinux operations. We don't care about selinux, + # we're using import to get a copy of the module. + if (fullname == 'ansible.module_utils.compat.selinux' + and exc.msg == 'unable to load libselinux.so'): + continue + + raise def _setup_excepthook(self): """ @@ -890,8 +955,7 @@ def _setup_args(self): # change the default encoding. This hack was removed from Ansible long ago, # but not before permeating into many third party modules. PREHISTORIC_HACK_RE = re.compile( - b(r'reload\s*\(\s*sys\s*\)\s*' - r'sys\s*\.\s*setdefaultencoding\([^)]+\)') + br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)', ) def _setup_program(self): @@ -899,7 +963,7 @@ def _setup_program(self): context=self.service_context, path=self.path, ) - self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source) + self.source = self.PREHISTORIC_HACK_RE.sub(b'', source) def _get_code(self): try: @@ -917,7 +981,7 @@ def _get_code(self): if mitogen.core.PY3: main_module_name = '__main__' else: - main_module_name = b('__main__') + main_module_name = b'__main__' def _handle_magic_exception(self, mod, exc): """ @@ -949,7 +1013,7 @@ def _get_module_package(self): approximation of the original package hierarchy, so that relative imports function correctly. """ - pkg, sep, modname = str_rpartition(self.py_module_name, '.') + pkg, sep, _ = self.py_module_name.rpartition('.') if not sep: return None if mitogen.core.PY3: @@ -992,7 +1056,7 @@ def _run(self): class JsonArgsRunner(ScriptRunner): - JSON_ARGS = b('<>') + JSON_ARGS = b'<>' def _get_args_contents(self): return json.dumps(self.args).encode() diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 2eb3b2e4a..abc0e3794 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -39,23 +39,25 @@ when a child has completed a job. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function from __future__ import unicode_literals +__metaclass__ = type import logging import os -import os.path import sys import threading import ansible.constants -import mitogen +from ansible.module_utils.six import reraise + +import mitogen.core import mitogen.service -import mitogen.utils import ansible_mitogen.loaders import ansible_mitogen.module_finder import ansible_mitogen.target +import ansible_mitogen.utils.unsafe LOG = logging.getLogger(__name__) @@ -66,20 +68,6 @@ ansible_mitogen.loaders.shell_loader.get('sh') -if sys.version_info[0] == 3: - def reraise(tp, value, tb): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value -else: - exec( - "def reraise(tp, value, tb=None):\n" - " raise tp, value, tb\n" - ) - - def _get_candidate_temp_dirs(): try: # >=2.5 @@ -91,7 +79,7 @@ def _get_candidate_temp_dirs(): remote_tmp = ansible.constants.DEFAULT_REMOTE_TMP system_tmpdirs = ('/var/tmp', '/tmp') - return mitogen.utils.cast([remote_tmp] + list(system_tmpdirs)) + return ansible_mitogen.utils.unsafe.cast([remote_tmp] + list(system_tmpdirs)) def key_from_dict(**kwargs): diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 792cfada1..440e58112 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -26,7 +26,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import os import signal import threading @@ -43,7 +45,8 @@ import ansible_mitogen.process import ansible.executor.process.worker -from ansible.utils.sentinel import Sentinel +import ansible.template +import ansible.utils.sentinel def _patch_awx_callback(): @@ -54,12 +57,11 @@ def _patch_awx_callback(): # AWX uses sitecustomize.py to force-load this package. If it exists, we're # running under AWX. try: - from awx_display_callback.events import EventContext - from awx_display_callback.events import event_context + import awx_display_callback.events except ImportError: return - if hasattr(EventContext(), '_local'): + if hasattr(awx_display_callback.events.EventContext(), '_local'): # Patched version. return @@ -68,8 +70,8 @@ def patch_add_local(self, **kwargs): ctx = tls.setdefault('_ctx', {}) ctx.update(kwargs) - EventContext._local = threading.local() - EventContext.add_local = patch_add_local + awx_display_callback.events.EventContext._local = threading.local() + awx_display_callback.events.EventContext.add_local = patch_add_local _patch_awx_callback() @@ -107,6 +109,7 @@ def wrap_action_loader__get(name, *args, **kwargs): 'lxc', 'lxd', 'machinectl', + 'podman', 'setns', 'ssh', ) @@ -278,7 +281,7 @@ def _queue_task(self, host, task, task_vars, play_context): name=task.action, class_only=True, ) - if play_context.connection is not Sentinel: + if play_context.connection is not ansible.utils.sentinel.Sentinel: # 2.8 appears to defer computing this until inside the worker. # TODO: figure out where it has moved. ansible_mitogen.loaders.connection_loader.get( @@ -324,3 +327,44 @@ def run(self, iterator, play_context, result=0): self._worker_model.on_strategy_complete() finally: ansible_mitogen.process.set_worker_model(None) + + def _smuggle_to_connection_reset(self, task, play_context, iterator, target_host): + """ + Create a templar and make it available for use in Connection.reset(). + This allows templated connection variables to be used when Mitogen + reconstructs its connection stack. + """ + variables = self._variable_manager.get_vars( + play=iterator._play, host=target_host, task=task, + _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all, + ) + templar = ansible.template.Templar( + loader=self._loader, variables=variables, + ) + + # Required for remote_user option set by variable (e.g. ansible_user). + # Without it remote_user in ansible.cfg gets used. + play_context = play_context.set_task_and_variable_override( + task=task, variables=variables, templar=templar, + ) + play_context.post_validate(templar=templar) + + # Required for timeout option set by variable (e.g. ansible_timeout). + # Without it the task timeout keyword (default: 0) gets used. + play_context.update_vars(variables) + + # Stash the task and templar somewhere Connection.reset() can find it + play_context.vars.update({ + '_mitogen.smuggled.reset_connection': (task, templar), + }) + return play_context + + def _execute_meta(self, task, play_context, iterator, target_host): + if task.args['_raw_params'] == 'reset_connection': + play_context = self._smuggle_to_connection_reset( + task, play_context, iterator, target_host, + ) + + return super(StrategyMixin, self)._execute_meta( + task, play_context, iterator, target_host, + ) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 652b5adcc..ee4cb398f 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -33,8 +33,13 @@ for file transfer, module execution and sundry bits like changing file modes. """ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + import errno import grp +import json +import logging import operator import os import pwd @@ -47,32 +52,9 @@ import traceback import types -# Absolute imports for <2.5. -logging = __import__('logging') - import mitogen.core -import mitogen.fork import mitogen.parent import mitogen.service -from mitogen.core import b - -try: - import json -except ImportError: - import simplejson as json - -try: - reduce -except NameError: - # Python 3.x. - from functools import reduce - -try: - BaseException -except NameError: - # Python 2.4 - BaseException = Exception - # Ansible since PR #41749 inserts "import __main__" into # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so @@ -82,6 +64,9 @@ sys.modules[str('__main__')] = types.ModuleType(str('__main__')) import ansible.module_utils.json_utils + +from ansible.module_utils.six.moves import reduce + import ansible_mitogen.runner @@ -144,7 +129,7 @@ def subprocess__Popen__close_fds(self, but): if ( sys.platform.startswith(u'linux') and - sys.version < u'3.0' and + sys.version_info < (3,) and hasattr(subprocess.Popen, u'_close_fds') and not mitogen.is_master ): @@ -369,11 +354,6 @@ def init_child(econtext, log_level, candidate_temp_dirs): LOG.setLevel(log_level) logging.getLogger('ansible_mitogen').setLevel(log_level) - # issue #536: if the json module is available, remove simplejson from the - # importer whitelist to avoid confusing certain Ansible modules. - if json.__name__ == 'json': - econtext.importer.whitelist.remove('simplejson') - global _fork_parent if FORK_SUPPORTED: mitogen.parent.upgrade_router(econtext) @@ -622,8 +602,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False): stdout, stderr = proc.communicate(in_data) if emulate_tty: - stdout = stdout.replace(b('\n'), b('\r\n')) - return proc.returncode, stdout, stderr or b('') + stdout = stdout.replace(b'\n', b'\r\n') + return proc.returncode, stdout, stderr or b'' def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False): @@ -652,7 +632,8 @@ def read_path(path): """ Fetch the contents of a filesystem `path` as bytes. """ - return open(path, 'rb').read() + with open(path, 'rb') as f: + return f.read() def set_file_owner(path, owner, group=None, fd=None): @@ -752,9 +733,7 @@ def set_file_mode(path, spec, fd=None): """ Update the permissions of a file using the same syntax as chmod(1). """ - if isinstance(spec, int): - new_mode = spec - elif not mitogen.core.PY3 and isinstance(spec, long): + if isinstance(spec, mitogen.core.integer_types): new_mode = spec elif spec.isdigit(): new_mode = int(spec, 8) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 8a2a13c13..b865f8b4f 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -26,9 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import -from __future__ import unicode_literals - """ Mitogen extends Ansible's target configuration mechanism in several ways that require some care: @@ -60,25 +57,26 @@ from HostVars. """ +from __future__ import absolute_import, division, print_function +from __future__ import unicode_literals +__metaclass__ = type + import abc +import logging import os + import ansible.utils.shlex import ansible.constants as C +import ansible.executor.interpreter_discovery +import ansible.utils.unsafe_proxy from ansible.module_utils.six import with_metaclass +from ansible.module_utils.parsing.convert_bool import boolean -# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary -try: - from ansible.executor.interpreter_discovery import discover_interpreter -except ImportError: - discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python' +import mitogen.core -try: - from ansible.utils.unsafe_proxy import AnsibleUnsafeText -except ImportError: - from ansible.vars.unsafe_proxy import AnsibleUnsafeText -import mitogen.core +LOG = logging.getLogger(__name__) def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_python): @@ -91,12 +89,12 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth # keep trying different interpreters until we don't error if action._finding_python_interpreter: return action._possible_python_interpreter - + if s in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']: # python is the only supported interpreter_name as of Ansible 2.8.8 interpreter_name = 'python' discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name - + if task_vars.get('ansible_facts') is None: task_vars['ansible_facts'] = {} @@ -113,12 +111,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth action._finding_python_interpreter = True # fake pipelining so discover_interpreter can be happy action._connection.has_pipelining = True - s = AnsibleUnsafeText(discover_interpreter( + s = ansible.executor.interpreter_discovery.discover_interpreter( action=action, interpreter_name=interpreter_name, discovery_mode=s, - task_vars=task_vars)) - + task_vars=task_vars, + ) + s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s) # cache discovered interpreter task_vars['ansible_facts'][discovered_interpreter_config] = s action._connection.has_pipelining = False @@ -136,7 +135,7 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth def parse_python_path(s, task_vars, action, rediscover_python): """ Given the string set for ansible_python_interpeter, parse it using shell - syntax and return an appropriate argument vector. If the value detected is + syntax and return an appropriate argument vector. If the value detected is one of interpreter discovery then run that first. Caches python interpreter discovery value in `facts_from_task_vars` like how Ansible handles this. """ @@ -214,6 +213,12 @@ def become(self): :data:`True` if privilege escalation should be active. """ + @abc.abstractmethod + def become_flags(self): + """ + The command line arguments passed to the become executable. + """ + @abc.abstractmethod def become_method(self): """ @@ -244,6 +249,12 @@ def python_path(self): Path to the Python interpreter on the target machine. """ + @abc.abstractmethod + def host_key_checking(self): + """ + Whether or not to check the keys of the target machine + """ + @abc.abstractmethod def private_key_file(self): """ @@ -285,10 +296,9 @@ def become_exe(self): @abc.abstractmethod def sudo_args(self): """ - The list of additional arguments that should be included in a become + The list of additional arguments that should be included in a sudo invocation. """ - # TODO: split out into sudo_args/become_args. @abc.abstractmethod def mitogen_via(self): @@ -354,6 +364,12 @@ def mitogen_machinectl_path(self): The path to the "machinectl" program for the 'setns' transport. """ + @abc.abstractmethod + def mitogen_podman_path(self): + """ + The path to the "podman" program for the 'podman' transport. + """ + @abc.abstractmethod def mitogen_ssh_keepalive_interval(self): """ @@ -406,6 +422,43 @@ def __init__(self, connection, play_context, transport, inventory_name): # used to run interpreter discovery self._action = connection._action + def _become_option(self, name): + plugin = self._connection.become + try: + return plugin.get_option(name, self._task_vars, self._play_context) + except AttributeError: + # A few ansible_mitogen connection plugins look more like become + # plugins. They don't quite fit Ansible's plugin.get_option() API. + # https://github.com/mitogen-hq/mitogen/issues/1173 + fallback_plugins = {'mitogen_doas', 'mitogen_sudo', 'mitogen_su'} + if self._connection.transport not in fallback_plugins: + raise + + fallback_options = { + 'become_exe', + 'become_flags', + } + if name not in fallback_options: + raise + + LOG.info( + 'Used fallback=PlayContext.%s for plugin=%r, option=%r', + name, self._connection, name, + ) + return getattr(self._play_context, name) + + def _connection_option(self, name, fallback_attr=None): + try: + return self._connection.get_option(name, hostvars=self._task_vars) + except KeyError: + if fallback_attr is None: + fallback_attr = name + LOG.info( + 'Used fallback=PlayContext.%s for plugin=%r, option=%r', + fallback_attr, self._connection, name, + ) + return getattr(self._play_context, fallback_attr) + def transport(self): return self._transport @@ -413,28 +466,31 @@ def inventory_name(self): return self._inventory_name def remote_addr(self): - return self._play_context.remote_addr + return self._connection_option('host', fallback_attr='remote_addr') def remote_user(self): - return self._play_context.remote_user + return self._connection_option('remote_user') def become(self): - return self._play_context.become + return self._connection.become + + def become_flags(self): + return self._become_option('become_flags') def become_method(self): - return self._play_context.become_method + return self._connection.become.name def become_user(self): - return self._play_context.become_user + return self._become_option('become_user') def become_pass(self): - return optional_secret(self._play_context.become_pass) + return optional_secret(self._become_option('become_pass')) def password(self): - return optional_secret(self._play_context.password) + return optional_secret(self._connection_option('password')) def port(self): - return self._play_context.port + return self._connection_option('port') def python_path(self, rediscover_python=False): s = self._connection.get_task_var('ansible_python_interpreter', @@ -449,58 +505,37 @@ def python_path(self, rediscover_python=False): action=self._action, rediscover_python=rediscover_python) + def host_key_checking(self): + return self._connection_option('host_key_checking') + def private_key_file(self): - return self._play_context.private_key_file + return self._connection_option('private_key_file') def ssh_executable(self): - return self._play_context.ssh_executable + return self._connection_option('ssh_executable') def timeout(self): - return self._play_context.timeout + return self._connection_option('timeout') def ansible_ssh_timeout(self): - return ( - self._connection.get_task_var('ansible_timeout') or - self._connection.get_task_var('ansible_ssh_timeout') or - self.timeout() - ) + return self.timeout() def ssh_args(self): return [ mitogen.core.to_text(term) for s in ( - getattr(self._play_context, 'ssh_args', ''), - getattr(self._play_context, 'ssh_common_args', ''), - getattr(self._play_context, 'ssh_extra_args', '') + self._connection_option('ssh_args'), + self._connection_option('ssh_common_args'), + self._connection_option('ssh_extra_args'), ) for term in ansible.utils.shlex.shlex_split(s or '') ] def become_exe(self): - # In Ansible 2.8, PlayContext.become_exe always has a default value due - # to the new options mechanism. Previously it was only set if a value - # ("somewhere") had been specified for the task. - # For consistency in the tests, here we make older Ansibles behave like - # newer Ansibles. - exe = self._play_context.become_exe - if exe is None and self._play_context.become_method == 'sudo': - exe = 'sudo' - return exe + return self._become_option('become_exe') def sudo_args(self): - return [ - mitogen.core.to_text(term) - for term in ansible.utils.shlex.shlex_split( - first_true(( - self._play_context.become_flags, - # Ansible <=2.7. - getattr(self._play_context, 'sudo_flags', ''), - # Ansible <=2.3. - getattr(C, 'DEFAULT_BECOME_FLAGS', ''), - getattr(C, 'DEFAULT_SUDO_FLAGS', '') - ), default='') - ) - ] + return ansible.utils.shlex.shlex_split(self.become_flags() or '') def mitogen_via(self): return self._connection.get_task_var('mitogen_via') @@ -529,6 +564,9 @@ def mitogen_lxc_attach_path(self): def mitogen_lxc_info_path(self): return self._connection.get_task_var('mitogen_lxc_info_path') + def mitogen_podman_path(self): + return self._connection.get_task_var('mitogen_podman_path') + def mitogen_ssh_keepalive_interval(self): return self._connection.get_task_var('mitogen_ssh_keepalive_interval') @@ -632,6 +670,9 @@ def remote_user(self): def become(self): return bool(self._become_user) + def become_flags(self): + return self._host_vars.get('ansible_become_flags') + def become_method(self): return ( self._become_method or @@ -644,12 +685,13 @@ def become_user(self): def become_pass(self): return optional_secret( - self._host_vars.get('ansible_become_password') or - self._host_vars.get('ansible_become_pass') + self._host_vars.get('ansible_become_pass') or + self._host_vars.get('ansible_become_password') ) def password(self): return optional_secret( + self._host_vars.get('ansible_ssh_password') or self._host_vars.get('ansible_ssh_pass') or self._host_vars.get('ansible_password') ) @@ -675,6 +717,14 @@ def python_path(self, rediscover_python=False): action=self._action, rediscover_python=rediscover_python) + def host_key_checking(self): + def candidates(): + yield self._host_vars.get('ansible_ssh_host_key_checking') + yield self._host_vars.get('ansible_host_key_checking') + yield C.HOST_KEY_CHECKING + val = next((v for v in candidates() if v is not None), True) + return boolean(val) + def private_key_file(self): # TODO: must come from PlayContext too. return ( @@ -684,10 +734,7 @@ def private_key_file(self): ) def ssh_executable(self): - return ( - self._host_vars.get('ansible_ssh_executable') or - C.ANSIBLE_SSH_EXECUTABLE - ) + return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) def timeout(self): # TODO: must come from PlayContext too. @@ -701,25 +748,13 @@ def ansible_ssh_timeout(self): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - ( - self._host_vars.get('ansible_ssh_args') or - getattr(C, 'ANSIBLE_SSH_ARGS', None) or - os.environ.get('ANSIBLE_SSH_ARGS') - # TODO: ini entry. older versions. - ), - ( - self._host_vars.get('ansible_ssh_common_args') or - os.environ.get('ANSIBLE_SSH_COMMON_ARGS') - # TODO: ini entry. - ), - ( - self._host_vars.get('ansible_ssh_extra_args') or - os.environ.get('ANSIBLE_SSH_EXTRA_ARGS') - # TODO: ini entry. - ), + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s) if s @@ -736,7 +771,7 @@ def sudo_args(self): mitogen.core.to_text(term) for s in ( self._host_vars.get('ansible_sudo_flags') or '', - self._host_vars.get('ansible_become_flags') or '', + self.become_flags() or '', ) for term in ansible.utils.shlex.shlex_split(s) ] @@ -760,7 +795,7 @@ def mitogen_kubectl_path(self): return self._host_vars.get('mitogen_kubectl_path') def mitogen_lxc_path(self): - return self.host_vars.get('mitogen_lxc_path') + return self._host_vars.get('mitogen_lxc_path') def mitogen_lxc_attach_path(self): return self._host_vars.get('mitogen_lxc_attach_path') @@ -768,6 +803,9 @@ def mitogen_lxc_attach_path(self): def mitogen_lxc_info_path(self): return self._host_vars.get('mitogen_lxc_info_path') + def mitogen_podman_path(self): + return self._host_vars.get('mitogen_podman_path') + def mitogen_ssh_keepalive_interval(self): return self._host_vars.get('mitogen_ssh_keepalive_interval') diff --git a/ansible_mitogen/utils/__init__.py b/ansible_mitogen/utils/__init__.py new file mode 100644 index 000000000..a01b261df --- /dev/null +++ b/ansible_mitogen/utils/__init__.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import re + +import ansible + +__all__ = [ + 'ansible_version', +] + + +def _parse(v_string): + # Adapted from distutils.version.LooseVersion.parse() + component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) + for component in component_re.split(v_string): + if not component or component == '.': + continue + try: + yield int(component) + except ValueError: + yield component + + +ansible_version = tuple(_parse(ansible.__version__)) + +del _parse +del re +del ansible diff --git a/ansible_mitogen/utils/unsafe.py b/ansible_mitogen/utils/unsafe.py new file mode 100644 index 000000000..b2c3d5338 --- /dev/null +++ b/ansible_mitogen/utils/unsafe.py @@ -0,0 +1,79 @@ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import ansible +import ansible.utils.unsafe_proxy + +import ansible_mitogen.utils + +import mitogen +import mitogen.core +import mitogen.utils + +__all__ = [ + 'cast', +] + +def _cast_to_dict(obj): return {cast(k): cast(v) for k, v in obj.items()} +def _cast_to_list(obj): return [cast(v) for v in obj] +def _cast_unsafe(obj): return obj._strip_unsafe() +def _passthrough(obj): return obj + + +# A dispatch table to cast objects based on their exact type. +# This is an optimisation, reliable fallbacks are required (e.g. isinstance()) +_CAST_DISPATCH = { + bytes: bytes, + dict: _cast_to_dict, + list: _cast_to_list, + tuple: _cast_to_list, + mitogen.core.UnicodeType: mitogen.core.UnicodeType, +} +_CAST_DISPATCH.update({t: _passthrough for t in mitogen.utils.PASSTHROUGH}) + +if hasattr(ansible.utils.unsafe_proxy.AnsibleUnsafeText, '_strip_unsafe'): + _CAST_DISPATCH.update({ + ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: _cast_unsafe, + ansible.utils.unsafe_proxy.AnsibleUnsafeText: _cast_unsafe, + ansible.utils.unsafe_proxy.NativeJinjaUnsafeText: _cast_unsafe, + }) +elif ansible_mitogen.utils.ansible_version[:2] <= (2, 16): + _CAST_DISPATCH.update({ + ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes, + ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType, + }) +else: + mitogen_ver = '.'.join(str(v) for v in mitogen.__version__) + raise ImportError("Mitogen %s can't unwrap Ansible %s AnsibleUnsafe objects" + % (mitogen_ver, ansible.__version__)) + + +def cast(obj): + """ + Return obj (or a copy) with subtypes of builtins cast to their supertype. + + This is an enhanced version of :func:`mitogen.utils.cast`. In addition it + handles ``ansible.utils.unsafe_proxy.AnsibleUnsafeText`` and variants. + + There are types handled by :func:`ansible.utils.unsafe_proxy.wrap_var()` + that this function currently does not handle (e.g. `set()`), or preserve + preserve (e.g. `tuple()`). Future enhancements may change this. + + :param obj: + Object to undecorate. + :returns: + Undecorated object. + """ + # Fast path: obj is a known type, dispatch directly + try: + unwrapper = _CAST_DISPATCH[type(obj)] + except KeyError: + pass + else: + return unwrapper(obj) + + # Slow path: obj is some unknown subclass + if isinstance(obj, dict): return _cast_to_dict(obj) + if isinstance(obj, (list, tuple)): return _cast_to_list(obj) + + return mitogen.utils.cast(obj) diff --git a/docs/_templates/ansible.html b/docs/_templates/ansible.html deleted file mode 100644 index 770e0a458..000000000 --- a/docs/_templates/ansible.html +++ /dev/null @@ -1,14 +0,0 @@ - -Mitogen for Ansible (Redirect) - - - - diff --git a/docs/_templates/github.html b/docs/_templates/github.html index bb2b5ee57..e6ed304a2 100644 --- a/docs/_templates/github.html +++ b/docs/_templates/github.html @@ -1,4 +1,4 @@


-Star +Star

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 97771afa3..a01b497fe 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -14,23 +14,5 @@ {% block footer %} {{ super() }} - - - - - {% endblock %} diff --git a/docs/_templates/piwik-config.js b/docs/_templates/piwik-config.js deleted file mode 100644 index ad24c9f64..000000000 --- a/docs/_templates/piwik-config.js +++ /dev/null @@ -1,5 +0,0 @@ -window._paq = []; -window._paq.push(['trackPageView']); -window._paq.push(['enableLinkTracking']); -window._paq.push(['enableHeartBeatTimer', 30]); -window._paq.push(['setSiteId', 6]); diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index c9bbcf518..3d80a290a 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -18,7 +18,7 @@ The extension is considered stable and real-world use is encouraged. .. _Ansible: https://www.ansible.com/ -.. _Bug reports: https://goo.gl/yLKZiJ +.. _Bug reports: https://github.com/mitogen-hq/mitogen/issues/new/choose Overview @@ -75,34 +75,6 @@ Installation ``mitogen_host_pinned`` strategies exists to mimic the ``free`` and ``host_pinned`` strategies. -4. - - .. raw:: html - -
- - - Get notified of new releases and important fixes. - -

-
- - - ↻ - - - -

- - - -

-

- Demo ~~~~ @@ -145,9 +117,35 @@ Testimonials Noteworthy Differences ---------------------- -* Ansible 2.3-2.9 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify - your installation is running one of these versions by checking ``ansible - --version`` output. +* Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. + Mitogen 0.3.1+ supports + + +-----------------+-----------------+ + | Ansible version | Python versions | + +=================+=================+ + | 2.10 | | + +-----------------+ | + | 3 | 2.7, 3.6 - 3.11 | + +-----------------+ | + | 4 | | + +-----------------+-----------------+ + | 5 | 3.8 - 3.11 | + +-----------------+-----------------+ + | 6 | | + +-----------------+ 3.8 - 3.13 | + | 7 | | + +-----------------+-----------------+ + | 8 | 3.9 - 3.13 | + +-----------------+-----------------+ + | 9 | | + +-----------------+ 3.10 - 3.13 | + | 10 | | + +-----------------+-----------------+ + | 11 | 3.11 - 3.13 | + +-----------------+-----------------+ + + Verify your installation is running one of these versions by checking + ``ansible --version`` output. * The ``raw`` action executes as a regular Mitogen connection, which requires Python on the target, precluding its use for installing Python. This will be @@ -185,9 +183,9 @@ Noteworthy Differences your_ssh_username = (ALL) NOPASSWD:/usr/bin/python -c* * The :ans:conn:`~buildah`, :ans:conn:`~docker`, :ans:conn:`~jail`, - :ans:conn:`~kubectl`, :ans:conn:`~local`, :ans:conn:`~lxd`, and - :ans:conn:`~ssh` built-in connection types are supported, along with - Mitogen-specific :ref:`machinectl `, :ref:`mitogen_doas `, + :ans:conn:`~kubectl`, :ans:conn:`~local`, :ans:conn:`~lxd`, + :ans:conn:`~podman`, & :ans:conn:`~ssh` connection types are supported; also + Mitogen-specific :ref:`mitogen_doas `, :ref:`machinectl `, :ref:`mitogen_su `, :ref:`mitogen_sudo `, and :ref:`setns ` types. File bugs to register interest in others. @@ -225,6 +223,15 @@ Noteworthy Differences part of the core library, and should therefore be straightforward to fix as part of 0.2.x. +* Connection and become timeouts are applied differently. Mitogen may consider + a connection to have timed out, when Ansible would have waited longer or + indefinately. For example if SSH authentication completes within the + timeout, but execution of login scripts exceeds it - then Mitogen will + consider the task to have timed out and that host to have failed. + +.. + tests/ansible/integration/ssh/timeouts.yml covers (some of) this behaviour. + .. * SSH and ``become`` are treated distinctly when applying timeouts, and timeouts apply up to the point when the new interpreter is ready to accept @@ -240,15 +247,14 @@ Noteworthy Differences * "Module Replacer" style modules are not supported. These rarely appear in practice, and light web searches failed to reveal many examples of them. -.. - * The ``ansible_python_interpreter`` variable is parsed using a restrictive - :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env - FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``, which - occur in practice. Jinja2 templating is also supported for complex task-level - interpreter settings. Ansible `documents this - `_ - as an absolute path, however the implementation passes it unquoted through - the shell, permitting arbitrary code to be injected. +* The ``ansible_python_interpreter`` variable is parsed using a restrictive + :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env + FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``. + Jinja2 templating is also supported for complex task-level + interpreter settings. Ansible documents `ansible_python_interpreter + `_ + as an absolute path and releases since June 2024 (e.g. Ansible 10.1) + reflect this. Older Ansible releases passed it to the shell unquoted. .. * Configurations will break that rely on the `hashbang argument splitting @@ -302,7 +308,8 @@ container. * Intermediary machines cannot use login and become passwords that were supplied to Ansible interactively. If an intermediary requires a password, it must be supplied via ``ansible_ssh_pass``, - ``ansible_password``, or ``ansible_become_pass`` inventory variables. + ``ansible_ssh_password``, ``ansible_password``, or + ``ansible_become_pass`` inventory variables. * Automatic tunnelling of SSH-dependent actions, such as the ``synchronize`` module, is not yet supported. This will be addressed in a @@ -816,6 +823,20 @@ Like the :ans:conn:`local` except connection delegation is supported. * ``ansible_python_interpreter`` +Podman +~~~~~~ + +Like :ans:conn:`podman` except connection delegation is supported. + +* ``ansible_host``: Name of container (default: inventory hostname). +* ``ansible_user``: Name of user within the container to execute as. +* ``mitogen_mask_remote_name``: if :data:`True`, mask the identity of the + Ansible controller process on remote machines. To simplify diagnostics, + Mitogen produces remote processes named like + `"mitogen:user@controller.name:1234"`, however this may be a privacy issue in + some circumstances. + + Process Model ^^^^^^^^^^^^^ @@ -993,7 +1014,8 @@ Like the :ans:conn:`ssh` except connection delegation is supported. * ``ansible_port``, ``ssh_port`` * ``ansible_ssh_executable``, ``ssh_executable`` * ``ansible_ssh_private_key_file`` -* ``ansible_ssh_pass``, ``ansible_password`` (default: assume passwordless) +* ``ansible_ssh_pass``, ``ansible_ssh_password``, ``ansible_password`` + (default: assume passwordless) * ``ssh_args``, ``ssh_common_args``, ``ssh_extra_args`` * ``mitogen_mask_remote_name``: if :data:`True`, mask the identity of the Ansible controller process on remote machines. To simplify diagnostics, @@ -1231,18 +1253,17 @@ with ``-vvv``. However, certain controller hangs may render ``MITOGEN_DUMP_THREAD_STACKS`` ineffective, or occur too infrequently for interactive reproduction. In these -cases `faulthandler `_ may be used: +cases :py:mod:`faulthandler` may be used with Python >= 3.3: -1. For Python 2, ``pip install faulthandler``. This is unnecessary on Python 3. -2. Once the hang occurs, observe the process tree using ``pstree`` or ``ps +1. Once the hang occurs, observe the process tree using ``pstree`` or ``ps --forest``. -3. The most likely process to be hung is the connection multiplexer, which can +2. The most likely process to be hung is the connection multiplexer, which can easily be identified as the parent of all SSH client processes. -4. Send ``kill -SEGV `` to the multiplexer PID, causing it to print all +3. Send ``kill -SEGV `` to the multiplexer PID, causing it to print all thread stacks. -5. `File a bug `_ including a copy - of the stacks, along with a description of the last task executing prior to - the hang. +4. `File a bug `_ + including a copy of the stacks and a description of the last task executing + before the hang It is possible the hang occurred in a process on a target. If ``strace`` is available, look for the host name not listed in Ansible output as reporting a @@ -1256,7 +1277,7 @@ on each process whose name begins with ``mitogen:``:: [pid 29858] futex(0x55ea9be52f60, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, 0xffffffff ^C - $ + $ This shows one thread waiting on IO (``poll``) and two more waiting on the same lock. It is taken from a real example of a deadlock due to a forking bug. @@ -1275,7 +1296,7 @@ Sample Profiles --------------- The summaries below may be reproduced using data and scripts maintained in the -`pcaps branch `_. Traces were +`pcaps branch `_. Traces were recorded using Ansible 2.5.14. @@ -1284,7 +1305,7 @@ Trivial Loop: Local Host This demonstrates Mitogen vs. SSH pipelining to the local machine running `bench/loop-100-items.yml -`_, +`_, executing a simple command 100 times. Most Ansible controller overhead is isolated, characterizing just module executor and connection layer performance. Mitogen requires **63x less bandwidth and 5.9x less time**. @@ -1312,7 +1333,7 @@ File Transfer: UK to France ~~~~~~~~~~~~~~~~~~~~~~~~~~~ `This playbook -`_ +`_ was used to compare file transfer performance over a ~26 ms link. It uses the ``with_filetree`` loop syntax to copy a directory of 1,000 0-byte files to the target. @@ -1374,20 +1395,3 @@ Despite the small margin for optimization, Mitogen still manages **6.2x less bandwidth and 1.8x less time**. .. image:: images/ansible/pcaps/costapp-uk-india.svg - - -.. raw:: html - - - diff --git a/docs/api.rst b/docs/api.rst index 7ab3274ec..3dab62c1d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -95,7 +95,7 @@ Connection Methods :param str container: The name of the Buildah container to connect to. - :param str doas_path: + :param str buildah_path: Filename or complete path to the ``buildah`` binary. ``PATH`` will be searched if given as a filename. Defaults to ``buildah``. :param str username: @@ -367,6 +367,20 @@ Connection Methods Filename or complete path to the ``lxc`` binary. ``PATH`` will be searched if given as a filename. Defaults to ``lxc``. +.. currentmodule:: mitogen.parent +.. method:: Router.podman (container=None, podman_path=None, username=None, \**kwargs) + + Construct a context on the local machine over a ``podman`` invocation. + Accepts all parameters accepted by :meth:`local`, in addition to: + + :param str container: + The name of the Podman container to connect to. + :param str podman_path: + Filename or complete path to the ``podman`` binary. ``PATH`` will be + searched if given as a filename. Defaults to ``podman``. + :param str username: + Username to use, defaults to unset. + .. method:: Router.setns (container, kind, username=None, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs) Construct a context in the style of :meth:`local`, but change the diff --git a/docs/changelog.rst b/docs/changelog.rst index 7acd4f8de..47bb04f7c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,21 +15,240 @@ Release Notes To avail of fixes in an unreleased version, please download a ZIP file -`directly from GitHub `_. +`directly from GitHub `_. -v0.3.0 (unreleased) + +In progress (unreleased) +------------------------ + +* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix :ans:mod:`wait_for_connection` + timeout with templated ``ansible_python_interpreter`` +* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix templated python interpreter + with `meta: reset_connection` +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated connection timeout + (e.g. ``ansible_timeout``). +* :gh:issue:`740` respect `interpreter_python` global configuration variable + + +v0.3.19 (2024-12-02) +-------------------- + +* :gh:issue:`1129` :mod:`ansible_mitogen`: Ansible 11 support + + +v0.3.18 (2024-11-07) +-------------------- + +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become method + (e.g. ``ansible_become_method``). +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become flag + (e.g. ``ansible_become_method``, ``become`` keyword). + + +v0.3.17 (2024-11-07) +-------------------- + +* :gh:issue:`1182` CI: Fix incorrect world readable/writable file permissions + on SSH key ``mitogen__has_sudo_pubkey.key`` during Ansible tests. +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated SSH private key file + (e.g. ``ansible_private_key_file``). +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated SSH host key checking + (e.g. ``ansible_host_key_checking``, ``ansible_ssh_host_key_checking``). +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated host address + (e.g. ``ansible_host``, ``ansible_ssh_host``) +* :gh:issue:`1184` Test templated SSH host key checking in task vars + + +v0.3.16 (2024-11-05) +-------------------- + +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become executable + (e.g. ``become_exe``). +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become executable + arguments (e.g. ``become_flags``). +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated ssh executable + (``ansible_ssh_executable``). +* :gh:issue:`1083` :mod:`ansible_mitogen`: Fixed templated connection options + during a ``meta: reset_connection`` task. +* :gh:issue:`1129` CI: Migrated macOS 12 runners to macOS 13, due to EOL. + + +v0.3.15 (2024-10-28) +-------------------- + +* :gh:issue:`905` :mod:`ansible_mitogen`: Support templated SSH command + arguments (e.g. ``ansible_ssh_args``, ``ansible_ssh_extra_args``). +* :gh:issue:`692` tests: Fix and re-enable several sudo tests +* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become password + (e.g. ``ansible_become_pass``, ``ansible_sudo_pass``) + + +v0.3.14 (2024-10-16) +-------------------- + +* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker + SSH tests +* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become username. + + +v0.3.13 (2024-10-09) +-------------------- + +* :gh:issue:`1138` CI: Complete migration from Azure DevOps Pipelines to + GitHub Actions +* :gh:issue:`1116` :mod:`ansible_mitogen`: Support for templated variable + `ansible_ssh_user`. +* :gh:issue:`978` :mod:`ansible_mitogen`: Support templated Ansible SSH port. +* :gh:issue:`1073` Python 3.13 support + + +v0.3.12 (2024-10-07) +-------------------- + +* :gh:issue:`1106` :mod:`ansible_mitogen`: Support for `ansible_ssh_password` + connection variable, and templated SSH connection password. +* :gh:issue:`1136` tests: Improve Ansible fail_msg formatting. +* :gh:issue:`1137` tests: Ignore inventory files of inactive tests & benchmarks +* :gh:issue:`1138` CI: Add re-actors/alls-green GitHub Actions job to simplify + branch protections configuration. + + +v0.3.11 (2024-09-30) +-------------------- + +* :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility + fallbacks and polyfills into :mod:`mitogen.core` +* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility + fallbacks for Python 2.4 & 2.5. +* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible + releases before 2.10 +* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3 + compatibility +* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions + + +v0.3.10 (2024-09-20) -------------------- +* :gh:issue:`950` Fix Solaris/Illumos/SmartOS compatibility with become +* :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template + module is called with a ``dest:`` filename that has an extension +* :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy + module is called with a file larger than 124 kibibytes + (:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`) +* :gh:issue:`905` Initial support for templated ``ansible_ssh_args``, + ``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables. + NB: play or task scoped variables will probably still fail. +* :gh:issue:`694` CI: Fixed a race condition and some resource leaks causing + some of intermittent failures when running the test suite. + + +v0.3.9 (2024-08-13) +------------------- + +* :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when + executing non new-style modules (e.g. JSONARGS style, WANT_JSON style). +* :gh:issue:`1074` Support Ansible 10 (ansible-core 2.17) + + +v0.3.8 (2024-07-30) +------------------- + +* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage +* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts + "ValueError: filedescriptor out of range in select()" +* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking` +* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs + + +v0.3.7 (2024-04-08) +------------------- + +* :gh:issue:`1021` Support for Ansible 8 (ansible-core 2.15) +* tests: Replace uses of ``include:`` & ``import:``, unsupported in Ansible 9 +* :gh:issue:`1053` Support for Ansible 9 (ansible-core 2.16) + + +v0.3.6 (2024-04-04) +------------------- + +* :gh:issue:`974` Support Ansible 7 +* :gh:issue:`1046` Raise :py:exc:`TypeError` in :func:`` + when casting a string subtype to `bytes()` or `str()` fails. This is + potentially an API breaking change. Failures previously passed silently. +* :gh:issue:`1046` Add :func:``, to cast + :class:`ansible.utils.unsafe_proxy.AnsibleUnsafe` objects in Ansible 7+. + + +v0.3.5 (2024-03-17) +------------------- + +* :gh:issue:`987` Support Python 3.11 +* :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when + becoming an unprivileged user with Python 3.x +* :gh:issue:`1033` Support `PEP 451 `_, + required by Python 3.12 +* :gh:issue:`1033` Support Python 3.12 + + +v0.3.4 (2023-07-02) +------------------- + +* :gh:issue:`929` Support Ansible 6 and ansible-core 2.13 +* :gh:issue:`832` Fix runtime error when using the ansible.builtin.dnf module multiple times +* :gh:issue:`925` :class:`ansible_mitogen.connection.Connection` no longer tries to close the + connection on destruction. This is expected to reduce cases of `mitogen.core.Error: An attempt + was made to enqueue a message with a Broker that has already exitted`. However it may result in + resource leaks. +* :gh:issue:`659` Removed :mod:`mitogen.compat.simplejson`, not needed with Python 2.7+, contained Python 3.x syntax errors +* :gh:issue:`983` CI: Removed PyPI faulthandler requirement from tests +* :gh:issue:`1001` CI: Fixed Debian 9 & 11 tests + +v0.3.3 (2022-06-03) +------------------- + +* :gh:issue:`906` Support packages dynamically inserted into sys.modules, e.g. `distro` >= 1.7.0 as `ansible.module_utils.distro`. +* :gh:issue:`918` Support Python 3.10 +* :gh:issue:`920` Support Ansible :ans:conn:`~podman` connection plugin +* :gh:issue:`836` :func:`mitogen.utils.with_router` decorator preserves the docstring in addition to the name. +* :gh:issue:`936` :ans:mod:`fetch` no longer emits `[DEPRECATION WARNING]: The '_remote_checksum()' method is deprecated.` + + +v0.3.2 (2022-01-12) +------------------- + +* :gh:issue:`891` Correct `Framework :: Ansible` Trove classifier + + +v0.3.1 (unreleased) +------------------- + +* :gh:issue:`874` Support for Ansible 5 (ansible-core 2.12) +* :gh:issue:`774` Fix bootstrap failures on macOS 11.x and 12.x, involving Python 2.7 wrapper +* :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11) +* :gh:issue:`869` Continuous Integration tests are now run with Tox +* :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04 +* :gh:issue:`860` Add initial support for podman connection (w/o Ansible support yet) +* :gh:issue:`873` `python -c ...` first stage no longer uses :py:mod:`platform`` to detect the macOS release +* :gh:issue:`876` `python -c ...` first stage no longer contains tab characters, to reduce size +* :gh:issue:`878` Continuous Integration tests now correctly perform comparisons of 2 digit versions +* :gh:issue:`878` Kubectl connector fixed with Ansible 2.10 and above + + +v0.3.0 (2021-11-24) +------------------- + This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+. -`See here for details `_. +`See here for details `_. +* :gh:issue:`827` NewStylePlanner: detect `ansible_collections` imports * :gh:issue:`770` better check for supported Ansible version * :gh:issue:`731` ansible 2.10 support * :gh:issue:`652` support for ansible collections import hook -* :gh:issue:`740` respect `interpreter_python` global configuration variable +* :gh:issue:`847` Removed historic Continuous Integration reverse shell -v0.2.10 (unreleased) +v0.2.10 (2021-11-24) -------------------- * :gh:issue:`597` mitogen does not support Ansible 2.8 Python interpreter detection @@ -41,7 +260,7 @@ v0.2.10 (unreleased) * :gh:issue:`756` ssh connections with `check_host_keys='accept'` would timeout, when using recent OpenSSH client versions. * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in - :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` + :py:meth:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` * :gh:issue:`775` Test with Python 3.9 * :gh:issue:`775` Add msvcrt to the default module deny list @@ -127,7 +346,7 @@ Mitogen for Ansible :linux:man7:`unix` sockets across privilege domains. * :gh:issue:`467`: an incompatibility running Mitogen under `Molecule - `_ was resolved. + `_ was resolved. * :gh:issue:`547`, :gh:issue:`598`: fix a deadlock during initialization of connections, ``async`` tasks, tasks using custom :mod:`module_utils`, @@ -1179,9 +1398,8 @@ Core Library parameter may specify an argument vector prefix rather than a string program path. -* :gh:issue:`300`: the broker could crash on - OS X during shutdown due to scheduled `kqueue - `_ filter changes for +* :gh:issue:`300`: the broker could crash on OS X during shutdown due to + scheduled :freebsd:man2:`kqueue` filter changes for descriptors that were closed before the IO loop resumes. As a temporary workaround, kqueue's bulk change feature is not used. diff --git a/docs/conf.py b/docs/conf.py index 54e3a5c71..b7dd15250 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,8 @@ -import os import sys -sys.path.append('..') sys.path.append('.') -import mitogen -VERSION = '%s.%s.%s' % mitogen.__version__ + +VERSION = '0.3.9' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' @@ -18,7 +16,6 @@ html_show_sourcelink = False html_show_sphinx = False html_sidebars = {'**': ['globaltoc.html', 'github.html']} -html_additional_pages = {'ansible': 'ansible.html'} html_static_path = ['_static'] html_theme = 'alabaster' html_theme_options = { @@ -44,15 +41,15 @@ domainrefs = { 'gh:commit': { 'text': '%s', - 'url': 'https://github.com/dw/mitogen/commit/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/commit/%s', }, 'gh:issue': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/issues/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/issues/%s', }, 'gh:pull': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/pull/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/pull/%s', }, 'ans:mod': { 'text': '%s module', @@ -64,30 +61,36 @@ }, 'freebsd:man2': { 'text': '%s(2)', - 'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s', + 'url': 'https://man.freebsd.org/cgi/man.cgi?query=%s', }, 'linux:man1': { 'text': '%s(1)', - 'url': 'http://man7.org/linux/man-pages/man1/%s.1.html', + 'url': 'https://man7.org/linux/man-pages/man1/%s.1.html', }, 'linux:man2': { 'text': '%s(2)', - 'url': 'http://man7.org/linux/man-pages/man2/%s.2.html', + 'url': 'https://man7.org/linux/man-pages/man2/%s.2.html', }, 'linux:man3': { 'text': '%s(3)', - 'url': 'http://man7.org/linux/man-pages/man3/%s.3.html', + 'url': 'https://man7.org/linux/man-pages/man3/%s.3.html', }, 'linux:man7': { 'text': '%s(7)', - 'url': 'http://man7.org/linux/man-pages/man7/%s.7.html', + 'url': 'https://man7.org/linux/man-pages/man7/%s.7.html', }, } +# > ## Official guidance +# > Query PyPI’s JSON API to determine where to download files from. +# > ## Predictable URLs +# > You can use our conveyor service to fetch this file, which exists for +# > cases where using the API is impractical or impossible. +# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls rst_epilog = """ .. |mitogen_version| replace:: %(VERSION)s -.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz `__ +.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz `__ """ % locals() diff --git a/docs/contributors.rst b/docs/contributors.rst index 584c4cd42..ad35f91cf 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -116,6 +116,7 @@ sponsorship and outstanding future-thinking of its early adopters. diff --git a/docs/examples.rst b/docs/examples.rst index c75d8f702..3d5738e44 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -201,7 +201,7 @@ nested.py: print('Connect local%d via %s' % (x, context)) context = router.local(via=context, name='local%d' % x) - context.call(os.system, 'pstree -s python -s mitogen') + context.call(subprocess.check_call, ['pstree', '-s', 'python', '-s', 'mitogen']) Output: diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 05c097e52..d7606b11b 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -813,7 +813,7 @@ executes under the runtime importer lock, ensuring :py:keyword:`import` statements executing in local threads are serialized. .. note:: - + In Python 2, :py:exc:`ImportError` is raised when :py:keyword:`import` is attempted while the runtime import lock is held by another thread, therefore imports must be serialized by only attempting them from the main @@ -1038,7 +1038,7 @@ receive items in the order they are requested, as they become available. Mitogen enables SSH compression by default, there are circumstances where disabling SSH compression is desirable, and many scenarios for future connection methods where transport-layer compression is not supported at - all. + all. .. [#f2] Compression may seem redundant, however it is basically free and reducing IO is always a good idea. The 33% / 200 byte saving may mean the presence or diff --git a/docs/index.rst b/docs/index.rst index 3cd53d32b..32083db0a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,7 +26,7 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or `Fabric`_ can be built, and while the API is quite friendly and comparable to `Fabric`_, ultimately it is not intended for direct use by consumer software. -.. _Salt: https://docs.saltstack.com/en/latest/ +.. _Salt: https://docs.saltproject.io/en/latest/ .. _Ansible: https://docs.ansible.com/ .. _Fabric: https://www.fabfile.org/ @@ -101,7 +101,7 @@ to your network topology**. container='billing0', ) - internal_box.call(os.system, './run-nightly-billing.py') + internal_box.call(subprocess.check_call, ['./run-nightly-billing.py']) The multiplexer also ensures the remote process is terminated if your Python program crashes, communication is lost, or the application code running in the @@ -250,7 +250,7 @@ After: """ Install our application. """ - os.system('tar zxvf app.tar.gz') + subprocess.check_call(['tar', 'zxvf', 'app.tar.gz']) context.call(install_app) @@ -258,7 +258,7 @@ Or even: .. code-block:: python - context.call(os.system, 'tar zxvf app.tar.gz') + context.call(subprocess.check_call, ['tar', 'zxvf', 'app.tar.gz']) Exceptions raised by function calls are propagated back to the parent program, and timeouts can be configured to ensure failed calls do not block progress of diff --git a/docs/requirements.txt b/docs/requirements.txt index 3c4674fdc..a2894c827 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ +docutils<0.18 +Jinja2<3 +MarkupSafe<2.1 Sphinx==2.1.2; python_version > '3.0' sphinxcontrib-programoutput==0.14; python_version > '3.0' alabaster==0.7.10; python_version > '3.0' diff --git a/examples/install_app.py b/examples/install_app.py index 566353a83..569c168e7 100644 --- a/examples/install_app.py +++ b/examples/install_app.py @@ -8,14 +8,14 @@ Where: Hostname to install to. """ -import os +import subprocess import sys import mitogen def install_app(): - os.system('tar zxvf my_app.tar.gz') + subprocess.check_call(['tar', 'zxvf', 'my_app.tar.gz']) @mitogen.main() diff --git a/examples/mitogen-fuse.py b/examples/mitogen-fuse.py index 55b272d91..73101fb87 100644 --- a/examples/mitogen-fuse.py +++ b/examples/mitogen-fuse.py @@ -119,7 +119,7 @@ def _chroot(path): os.chroot(path) -class Operations(fuse.Operations): # fuse.LoggingMixIn, +class Operations(fuse.Operations): # fuse.LoggingMixIn, def __init__(self, host, path='.'): self.host = host self.root = path diff --git a/examples/mitop.py b/examples/mitop.py index 8749e12a9..72a60bf3a 100644 --- a/examples/mitop.py +++ b/examples/mitop.py @@ -61,7 +61,7 @@ def child_main(sender, delay): Executed on the main thread of the Python interpreter running on each target machine, Context.call() from the master. It simply sends the output of the UNIX 'ps' command at regular intervals toward a Receiver on master. - + :param mitogen.core.Sender sender: The Sender to use for delivering our result. This could target anywhere, but the sender supplied by the master simply causes results diff --git a/examples/the_basics.py b/examples/the_basics.py index 0dcd4049d..af9903b00 100644 --- a/examples/the_basics.py +++ b/examples/the_basics.py @@ -10,7 +10,6 @@ import hashlib import io import os -import spwd import mitogen.core import mitogen.master @@ -57,21 +56,6 @@ def streamy_download_file(context, path): } -def get_password_hash(username): - """ - Fetch a user's password hash. - """ - try: - h = spwd.getspnam(username) - except KeyError: - return None - - # mitogen.core.Secret() is a Unicode subclass with a repr() that hides the - # secret data. This keeps secret stuff out of logs. Like blobs, secrets can - # also be serialized. - return mitogen.core.Secret(h) - - def md5sum(path): """ Return the MD5 checksum for a file. diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 9e709d7d5..18932405f 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ #: Library version as a tuple. -__version__ = (0, 3, 0, 'rc', 1) +__version__ = (0, 3, 20, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/buildah.py b/mitogen/buildah.py index f850234d6..7a1e3f808 100644 --- a/mitogen/buildah.py +++ b/mitogen/buildah.py @@ -30,7 +30,6 @@ import logging -import mitogen.core import mitogen.parent diff --git a/mitogen/core.py b/mitogen/core.py index 802ac45ec..49f92cae2 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -34,6 +34,34 @@ bootstrap implementation sent to every new slave context. """ +import sys +try: + import _frozen_importlib_external +except ImportError: + pass +else: + class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder): + """ + Meta path finder for sys.path and package __path__ attributes. + + Patched for https://github.com/python/cpython/issues/115911. + """ + @classmethod + def _path_importer_cache(cls, path): + if path == '': + try: + path = _frozen_importlib_external._os.getcwd() + except (FileNotFoundError, PermissionError): + return None + return super()._path_importer_cache(path) + + if sys.version_info[:2] <= (3, 12): + for i, mpf in enumerate(sys.meta_path): + if mpf is _frozen_importlib_external.PathFinder: + sys.meta_path[i] = MonkeyPatchedPathFinder + del i, mpf + + import binascii import collections import encodings.latin_1 @@ -49,18 +77,22 @@ import signal import socket import struct -import sys import syslog import threading import time import traceback +import types import warnings import weakref import zlib -# Python >3.7 deprecated the imp module. -warnings.filterwarnings('ignore', message='the imp module is deprecated') -import imp +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp # Absolute imports for <2.5. select = __import__('select') @@ -70,21 +102,6 @@ except ImportError: cProfile = None -try: - import thread -except ImportError: - import threading as thread - -try: - import cPickle as pickle -except ImportError: - import pickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - try: BaseException except NameError: @@ -137,31 +154,35 @@ #: :meth:`mitogen.core.Router.add_handler` callbacks to clean up. IS_DEAD = 999 -try: - BaseException -except NameError: - BaseException = Exception - PY24 = sys.version_info < (2, 5) PY3 = sys.version_info > (3,) if PY3: + import pickle + import _thread as thread + from io import BytesIO b = str.encode BytesType = bytes UnicodeType = str FsPathTypes = (str,) BufferType = lambda buf, start: memoryview(buf)[start:] - long = int + integer_types = (int,) + iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values else: + import cPickle as pickle + import thread + from cStringIO import StringIO as BytesIO b = str BytesType = str FsPathTypes = (str, unicode) BufferType = buffer UnicodeType = unicode + integer_types = (int, long) + iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues AnyTextType = (BytesType, UnicodeType) try: - next + next = next except NameError: next = lambda it: it.next() @@ -368,12 +389,19 @@ def to_text(o): # Python 2.4 try: - any + all, any = all, any except NameError: + def all(it): + for elem in it: + if not elem: + return False + return True + def any(it): for elem in it: if elem: return True + return False def _partition(s, sep, find): @@ -386,6 +414,20 @@ def _partition(s, sep, find): return left, sep, s[len(left)+len(sep):] +def threading__current_thread(): + try: + return threading.current_thread() # Added in Python 2.6+ + except AttributeError: + return threading.currentThread() # Deprecated in Python 3.10+ + + +def threading__thread_name(thread): + try: + return thread.name # Added in Python 2.6+ + except AttributeError: + return thread.getName() # Deprecated in Python 3.10+ + + if hasattr(UnicodeType, 'rpartition'): str_partition = UnicodeType.partition str_rpartition = UnicodeType.rpartition @@ -1019,8 +1061,8 @@ def __reduce__(self): def _unpickle_sender(router, context_id, dst_handle): if not (isinstance(router, Router) and - isinstance(context_id, (int, long)) and context_id >= 0 and - isinstance(dst_handle, (int, long)) and dst_handle > 0): + isinstance(context_id, integer_types) and context_id >= 0 and + isinstance(dst_handle, integer_types) and dst_handle > 0): raise TypeError('cannot unpickle Sender: bad input or missing router') return Sender(Context(router, context_id), dst_handle) @@ -1254,6 +1296,7 @@ class Importer(object): 'minify', 'os_fork', 'parent', + 'podman', 'select', 'service', 'setns', @@ -1338,6 +1381,19 @@ def _install_handler(self, router): def __repr__(self): return 'Importer' + @staticmethod + def _loader_from_module(module, default=None): + """Return the loader for a module object.""" + try: + return module.__spec__.loader + except AttributeError: + pass + try: + return module.__loader__ + except AttributeError: + pass + return default + def builtin_find_module(self, fullname): # imp.find_module() will always succeed for __main__, because it is a # built-in module. That means it exists on a special linked list deep @@ -1345,17 +1401,35 @@ def builtin_find_module(self, fullname): if fullname == '__main__': raise ModuleNotFoundError() + # For a module inside a package (e.g. pkg_a.mod_b) use the search path + # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']). parent, _, modname = str_rpartition(fullname, '.') if parent: path = sys.modules[parent].__path__ else: path = None + # For a top-level module search builtin modules, frozen modules, + # system specific locations (e.g. Windows registry, site-packages). + # Otherwise use search path of the parent package. + # Works for both stdlib modules & third-party modules. + # If the search is unsuccessful then raises ImportError. fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() def find_module(self, fullname, path=None): + """ + Return a loader (ourself) or None, for the module with fullname. + + Implements importlib.abc.MetaPathFinder.find_module(). + Deprecrated in Python 3.4+, replaced by find_spec(). + Raises ImportWarning in Python 3.10+. + Removed in Python 3.12. + + fullname Fully qualified module name, e.g. "os.path". + path __path__ of parent packge. None for a top level module. + """ if hasattr(_tls, 'running'): return None @@ -1363,14 +1437,13 @@ def find_module(self, fullname, path=None): try: #_v and self._log.debug('Python requested %r', fullname) fullname = to_text(fullname) - pkgname, dot, _ = str_rpartition(fullname, '.') + pkgname, _, suffix = str_rpartition(fullname, '.') pkg = sys.modules.get(pkgname) if pkgname and getattr(pkg, '__loader__', None) is not self: self._log.debug('%s is submodule of a locally loaded package', fullname) return None - suffix = fullname[len(pkgname+dot):] if pkgname and suffix not in self._present.get(pkgname, ()): self._log.debug('%s has no submodule %s', pkgname, suffix) return None @@ -1390,6 +1463,66 @@ def find_module(self, fullname, path=None): finally: del _tls.running + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`, allowing other finders to try. + + fullname Fully qualified name of the module (e.g. foo.bar.baz) + path Path entries to search. None for a top-level module. + target Existing module to be reloaded (if any). + + Implements importlib.abc.MetaPathFinder.find_spec() + Python 3.4+. + """ + # Presence of _tls.running indicates we've re-invoked importlib. + # Abort early to prevent infinite recursion. See below. + if hasattr(_tls, 'running'): + return None + + log = self._log.getChild('find_spec') + + if fullname.endswith('.'): + return None + + pkgname, _, modname = fullname.rpartition('.') + if pkgname and modname not in self._present.get(pkgname, ()): + log.debug('Skipping %s. Parent %s has no submodule %s', + fullname, pkgname, modname) + return None + + pkg = sys.modules.get(pkgname) + pkg_loader = self._loader_from_module(pkg) + if pkgname and pkg_loader is not self: + log.debug('Skipping %s. Parent %s was loaded by %r', + fullname, pkgname, pkg_loader) + return None + + # #114: whitelisted prefixes override any system-installed package. + if self.whitelist != ['']: + if any(s and fullname.startswith(s) for s in self.whitelist): + log.debug('Handling %s. It is whitelisted', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + if fullname == '__main__': + log.debug('Handling %s. A special case', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + # Re-invoke the import machinery to allow other finders to try. + # Set a guard, so we don't infinitely recurse. See top of this method. + _tls.running = True + try: + spec = importlib.util._find_spec(fullname, path, target) + finally: + del _tls.running + + if spec: + log.debug('Skipping %s. Available as %r', fullname, spec) + return spec + + log.debug('Handling %s. Unavailable locally', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + blacklisted_msg = ( '%r is present in the Mitogen importer blacklist, therefore this ' 'context will not attempt to request it from the master, as the ' @@ -1476,7 +1609,71 @@ def _request_module(self, fullname, callback): if present: callback() + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + self._log.debug('Creating module for %r', spec) + + # FIXME Should this be done in find_spec()? Can it? + self._refuse_imports(spec.name) + + # FIXME "create_module() should properly handle the case where it is + # called more than once for the same spec/module." -- PEP-451 + event = threading.Event() + self._request_module(spec.name, callback=event.set) + event.wait() + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[spec.name] + + if path is None: + raise ImportError(self.absent_msg % (spec.name)) + + spec.origin = self.get_filename(spec.name) + if pkg_present is not None: + # TODO Namespace packages + spec.submodule_search_locations = [] + self._present[spec.name] = pkg_present + + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + name = module.__spec__.name + origin = module.__spec__.origin + self._log.debug('Executing %s from %s', name, origin) + source = self.get_source(name) + try: + # Compile the source into a code object. Don't add any __future__ + # flags and don't inherit any from this module. + # FIXME Should probably be exposed as get_code() + code = compile(source, origin, 'exec', flags=0, dont_inherit=True) + except SyntaxError: + # FIXME Why is this LOG, rather than self._log? + LOG.exception('while importing %r', name) + raise + + exec(code, module.__dict__) + def load_module(self, fullname): + """ + Return the loaded module specified by fullname. + + Implements importlib.abc.Loader.load_module(). + Deprecated in Python 3.4+, replaced by create_module() & exec_module(). + """ fullname = to_text(fullname) _v and self._log.debug('requesting %s', fullname) self._refuse_imports(fullname) @@ -1485,11 +1682,11 @@ def load_module(self, fullname): self._request_module(fullname, event.set) event.wait() - ret = self._cache[fullname] - if ret[2] is None: + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[fullname] + if path is None: raise ModuleNotFoundError(self.absent_msg % (fullname,)) - pkg_present = ret[1] mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = self.get_filename(fullname) mod.__loader__ = self @@ -2307,7 +2504,7 @@ def __repr__(self): def _unpickle_context(context_id, name, router=None): - if not (isinstance(context_id, (int, long)) and context_id >= 0 and ( + if not (isinstance(context_id, integer_types) and context_id >= 0 and ( (name is None) or (isinstance(name, UnicodeType) and len(name) < 100)) ): @@ -2322,8 +2519,7 @@ class Poller(object): """ A poller manages OS file descriptors the user is waiting to become available for IO. The :meth:`poll` method blocks the calling thread - until one or more become ready. The default implementation is based on - :func:`select.poll`. + until one or more become ready. Each descriptor has an associated `data` element, which is unique for each readiness type, and defaults to being the same as the file descriptor. The @@ -2345,19 +2541,13 @@ class Poller(object): a resource leak. Pollers may only be used by one thread at a time. + + This implementation uses :func:`select.select` for wider platform support. + That is considered an implementation detail. Previous versions have used + :func:`select.poll`. Future versions may decide at runtime. """ SUPPORTED = True - # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has - # no upper FD limit, it is suitable for use with Latch, which must handle - # FDs larger than select's limit during many-host runs. We want this - # because poll() requires no setup and teardown: just a single system call, - # which is important because Latch.get() creates a Poller on each - # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in - # this scenario. If select() must return in future, it is important - # Latch.poller_class is set from parent.py to point to the industrial - # strength poller for the OS, otherwise Latch will fail randomly. - #: Increments on every poll(). Used to version _rfds and _wfds. _generation = 1 @@ -2480,11 +2670,10 @@ class Latch(object): See :ref:`waking-sleeping-threads` for further discussion. """ - #: The :class:`Poller` implementation to use for waiting. Since the poller - #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller` - #: if it is available, or :class:`mitogen.core.Poller` otherwise, since - #: these implementations require no system calls to create, configure or - #: destroy. + #: The :class:`Poller` implementation to use. Instances are short lived so + #: prefer :class:`mitogen.parent.PollPoller` if it's available, otherwise + #: :class:`mitogen.core.Poller`. They don't need syscalls to create, + #: configure, or destroy. Replaced during import of :mod:`mitogen.parent`. poller_class = Poller #: If not :data:`None`, a function invoked as `notify(latch)` after a @@ -2686,7 +2875,7 @@ def _get_sleep(self, poller, timeout, block, rsock, wsock, cookie): raise e assert cookie == got_cookie, ( - "Cookie incorrect; got %r, expected %r" \ + "Cookie incorrect; got %r, expected %r" % (binascii.hexlify(got_cookie), binascii.hexlify(cookie)) ) @@ -2741,7 +2930,7 @@ def __repr__(self): return 'Latch(%#x, size=%d, t=%r)' % ( id(self), len(self._queue), - threading.currentThread().getName(), + threading__thread_name(threading__current_thread()), ) @@ -3101,7 +3290,7 @@ def stream_by_id(self, dst_id): This can be used from any thread, but its output is only meaningful from the context of the :class:`Broker` thread, as disconnection or replacement could happen in parallel on the broker thread at any - moment. + moment. """ return ( self._stream_by_id.get(dst_id) or @@ -3641,7 +3830,6 @@ def _on_broker_shutdown(self): self._service_recv.notify = None self.recv.close() - @classmethod @takes_econtext def forget_chain(cls, chain_id, econtext): @@ -3891,7 +4079,7 @@ def _setup_importer(self): def _setup_package(self): global mitogen - mitogen = imp.new_module('mitogen') + mitogen = types.ModuleType('mitogen') mitogen.__package__ = 'mitogen' mitogen.__path__ = [] mitogen.__loader__ = self.importer diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index e62cf84a7..2d6602482 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -103,7 +103,6 @@ import threading import mitogen.core -import mitogen.master import mitogen.parent from mitogen.core import LOG, IOLOG @@ -200,7 +199,7 @@ def _on_proc_exit(self, status): def _on_stdin(self, msg): if msg.is_dead: - IOLOG.debug('%r._on_stdin() -> %r', self, data) + IOLOG.debug('%r._on_stdin() -> %r', self, msg) self.pump.protocol.close() return @@ -437,7 +436,7 @@ def run(dest, router, args, deadline=None, econtext=None): fp.write(inspect.getsource(mitogen.core)) fp.write('\n') fp.write('ExternalContext(%r).main()\n' % ( - _get_econtext_config(context, sock2), + _get_econtext_config(econtext, sock2), )) finally: fp.close() diff --git a/mitogen/kubectl.py b/mitogen/kubectl.py index 374ab7470..5d3994ae5 100644 --- a/mitogen/kubectl.py +++ b/mitogen/kubectl.py @@ -28,7 +28,6 @@ # !mitogen: minify_safe -import mitogen.core import mitogen.parent diff --git a/mitogen/lxc.py b/mitogen/lxc.py index a86ce5f0f..21dfef59e 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -28,7 +28,6 @@ # !mitogen: minify_safe -import mitogen.core import mitogen.parent diff --git a/mitogen/lxd.py b/mitogen/lxd.py index 675dddcdc..09034abf6 100644 --- a/mitogen/lxd.py +++ b/mitogen/lxd.py @@ -28,7 +28,6 @@ # !mitogen: minify_safe -import mitogen.core import mitogen.parent diff --git a/mitogen/master.py b/mitogen/master.py index e54795cb4..865c9dc14 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -37,7 +37,6 @@ import dis import errno -import imp import inspect import itertools import logging @@ -50,6 +49,16 @@ import types import zlib +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util + from _imp import is_builtin as _is_builtin +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + from imp import is_builtin as _is_builtin + try: import sysconfig except ImportError: @@ -65,9 +74,11 @@ import mitogen.minify import mitogen.parent +from mitogen.core import any from mitogen.core import b from mitogen.core import IOLOG from mitogen.core import LOG +from mitogen.core import next from mitogen.core import str_partition from mitogen.core import str_rpartition from mitogen.core import to_text @@ -75,17 +86,6 @@ imap = getattr(itertools, 'imap', map) izip = getattr(itertools, 'izip', zip) -try: - any -except NameError: - from mitogen.core import any - -try: - next -except NameError: - from mitogen.core import next - - RLOG = logging.getLogger('mitogen.ctx') @@ -108,7 +108,7 @@ def _stdlib_paths(): ] prefixes = (getattr(sys, a, None) for a in attr_candidates) version = 'python%s.%s' % sys.version_info[0:2] - s = set(os.path.abspath(os.path.join(p, 'lib', version)) + s = set(os.path.realpath(os.path.join(p, 'lib', version)) for p in prefixes if p is not None) # When running 'unit2 tests/module_finder_test.py' in a Py2 venv on Ubuntu @@ -122,7 +122,16 @@ def is_stdlib_name(modname): """ Return :data:`True` if `modname` appears to come from the standard library. """ - if imp.is_builtin(modname) != 0: + # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib. + # Returns 1 if modname names a module that is "builtin" to the the Python + # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces'). + # + # """ + # Main is a little special - imp.is_builtin("__main__") will return False, + # but BuiltinImporter is still the most appropriate initial setting for + # its __loader__ attribute. + # """ -- comment in CPython pylifecycle.c:add_main_module() + if _is_builtin(modname) != 0: return True module = sys.modules.get(modname) @@ -453,6 +462,9 @@ class FinderMethod(object): name according to the running Python interpreter. You'd think this was a simple task, right? Naive young fellow, welcome to the real world. """ + def __init__(self): + self.log = LOG.getChild(self.__class__.__name__) + def __repr__(self): return '%s()' % (type(self).__name__,) @@ -512,42 +524,57 @@ def find(self, fullname): Find `fullname` using :func:`pkgutil.find_loader`. """ try: + # If fullname refers to a submodule that's not already imported + # then the containing package is imported. # Pre-'import spec' this returned None, in Python3.6 it raises # ImportError. loader = pkgutil.find_loader(fullname) except ImportError: e = sys.exc_info()[1] - LOG.debug('%r._get_module_via_pkgutil(%r): %s', - self, fullname, e) + LOG.debug('%r: find_loader(%r) failed: %s', self, fullname, e) return None - IOLOG.debug('%r._get_module_via_pkgutil(%r) -> %r', - self, fullname, loader) if not loader: + LOG.debug('%r: find_loader(%r) returned %r, aborting', + self, fullname, loader) return try: - path, is_special = _py_filename(loader.get_filename(fullname)) - source = loader.get_source(fullname) - is_pkg = loader.is_package(fullname) - - # workaround for special python modules that might only exist in memory - if is_special and is_pkg and not source: - source = '\n' - except (AttributeError, ImportError): - # - Per PEP-302, get_source() and is_package() are optional, - # calling them may throw AttributeError. + path = loader.get_filename(fullname) + except (AttributeError, ImportError, ValueError): # - get_filename() may throw ImportError if pkgutil.find_loader() # picks a "parent" package's loader for some crap that's been # stuffed in sys.modules, for example in the case of urllib3: # "loader for urllib3.contrib.pyopenssl cannot handle # requests.packages.urllib3.contrib.pyopenssl" e = sys.exc_info()[1] - LOG.debug('%r: loading %r using %r failed: %s', - self, fullname, loader, e) + LOG.debug('%r: %r.get_file_name(%r) failed: %r', self, loader, fullname, e) return + path, is_special = _py_filename(path) + + try: + source = loader.get_source(fullname) + except AttributeError: + # Per PEP-302, get_source() is optional, + e = sys.exc_info()[1] + LOG.debug('%r: %r.get_source() failed: %r', self, loader, fullname, e) + return + + try: + is_pkg = loader.is_package(fullname) + except AttributeError: + # Per PEP-302, is_package() is optional, + e = sys.exc_info()[1] + LOG.debug('%r: %r.is_package(%r) failed: %r', self, loader, fullname, e) + return + + # workaround for special python modules that might only exist in memory + if is_special and is_pkg and not source: + source = '\n' + if path is None or source is None: + LOG.debug('%r: path=%r, source=%r, aborting', self, path, source) return if isinstance(source, mitogen.core.UnicodeType): @@ -567,23 +594,37 @@ def find(self, fullname): """ Find `fullname` using its :data:`__file__` attribute. """ - module = sys.modules.get(fullname) + try: + module = sys.modules[fullname] + except KeyError: + LOG.debug('%r: sys.modules[%r] absent, aborting', self, fullname) + return + if not isinstance(module, types.ModuleType): - LOG.debug('%r: sys.modules[%r] absent or not a regular module', - self, fullname) + LOG.debug('%r: sys.modules[%r] is %r, aborting', + self, fullname, module) return - LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) - alleged_name = getattr(module, '__name__', None) - if alleged_name != fullname: - LOG.debug('sys.modules[%r].__name__ is incorrect, assuming ' - 'this is a hacky module alias and ignoring it. ' - 'Got %r, module object: %r', - fullname, alleged_name, module) + try: + resolved_name = module.__name__ + except AttributeError: + LOG.debug('%r: %r has no __name__, aborting', self, module) + return + + if resolved_name != fullname: + LOG.debug('%r: %r.__name__ is %r, aborting', + self, module, resolved_name) return - path, _ = _py_filename(getattr(module, '__file__', '')) + try: + path = module.__file__ + except AttributeError: + LOG.debug('%r: %r has no __file__, aborting', self, module) + return + + path, _ = _py_filename(path) if not path: + LOG.debug('%r: %r.__file__ is %r, aborting', self, module, path) return LOG.debug('%r: sys.modules[%r]: found %s', self, fullname, path) @@ -605,13 +646,13 @@ def find(self, fullname): return path, source, is_pkg -class ParentEnumerationMethod(FinderMethod): +class ParentImpEnumerationMethod(FinderMethod): """ Attempt to fetch source code by examining the module's (hopefully less insane) parent package, and if no insane parents exist, simply use :mod:`sys.path` to search for it from scratch on the filesystem using the normal Python lookup mechanism. - + This is required for older versions of :mod:`ansible.compat.six`, :mod:`plumbum.colors`, Ansible 2.8 :mod:`ansible.module_utils.distro` and its submodule :mod:`ansible.module_utils.distro._distro`. @@ -628,10 +669,24 @@ class ParentEnumerationMethod(FinderMethod): module object or any parent package's :data:`__path__`, since they have all been overwritten. Some men just want to watch the world burn. """ + + @staticmethod + def _iter_parents(fullname): + """ + >>> list(ParentEnumerationMethod._iter_parents('a')) + [('', 'a')] + >>> list(ParentEnumerationMethod._iter_parents('a.b.c')) + [('a.b', 'c'), ('a', 'b'), ('', 'a')] + """ + while fullname: + fullname, _, modname = str_rpartition(fullname, u'.') + yield fullname, modname + def _find_sane_parent(self, fullname): """ Iteratively search :data:`sys.modules` for the least indirect parent of - `fullname` that is loaded and contains a :data:`__path__` attribute. + `fullname` that's from the same package and has a :data:`__path__` + attribute. :return: `(parent_name, path, modpath)` tuple, where: @@ -644,21 +699,40 @@ def _find_sane_parent(self, fullname): * `modpath`: list of module name components leading from `path` to the target module. """ - path = None modpath = [] - while True: - pkgname, _, modname = str_rpartition(to_text(fullname), u'.') + for pkgname, modname in self._iter_parents(fullname): modpath.insert(0, modname) if not pkgname: return [], None, modpath - pkg = sys.modules.get(pkgname) - path = getattr(pkg, '__path__', None) - if pkg and path: - return pkgname.split('.'), path, modpath + try: + pkg = sys.modules[pkgname] + except KeyError: + LOG.debug('%r: sys.modules[%r] absent, skipping', self, pkgname) + continue + + try: + resolved_pkgname = pkg.__name__ + except AttributeError: + LOG.debug('%r: %r has no __name__, skipping', self, pkg) + continue + + if resolved_pkgname != pkgname: + LOG.debug('%r: %r.__name__ is %r, skipping', + self, pkg, resolved_pkgname) + continue + + try: + path = pkg.__path__ + except AttributeError: + LOG.debug('%r: %r has no __path__, skipping', self, pkg) + continue + + if not path: + LOG.debug('%r: %r.__path__ is %r, skipping', self, pkg, path) + continue - LOG.debug('%r: %r lacks __path__ attribute', self, pkgname) - fullname = pkgname + return pkgname.split('.'), path, modpath def _found_package(self, fullname, path): path = os.path.join(path, '__init__.py') @@ -690,6 +764,7 @@ def _found_module(self, fullname, path, fp, is_pkg=False): def _find_one_component(self, modname, search_path): try: #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) + # FIXME The imp module was removed in Python 3.12. return imp.find_module(modname, search_path) except ImportError: e = sys.exc_info()[1] @@ -701,6 +776,9 @@ def find(self, fullname): """ See implementation for a description of how this works. """ + if sys.version_info >= (3, 4): + return None + #if fullname not in sys.modules: # Don't attempt this unless a module really exists in sys.modules, # else we could return junk. @@ -729,6 +807,99 @@ def find(self, fullname): return self._found_module(fullname, path, fp) +class ParentSpecEnumerationMethod(ParentImpEnumerationMethod): + def _find_parent_spec(self, fullname): + #history = [] + debug = self.log.debug + children = [] + for parent_name, child_name in self._iter_parents(fullname): + children.insert(0, child_name) + if not parent_name: + debug('abandoning %r, reached top-level', fullname) + return None, children + + try: + parent = sys.modules[parent_name] + except KeyError: + debug('skipping %r, not in sys.modules', parent_name) + continue + + try: + spec = parent.__spec__ + except AttributeError: + debug('skipping %r: %r.__spec__ is absent', + parent_name, parent) + continue + + if not spec: + debug('skipping %r: %r.__spec__=%r', + parent_name, parent, spec) + continue + + if spec.name != parent_name: + debug('skipping %r: %r.__spec__.name=%r does not match', + parent_name, parent, spec.name) + continue + + if not spec.submodule_search_locations: + debug('skipping %r: %r.__spec__.submodule_search_locations=%r', + parent_name, parent, spec.submodule_search_locations) + continue + + return spec, children + + raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + def find(self, fullname): + # Returns absolute path, ParentImpEnumerationMethod returns relative + # >>> spec_pem.find('six_brokenpkg._six')[::2] + # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False) + + if sys.version_info < (3, 4): + return None + + fullname = to_text(fullname) + spec, children = self._find_parent_spec(fullname) + for child_name in children: + if spec: + name = '%s.%s' % (spec.name, child_name) + submodule_search_locations = spec.submodule_search_locations + else: + name = child_name + submodule_search_locations = None + spec = importlib.util._find_spec(name, submodule_search_locations) + if spec is None: + self.log.debug('%r spec unavailable from %s', fullname, spec) + return None + + is_package = spec.submodule_search_locations is not None + if name != fullname: + if not is_package: + self.log.debug('%r appears to be child of non-package %r', + fullname, spec) + return None + continue + + if not spec.has_location: + self.log.debug('%r.origin cannot be read as a file', spec) + return None + + if os.path.splitext(spec.origin)[1] != '.py': + self.log.debug('%r.origin does not contain Python source code', + spec) + return None + + # FIXME This should use loader.get_source() + with open(spec.origin, 'rb') as f: + source = f.read() + + return spec.origin, source, is_package + + raise ValueError('%s.find(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + class ModuleFinder(object): """ Given the name of a loaded module, make a best-effort attempt at finding @@ -769,7 +940,8 @@ def add_source_override(self, fullname, path, source, is_pkg): DefectivePython3xMainMethod(), PkgutilMethod(), SysModulesMethod(), - ParentEnumerationMethod(), + ParentSpecEnumerationMethod(), + ParentImpEnumerationMethod(), ] def get_module_source(self, fullname): @@ -1167,7 +1339,7 @@ class Broker(mitogen.core.Broker): def __init__(self, install_watcher=True): if install_watcher: self._watcher = ThreadWatcher.watch( - target=threading.currentThread(), + target=mitogen.core.threading__current_thread(), on_join=self.shutdown, ) super(Broker, self).__init__() diff --git a/mitogen/os_fork.py b/mitogen/os_fork.py index da832c65e..9c649d07f 100644 --- a/mitogen/os_fork.py +++ b/mitogen/os_fork.py @@ -35,7 +35,6 @@ import os import socket import sys -import threading import weakref import mitogen.core @@ -158,7 +157,7 @@ def cork(self): held. This will not return until each thread acknowledges it has ceased execution. """ - current = threading.currentThread() + current = mitogen.core.threading__current_thread() s = mitogen.core.b('CORK') * ((128 // 4) * 1024) self._rsocks = [] diff --git a/mitogen/parent.py b/mitogen/parent.py index 3b4dca8af..fa3092c19 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -34,7 +34,7 @@ connection. """ -import codecs +import binascii import errno import fcntl import getpass @@ -42,7 +42,6 @@ import inspect import logging import os -import platform import re import signal import socket @@ -57,15 +56,13 @@ # Absolute imports for <2.5. select = __import__('select') -try: - import thread -except ImportError: - import threading as thread - import mitogen.core from mitogen.core import b from mitogen.core import bytes_partition from mitogen.core import IOLOG +from mitogen.core import itervalues +from mitogen.core import next +from mitogen.core import thread LOG = logging.getLogger(__name__) @@ -81,15 +78,6 @@ SELINUX_ENABLED = False -try: - next -except NameError: - # Python 2.4/2.5 - from mitogen.core import next - - -itervalues = getattr(dict, 'itervalues', dict.values) - if mitogen.core.PY3: xrange = range closure_attr = '__closure__' @@ -148,6 +136,8 @@ def _ioctl_cast(n): LINUX_TIOCSPTLCK = _ioctl_cast(1074025521) IS_LINUX = os.uname()[0] == 'Linux' +IS_SOLARIS = os.uname()[0] == 'SunOS' + SIGNAL_BY_NUM = dict( (getattr(signal, name), name) @@ -412,7 +402,7 @@ def _acquire_controlling_tty(): # On Linux, the controlling tty becomes the first tty opened by a # process lacking any prior tty. os.close(os.open(os.ttyname(2), os.O_RDWR)) - if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL: + if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL and not IS_SOLARIS: # #550: prehistoric WSL does not like TIOCSCTTY. # On BSD an explicit ioctl is required. For some inexplicable reason, # Python 2.6 on Travis also requires it. @@ -480,7 +470,8 @@ def openpty(): master_fp = os.fdopen(master_fd, 'r+b', 0) slave_fp = os.fdopen(slave_fd, 'r+b', 0) - disable_echo(master_fd) + if not IS_SOLARIS: + disable_echo(master_fd) disable_echo(slave_fd) mitogen.core.set_block(slave_fd) return master_fp, slave_fp @@ -640,7 +631,7 @@ def __init__(self): def get_timeout(self): """ Return the floating point seconds until the next event is due. - + :returns: Floating point delay, or 0.0, or :data:`None` if no events are scheduled. @@ -746,8 +737,7 @@ def _upgrade_broker(broker): broker.timers = TimerList() LOG.debug('upgraded %r with %r (new: %d readers, %d writers; ' 'old: %d readers, %d writers)', old, new, - len(new.readers), len(new.writers), - len(old.readers), len(old.writers)) + len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds)) @mitogen.core.takes_econtext @@ -903,22 +893,18 @@ def __repr__(self): class PollPoller(mitogen.core.Poller): """ Poller based on the POSIX :linux:man2:`poll` interface. Not available on - some versions of OS X, otherwise it is the preferred poller for small FD - counts, as there is no setup/teardown/configuration system call overhead. + some Python/OS X combinations. Otherwise the preferred poller for small + FD counts; or if many pollers are created, used once, then closed. + There there is no setup/teardown/configuration system call overhead. """ SUPPORTED = hasattr(select, 'poll') - _repr = 'PollPoller()' + _readmask = SUPPORTED and select.POLLIN | select.POLLHUP def __init__(self): super(PollPoller, self).__init__() self._pollobj = select.poll() # TODO: no proof we dont need writemask too - _readmask = ( - getattr(select, 'POLLIN', 0) | - getattr(select, 'POLLHUP', 0) - ) - def _update(self, fd): mask = (((fd in self._rfds) and self._readmask) | ((fd in self._wfds) and select.POLLOUT)) @@ -953,7 +939,6 @@ class KqueuePoller(mitogen.core.Poller): Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface. """ SUPPORTED = hasattr(select, 'kqueue') - _repr = 'KqueuePoller()' def __init__(self): super(KqueuePoller, self).__init__() @@ -1028,10 +1013,10 @@ def _poll(self, timeout): class EpollPoller(mitogen.core.Poller): """ - Poller based on the Linux :linux:man2:`epoll` interface. + Poller based on the Linux :linux:man7:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') - _repr = 'EpollPoller()' + _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP def __init__(self): super(EpollPoller, self).__init__() @@ -1078,9 +1063,6 @@ def stop_transmit(self, fd): self._wfds.pop(fd, None) self._control(fd) - _inmask = (getattr(select, 'EPOLLIN', 0) | - getattr(select, 'EPOLLHUP', 0)) - def _poll(self, timeout): the_timeout = -1 if timeout is not None: @@ -1101,18 +1083,14 @@ def _poll(self, timeout): yield data -# 2.4 and 2.5 only had select.select() and select.poll(). -for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller: - if _klass.SUPPORTED: - PREFERRED_POLLER = _klass +POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller) +PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED) + # For processes that start many threads or connections, it's possible Latch # will also get high-numbered FDs, and so select() becomes useless there too. -# So swap in our favourite poller. -if PollPoller.SUPPORTED: - mitogen.core.Latch.poller_class = PollPoller -else: - mitogen.core.Latch.poller_class = PREFERRED_POLLER +POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER +mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT class LineLoggingProtocolMixin(object): @@ -1406,13 +1384,24 @@ def __repr__(self): # file descriptor 0 as 100, creates a pipe, then execs a new interpreter # with a custom argv. # * Optimized for minimum byte count after minification & compression. + # The script preamble_size.py measures this. # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with # their respective values. # * CONTEXT_NAME must be prefixed with the name of the Python binary in # order to allow virtualenvs to detect their install prefix. - # * For Darwin, OS X installs a craptacular argv0-introspecting Python - # version switcher as /usr/bin/python. Override attempts to call it - # with an explicit call to python2.7 + # + # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible + # module test_echo_module, used by the integration tests. + # * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version + # switcher as /usr/bin/python, which introspects argv0. To workaround + # it we redirect attempts to call /usr/bin/python with an explicit + # call to /usr/bin/python2.7. macOS 10.15 (Darwin 19) removed it. + # * macOS 11.x (Darwin 20, Big Sur) and macOS 12.x (Darwin 21, Montery) + # do something slightly different. The Python executable is patched to + # perform an extra execvp(). I don't fully understand the details, but + # setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it. + # * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python. + # https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python # # Locals: # R: read side of interpreter stdin. @@ -1435,15 +1424,12 @@ def _first_stage(): os.close(r) os.close(W) os.close(w) - # this doesn't apply anymore to Mac OSX 10.15+ (Darwin 19+), new interpreter looks like this: - # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python - if sys.platform == 'darwin' and sys.executable == '/usr/bin/python' and \ - int(platform.release()[:2]) < 19: - sys.executable += sys.version[:3] + if os.uname()[0]=='Darwin'and os.uname()[2][:2]<'19'and sys.executable=='/usr/bin/python':sys.executable='/usr/bin/python2.7' + if os.uname()[0]=='Darwin'and os.uname()[2][:2]in'2021'and sys.version[:3]=='2.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1' os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') os.write(1,'MITO000\n'.encode()) - C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') + C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN)) fp=os.fdopen(W,'wb',0) fp.write(C) fp.close() @@ -1469,22 +1455,22 @@ def get_python_argv(self): def get_boot_command(self): source = inspect.getsource(self._first_stage) source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:])) - source = source.replace(' ', '\t') + source = source.replace(' ', ' ') source = source.replace('CONTEXT_NAME', self.options.remote_name) preamble_compressed = self.get_preamble() source = source.replace('PREAMBLE_COMPRESSED_LEN', str(len(preamble_compressed))) compressed = zlib.compress(source.encode(), 9) - encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) - # We can't use bytes.decode() in 3.x since it was restricted to always - # return unicode, so codecs.decode() is used instead. In 3.x - # codecs.decode() requires a bytes object. Since we must be compatible - # with 2.4 (no bytes literal), an extra .encode() either returns the - # same str (2.x) or an equivalent bytes (3.x). + encoded = binascii.b2a_base64(compressed).replace(b('\n'), b('')) + + # Just enough to decode, decompress, and exec the first stage. + # Priorities: wider compatibility, faster startup, shorter length. + # `import os` here, instead of stage 1, to save a few bytes. + # `sys.path=...` for https://github.com/python/cpython/issues/115911. return self.get_python_argv() + [ '-c', - 'import codecs,os,sys;_=codecs.decode;' - 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),) + 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;' + 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),), ] def get_econtext_config(self): @@ -1506,7 +1492,7 @@ def get_econtext_config(self): def get_preamble(self): suffix = ( - '\nExternalContext(%r).main()\n' %\ + '\nExternalContext(%r).main()\n' % (self.get_econtext_config(),) ) partial = get_core_source_partial() @@ -2505,6 +2491,9 @@ def sudo(self, **kwargs): def ssh(self, **kwargs): return self.connect(u'ssh', **kwargs) + def podman(self, **kwargs): + return self.connect(u'podman', **kwargs) + class Reaper(object): """ @@ -2545,7 +2534,7 @@ def _signal_child(self, signum): # because it is setuid, so this is best-effort only. LOG.debug('%r: sending %s', self.proc, SIGNAL_BY_NUM[signum]) try: - os.kill(self.proc.pid, signum) + self.proc.send_signal(signum) except OSError: e = sys.exc_info()[1] if e.args[0] != errno.EPERM: @@ -2665,6 +2654,17 @@ def poll(self): """ raise NotImplementedError() + def send_signal(self, sig): + os.kill(self.pid, sig) + + def terminate(self): + "Ask the process to gracefully shutdown." + self.send_signal(signal.SIGTERM) + + def kill(self): + "Ask the operating system to forcefully destroy the process." + self.send_signal(signal.SIGKILL) + class PopenProcess(Process): """ @@ -2681,6 +2681,9 @@ def __init__(self, proc, stdin, stdout, stderr=None): def poll(self): return self.proc.poll() + def send_signal(self, sig): + self.proc.send_signal(sig) + class ModuleForwarder(object): """ diff --git a/mitogen/podman.py b/mitogen/podman.py new file mode 100644 index 000000000..acc46a339 --- /dev/null +++ b/mitogen/podman.py @@ -0,0 +1,73 @@ +# Copyright 2019, David Wilson +# Copyright 2021, Mitogen contributors +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Options(mitogen.parent.Options): + container = None + username = None + podman_path = 'podman' + + def __init__(self, container=None, podman_path=None, username=None, + **kwargs): + super(Options, self).__init__(**kwargs) + assert container is not None + self.container = container + if podman_path: + self.podman_path = podman_path + if username: + self.username = username + + +class Connection(mitogen.parent.Connection): + options_class = Options + child_is_immediate_subprocess = False + + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + + def _get_name(self): + return u'podman.' + self.options.container + + def get_boot_command(self): + args = [self.options.podman_path, 'exec'] + if self.options.username: + args += ['--user=' + self.options.username] + args += ["--interactive", "--", self.options.container] + return args + super(Connection, self).get_boot_command() diff --git a/mitogen/profiler.py b/mitogen/profiler.py index bbf6086ad..512a593ed 100644 --- a/mitogen/profiler.py +++ b/mitogen/profiler.py @@ -90,7 +90,7 @@ def merge_stats(outpath, inpaths): break time.sleep(0.2) - stats.dump_stats(outpath) + pstats.dump_stats(outpath) def generate_stats(outpath, tmpdir): diff --git a/mitogen/service.py b/mitogen/service.py index 249a8781f..ffc1085be 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -31,7 +31,6 @@ import grp import logging import os -import os.path import pprint import pwd import stat @@ -40,18 +39,10 @@ import mitogen.core import mitogen.select +from mitogen.core import all from mitogen.core import b from mitogen.core import str_rpartition -try: - all -except NameError: - def all(it): - for elem in it: - if not elem: - return False - return True - LOG = logging.getLogger(__name__) @@ -109,7 +100,8 @@ def get_or_create_pool(size=None, router=None, context=None): def get_thread_name(): - return threading.currentThread().getName() + thread = mitogen.core.threading__current_thread() + return mitogen.core.threading__thread_name(thread) def call(service_name, method_name, call_context=None, **kwargs): @@ -752,10 +744,12 @@ def propagate_paths_and_modules(self, context, paths, overridden_sources=None, e One size fits all method to ensure a target context has been preloaded with a set of small files and Python modules. - overridden_sources: optional dict containing source code to override path's source code - extra_sys_paths: loads additional sys paths for use in finding modules; beneficial - in situations like loading Ansible Collections because source code - dependencies come from different file paths than where the source lives + :param dict overridden_sources: + Optional dict containing source code to override path's source code + :param extra_sys_paths: + Loads additional sys paths for use in finding modules; beneficial + in situations like loading Ansible Collections because source code + dependencies come from different file paths than where the source lives """ for path in paths: overridden_source = None diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 656dc72ca..f32d2cabb 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -43,11 +43,6 @@ import mitogen.parent from mitogen.core import b -try: - any -except NameError: - from mitogen.core import any - LOG = logging.getLogger(__name__) diff --git a/mitogen/su.py b/mitogen/su.py index 080c97829..9b908460c 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -34,11 +34,6 @@ import mitogen.core import mitogen.parent -try: - any -except NameError: - from mitogen.core import any - LOG = logging.getLogger(__name__) diff --git a/mitogen/unix.py b/mitogen/unix.py index 1af1c0ec6..b241a4037 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -143,19 +143,23 @@ def on_shutdown(self, broker): def on_accept_client(self, sock): sock.setblocking(True) try: - pid, = struct.unpack('>L', sock.recv(4)) + data = sock.recv(4) + pid, = struct.unpack('>L', data) except (struct.error, socket.error): - LOG.error('listener: failed to read remote identity: %s', - sys.exc_info()[1]) + LOG.error('listener: failed to read remote identity, got %d bytes: %s', + len(data), sys.exc_info()[1]) + sock.close() return context_id = self._router.id_allocator.allocate() try: + # FIXME #1109 send() returns number of bytes sent, check it sock.send(struct.pack('>LLL', context_id, mitogen.context_id, os.getpid())) except socket.error: LOG.error('listener: failed to assign identity to PID %d: %s', pid, sys.exc_info()[1]) + sock.close() return context = mitogen.parent.Context(self._router, context_id) diff --git a/mitogen/utils.py b/mitogen/utils.py index b1347d022..9d1c1bc9d 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -29,22 +29,15 @@ # !mitogen: minify_safe import datetime +import functools import logging import os import sys -import mitogen import mitogen.core import mitogen.master -import mitogen.parent - -iteritems = getattr(dict, 'iteritems', dict.items) - -if mitogen.core.PY3: - iteritems = dict.items -else: - iteritems = dict.iteritems +from mitogen.core import iteritems def setup_gil(): @@ -173,12 +166,9 @@ def do_stuff(router, arg): do_stuff(blah, 123) """ + @functools.wraps(func) def wrapper(*args, **kwargs): return run_with_router(func, *args, **kwargs) - if mitogen.core.PY3: - wrapper.func_name = func.__name__ - else: - wrapper.func_name = func.func_name return wrapper @@ -194,10 +184,13 @@ def wrapper(*args, **kwargs): def cast(obj): """ + Return obj (or a copy) with subtypes of builtins cast to their supertype. + Subtypes of those in :data:`PASSTHROUGH` are not modified. + Many tools love to subclass built-in types in order to implement useful functionality, such as annotating the safety of a Unicode string, or adding - additional methods to a dict. However, cPickle loves to preserve those - subtypes during serialization, resulting in CallError during :meth:`call + additional methods to a dict. However :py:mod:`pickle` serializes these + exactly, leading to :exc:`mitogen.CallError` during :meth:`Context.call ` in the target when it tries to deserialize the data. @@ -205,6 +198,9 @@ def cast(obj): custom sub-types removed. The functionality is not default since the resulting walk may be computationally expensive given a large enough graph. + Raises :py:exc:`TypeError` if an unknown subtype is encountered, or + casting does not return the desired supertype. + See :ref:`serialization-rules` for a list of supported types. :param obj: @@ -219,8 +215,16 @@ def cast(obj): if isinstance(obj, PASSTHROUGH): return obj if isinstance(obj, mitogen.core.UnicodeType): - return mitogen.core.UnicodeType(obj) + return _cast(obj, mitogen.core.UnicodeType) if isinstance(obj, mitogen.core.BytesType): - return mitogen.core.BytesType(obj) + return _cast(obj, mitogen.core.BytesType) raise TypeError("Cannot serialize: %r: %r" % (type(obj), obj)) + + +def _cast(obj, desired_type): + result = desired_type(obj) + if type(result) is not desired_type: + raise TypeError("Cast of %r to %r failed, got %r" + % (type(obj), desired_type, type(result))) + return result diff --git a/preamble_size.py b/preamble_size.py old mode 100644 new mode 100755 index f0d1e8041..efb46eaec --- a/preamble_size.py +++ b/preamble_size.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ Print the size of a typical SSH command line and the bootstrap code sent to new contexts. diff --git a/run_tests b/run_tests index b583af3b1..75663d15c 100755 --- a/run_tests +++ b/run_tests @@ -28,10 +28,6 @@ NOCOVERAGE="${NOCOVERAGE:-}" NOCOVERAGE_ERASE="${NOCOVERAGE_ERASE:-$NOCOVERAGE}" NOCOVERAGE_REPORT="${NOCOVERAGE_REPORT:-$NOCOVERAGE}" -if [ ! "$UNIT2" ]; then - UNIT2="$(which unit2)" -fi - if [ ! "$NOCOVERAGE_ERASE" ]; then coverage erase fi @@ -39,12 +35,12 @@ fi # First run overwites coverage output. [ "$SKIP_MITOGEN" ] || { if [ ! "$NOCOVERAGE" ]; then - coverage run -a "${UNIT2}" discover \ + coverage run -a -m unittest discover \ --start-directory "tests" \ --pattern '*_test.py' \ "$@" else - "${UNIT2}" discover \ + python -m unittest discover \ --start-directory "tests" \ --pattern '*_test.py' \ "$@" @@ -60,12 +56,12 @@ fi [ "$SKIP_ANSIBLE" ] || { export PYTHONPATH=`pwd`/tests:$PYTHONPATH if [ ! "$NOCOVERAGE" ]; then - coverage run -a "${UNIT2}" discover \ + coverage run -a -m unittest discover \ --start-directory "tests/ansible" \ --pattern '*_test.py' \ "$@" else - "${UNIT2}" discover \ + python -m unittest discover \ --start-directory "tests/ansible" \ --pattern '*_test.py' \ "$@" diff --git a/scripts/pogrep.py b/scripts/pogrep.py index b837bcfd6..80094c74f 100644 --- a/scripts/pogrep.py +++ b/scripts/pogrep.py @@ -6,6 +6,7 @@ # - apt-get source libpam0g # - cd */po/ # - python ~/pogrep.py "Password: " +from __future__ import print_function import sys import shlex @@ -31,7 +32,7 @@ if last_word == 'msgid' and word == 'msgstr': if last_rest == sys.argv[1]: thing = rest.rstrip(': ').decode('utf-8').lower().encode('utf-8').encode('base64').rstrip() - print ' %-60s # %s' % (repr(thing)+',', path) + print(' %-60s # %s' % (repr(thing)+',', path)) last_word = word last_rest = rest diff --git a/setup.py b/setup.py index bd1051478..9cb304272 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import ast import os from setuptools import find_packages, setup @@ -37,7 +38,8 @@ def grep_version(): for line in fp: if line.startswith('__version__'): _, _, s = line.partition('=') - return '%i.%i.%i%s%i' % eval(s) + parts = ast.literal_eval(s.strip()) + return '.'.join(str(part) for part in parts) def long_description(): @@ -58,25 +60,26 @@ def long_description(): license = 'New BSD', url = 'https://github.com/mitogen-hq/mitogen/', packages = find_packages(exclude=['tests', 'examples']), - python_requires='>=2.4, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', zip_safe = False, classifiers = [ 'Environment :: Console', + 'Framework :: Ansible', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.4', - 'Programming Language :: Python :: 2.5', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/README.md b/tests/README.md index 514649892..06bf7ad7c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,7 +7,7 @@ started in September 2017. Pull requests in this area are very welcome! ## Running The Tests -[![Build Status](https://api.travis-ci.org/dw/mitogen.svg?branch=master)](https://travis-ci.org/dw/mitogen) +[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster) Your computer should have an Internet connection, and the ``docker`` command line tool should be able to connect to a working Docker daemon (localhost or @@ -30,11 +30,19 @@ and run the tests there. 1. Run ``test`` -# Selecting a target distribution +# Selecting target distributions -Docker target images exist for testing against CentOS and Debian, with the -default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in -the environment. +Linux container images for testing are available at + +- https://github.com/orgs/mitogen-hq/packages +- https://public.ecr.aws/n5z0e8q9 + +The images used are determined by two environment variables + +- `MITOGEN_TEST_DISTRO_SPECS` +- `MITOGEN_TEST_IMAGE_TEMPLATE` + +Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py` # User Accounts diff --git a/tests/ansible/README.md b/tests/ansible/README.md index 50e747fe3..51bb1e60b 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -10,7 +10,7 @@ demonstrator for what does and doesn't work. ## Preparation -See `../image_prep/README.md`. +See [`../image_prep/README.md`](../image_prep/README.md). ## `run_ansible_playbook.py` diff --git a/tests/ansible/all.yml b/tests/ansible/all.yml index 06f3acdb5..7a3e70001 100644 --- a/tests/ansible/all.yml +++ b/tests/ansible/all.yml @@ -1,4 +1,6 @@ -- include: setup/all.yml -- include: regression/all.yml -- include: integration/all.yml - +- import_playbook: setup/all.yml + tags: setup +- import_playbook: regression/all.yml + tags: regression +- import_playbook: integration/all.yml + tags: integration diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 59752492d..37edfd94a 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -1,16 +1,28 @@ [defaults] +any_errors_fatal = true +# callbacks_enabled was added in Ansible 4 (ansible-core 2.11). +# profile_tasks: Displays timing for each task and summary table of top N tasks +# timer: Displays "Playbook run took 0 days, 0 hours, ..." +callbacks_enabled = + profile_tasks, + timer +# callback_whitelist was deprecated in Ansible >= 8 (ansible-core >= 2.15). +callback_whitelist = + profile_tasks, + timer inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy inventory_plugins = lib/inventory action_plugins = lib/action callback_plugins = lib/callback -stdout_callback = nice_stdout +stdout_callback = yaml vars_plugins = lib/vars library = lib/modules filter_plugins = lib/filters module_utils = lib/module_utils retry_files_enabled = False +show_task_path_on_failure = true # Added in ansible-core 2.11 display_args_to_stdout = True forks = 100 @@ -26,11 +38,37 @@ transport = ssh no_target_syslog = True # Required by integration/ssh/timeouts.yml -timeout = 10 +timeout = 30 -# On Travis, paramiko check fails due to host key checking enabled. +# Ideally this would be true here and and overridden for hosts/groups. However +# ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for +# static inventory hosts (ansible/ansible#49254, ansible/ansible#73708) host_key_checking = False +[inventory] +any_unparsed_is_failed = true +host_pattern_mismatch = error +ignore_extensions = ~, .bak, .disabled + +[callback_profile_tasks] +task_output_limit = 10 + [ssh_connection] -ssh_args = -o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s +# https://www.openssh.com/legacy.html +# ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. +# Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). +# Duplicated cases in +# - tests/ansible/ansible.cfg +# - tests/ansible/integration/connection_delegation/delegate_to_template.yml +# - tests/ansible/integration/connection_delegation/stack_construction.yml +# - tests/ansible/integration/process/unix_socket_cleanup.yml +# - tests/ansible/integration/ssh/variables.yml +# - tests/testlib.py +ssh_args = + -o ControlMaster=auto + -o ControlPersist=60s + -o ForwardAgent=yes + -o HostKeyAlgorithms=+ssh-rsa + -o PubkeyAcceptedKeyTypes=+ssh-rsa + -o UserKnownHostsFile=/dev/null pipelining = True diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml index f6702f587..1c8dce79b 100644 --- a/tests/ansible/bench/file_transfer.yml +++ b/tests/ansible/bench/file_transfer.yml @@ -1,15 +1,16 @@ - - name: bench/file_transfer.yml hosts: test-targets - any_errors_fatal: true tasks: - - name: Make 32MiB file delegate_to: localhost + run_once: true shell: openssl rand 33554432 > /tmp/bigfile.in + args: + creates: /tmp/bigfile.in - name: Make 320MiB file delegate_to: localhost + run_once: true shell: > cat /tmp/bigfile.in @@ -23,6 +24,8 @@ /tmp/bigfile.in /tmp/bigfile.in > /tmp/bigbigfile.in + args: + creates: /tmp/bigbigfile.in - name: Delete SSH file is present. file: @@ -37,35 +40,74 @@ copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out + mode: ugo=rw - name: Copy 320MiB file via SSH copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + mode: ugo=rw - name: Delete localhost sudo file if present. file: path: "{{item}}" state: absent delegate_to: localhost + run_once: true become: true with_items: - /tmp/bigfile.out - /tmp/bigbigfile.out + tags: + - requires_local_sudo - name: Copy 32MiB file via localhost sudo delegate_to: localhost + run_once: true become: true copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out + mode: ugo=rw + tags: + - requires_local_sudo - name: Copy 320MiB file via localhost sudo delegate_to: localhost + run_once: true become: true copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + mode: ugo=rw + tags: + - requires_local_sudo + + - name: Local cleanup + file: + path: "{{ item.path }}" + state: absent + loop: + - /tmp/bigfile.in + - /tmp/bigfile.out + - /tmp/bigbigfile.in + - /tmp/bigbigfile.out + delegate_to: localhost + run_once: true + tags: + - cleanup_local + - cleanup + + - name: Target cleanup + file: + path: "{{ item.path }}" + state: absent + loop: + - /tmp/bigfile.out + - /tmp/bigbigfile.out + tags: + - cleanup_target + - cleanup tags: - resource_intensive diff --git a/tests/ansible/bench/includes.yml b/tests/ansible/bench/includes.yml index 960798747..6241a4853 100644 --- a/tests/ansible/bench/includes.yml +++ b/tests/ansible/bench/includes.yml @@ -1,4 +1,5 @@ -- hosts: test-targets +- name: bench/includes.yml + hosts: test-targets tasks: - include_tasks: _includes.yml with_sequence: start=1 end=1000 diff --git a/tests/ansible/bench/loop-100-copies.yml b/tests/ansible/bench/loop-100-copies.yml index e25ae552d..2a8e7d6f0 100644 --- a/tests/ansible/bench/loop-100-copies.yml +++ b/tests/ansible/bench/loop-100-copies.yml @@ -1,6 +1,5 @@ - -- hosts: all - any_errors_fatal: true +- name: bench/loop-100-copies.yml + hosts: all tasks: - name: Create file tree diff --git a/tests/ansible/bench/loop-100-items.yml b/tests/ansible/bench/loop-100-items.yml index e711301dc..ad2986387 100644 --- a/tests/ansible/bench/loop-100-items.yml +++ b/tests/ansible/bench/loop-100-items.yml @@ -4,7 +4,8 @@ # # See also: loop-100-tasks.yml # -- hosts: all +- name: bench/loop-100-items.yml + hosts: all tasks: - command: hostname with_sequence: start=1 end="{{end|default(100)}}" diff --git a/tests/ansible/bench/loop-100-tasks.yml b/tests/ansible/bench/loop-100-tasks.yml index 4a76c4fec..211cecd1d 100644 --- a/tests/ansible/bench/loop-100-tasks.yml +++ b/tests/ansible/bench/loop-100-tasks.yml @@ -8,7 +8,8 @@ # # See also: loop-100-items.yml # -- hosts: all +- name: bench/loop-100-tasks.yml + hosts: all tasks: - command: hostname - command: hostname diff --git a/tests/ansible/bench/loop-20-templates.yml b/tests/ansible/bench/loop-20-templates.yml index 17dc77779..4035ea34a 100644 --- a/tests/ansible/bench/loop-20-templates.yml +++ b/tests/ansible/bench/loop-20-templates.yml @@ -1,12 +1,14 @@ - -- hosts: all +- name: bench/loop-20-templates.yml + hosts: all tasks: - - file: + - name: Create loop templates dir + file: dest: /tmp/templates state: "{{item}}" with_items: ["absent", "directory"] - - copy: + - name: Copy loop files + copy: dest: /tmp/templates/{{item}} mode: 0755 content: diff --git a/tests/ansible/files/cwd_show b/tests/ansible/files/cwd_show new file mode 100755 index 000000000..42ef3194c --- /dev/null +++ b/tests/ansible/files/cwd_show @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Show permissions and identities that impact the current working directory. +# On macOS libc cwd() can return EACCES after su or sudo. +# See also +# - https://github.com/ansible/ansible/pull/7078 +# - https://github.com/python/cpython/issues/115911 + +set -o errexit +set -o nounset +set -o pipefail + +whoami +groups +pwd + +d=$(pwd) +while [[ "$d" != "/" && -n "$d" ]]; do + ls -ld "$d" + d=$(dirname "$d") +done +ls -ld / diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts index a18b90d21..249246cb4 100644 --- a/tests/ansible/hosts/become_same_user.hosts +++ b/tests/ansible/hosts/become_same_user.hosts @@ -1,4 +1,9 @@ +# code: language=ini +# vim: syntax=dosini +[become_same_user] # become_same_user.yml bsu-joe ansible_user=joe +[become_same_user:vars] +ansible_python_interpreter=python3000 diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts index a22bd5df9..7de798a17 100644 --- a/tests/ansible/hosts/connection_delegation.hosts +++ b/tests/ansible/hosts/connection_delegation.hosts @@ -1,9 +1,10 @@ +# code: language=ini # vim: syntax=dosini # Connection delegation scenarios. It's impossible to connect to them, but their would-be # config can be inspected using "mitogen_get_stack" action. - +[cd] # Normal inventory host, no aliasing. cd-normal ansible_connection=mitogen_doas ansible_user=normal-user @@ -22,6 +23,8 @@ cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-norma # doas:newuser via host. cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user +[cd:vars] +ansible_python_interpreter = python3000 [connection-delegation-test] cd-bastion diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index d40c3dd0a..3d20e73df 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -1,9 +1,55 @@ +# code: language=ini # vim: syntax=dosini # When running the tests outside CI, make a single 'target' host which is the # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. -target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" +# os.environ['USER'] is an empty string on GitHub Actions macOS runners. +target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] target + +[linux_containers] + +[issue905] +ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" + +[issue905:vars] +ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}" +ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ inventory_hostname }} + +[issue1079] +wait-for-connection ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" + +[tt_targets_bare] +tt-bare + +[tt_become_bare] +tt-become-bare + +[tt_become_bare:vars] +ansible_host=localhost +ansible_user="{{ lookup('pipe', 'whoami') }}" + +[tt_become_by_inv] +tt-become ansible_become="{{ 'true' | trim }}" ansible_become_user=root +tt-become-exe ansible_become=true ansible_become_exe="{{ 'sudo' | trim }}" ansible_become_user=root +tt-become-flags ansible_become=true ansible_become_flags="{{ '--set-home --stdin --non-interactive' | trim }}" ansible_become_user=root +tt-become-method ansible_become=true ansible_become_method="{{ 'sudo' | trim }}" ansible_become_user=root +tt-become-pass ansible_become=true ansible_become_pass="{{ 'pw_required_password' | trim }}" ansible_become_user=mitogen__pw_required +tt-become-user ansible_become=true ansible_become_user="{{ 'root' | trim }}" + +[tt_become_by_inv:vars] +ansible_host=localhost +ansible_user="{{ lookup('pipe', 'whoami') }}" + +[tt_targets_inventory] +tt-host ansible_host="{{ 'localhost' | trim }}" ansible_password=has_sudo_nopw_password ansible_user=mitogen__has_sudo_nopw +tt-host-key-checking ansible_host=localhost ansible_host_key_checking="{{ 'false' | trim }}" ansible_password=has_sudo_nopw_password ansible_user=mitogen__has_sudo_nopw +tt-password ansible_host=localhost ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw +tt-port ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw +tt-private-key-file ansible_host=localhost ansible_private_key_file="{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_user=mitogen__has_sudo_pubkey +tt-remote-user ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_user="{{ 'mitogen__has_sudo_nopw' | trim }}" +tt-ssh-executable ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_ssh_executable="{{ 'ssh' | trim }}" ansible_user=mitogen__has_sudo_nopw +tt-timeout ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_timeout="{{ 5 | int }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml new file mode 100644 index 000000000..44e660fc1 --- /dev/null +++ b/tests/ansible/hosts/group_vars/all.yml @@ -0,0 +1,30 @@ +--- +become_unpriv_available: >- + {# + Vanilla Ansible >= 4 (ansible-core >= 2.11) can use `setfacl` for + unpriv -> unpriv, but Mitogen test containers lack setfacl + https://github.com/mitogen-hq/mitogen/issues/1118 + + Mitogen + Ansible can do unpriv -> unpriv without temporary files, + but Ansible >= 11 (ansible-core >= 2.18) detection tries to use Python + 3.13 which hits https://github.com/python/cpython/issues/115911 on macOS. + #} + {{- + ( + not is_mitogen + and ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True) + ) + or ( + is_mitogen + and not ansible_facts.distribution in ["MacOSX"] + ) + or ( + is_mitogen + and ansible_python_interpreter is not defined + and ansible_version.full is version("2.18", "<", strict=True) + ) + -}} + +pkg_mgr_python_interpreter: python +pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml new file mode 100644 index 000000000..c90dd5f46 --- /dev/null +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -0,0 +1,28 @@ +--- +pkg_mgr_python_interpreter: /usr/libexec/platform-python + +pkg_repos_overrides: + - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo + content: | + [appstream] + name=CentOS Linux $releasever - AppStream + baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo + content: | + [baseos] + name=CentOS Linux $releasever - BaseOS + baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo + content: | + [extras] + name=CentOS Linux $releasever - Extras + baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml new file mode 100644 index 000000000..4b180b138 --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -0,0 +1,4 @@ +pkg_repos_overrides: + - dest: /etc/apt/sources.list + content: | + deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/hosts/k3.hosts b/tests/ansible/hosts/k3.hosts.disabled similarity index 91% rename from tests/ansible/hosts/k3.hosts rename to tests/ansible/hosts/k3.hosts.disabled index 34e1ff958..b210164bd 100644 --- a/tests/ansible/hosts/k3.hosts +++ b/tests/ansible/hosts/k3.hosts.disabled @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # Used for manual testing. diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts.disabled similarity index 96% rename from tests/ansible/hosts/localhost.hosts rename to tests/ansible/hosts/localhost.hosts.disabled index 41af412e5..e42221e7b 100644 --- a/tests/ansible/hosts/localhost.hosts +++ b/tests/ansible/hosts/localhost.hosts.disabled @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # issue #511, #536: we must not define an explicit localhost, as some diff --git a/tests/ansible/hosts/osa-containers b/tests/ansible/hosts/osa-containers.hosts.disabled similarity index 100% rename from tests/ansible/hosts/osa-containers rename to tests/ansible/hosts/osa-containers.hosts.disabled diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 856c6a0f0..5d5c38346 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -1,49 +1,86 @@ +# code: language=ini +# vim: syntax=dosini + # integration/transport_config # Hosts with twiddled configs that need to be checked somehow. +[transport_config:children] +transport_config_undiscover +tc_python_path + +[transport_config_undiscover:children] +tc_become +tc_become_method +tc_become_pass +tc_become_user +tc_host_key_checking +tc_password +tc_port +tc_remote_addr +tc_remote_user +tc_transport + +[transport_config_undiscover:vars] +# If ansible_*_interpreter isn't set Ansible tries to connect & discover it. +# If that target doesn't exist we must wait $timeout seconds for each attempt. +# Setting a known (invalid) interpreter skips discovery & the many timeouts. +# This optimisation should not be relied in any test. +# Note: tc-python-path-* are intentionally not included. +ansible_python_interpreter = python3000 -# tansport() +[tc_transport] tc-transport-unset tc-transport-local ansible_connection=local tc-transport-smart ansible_connection=smart -# python_path() +[tc_python_path] tc-python-path-unset tc-python-path-hostvar ansible_python_interpreter=/hostvar/path/to/python tc-python-path-local-unset ansible_connection=local tc-python-path-local-explicit ansible_connection=local ansible_python_interpreter=/a/b/c -# remote_addr() +[tc_remote_addr] tc-remote-addr-unset # defaults to inventory_hostname tc-remote-addr-explicit-ssh ansible_ssh_host=ansi.ssh.host tc-remote-addr-explicit-host ansible_host=ansi.host tc-remote-addr-explicit-both ansible_ssh_host=a.b.c ansible_host=b.c.d -# password() +[tc_password] tc-password-unset tc-password-explicit-ssh ansible_ssh_pass=ansi-ssh-pass -tc-password-explicit-user ansible_password=ansi-pass +tc-password-explicit-pass ansible_password=ansi-pass tc-password-explicit-both ansible_password=a.b.c ansible_ssh_pass=c.b.a -# become() +[tc_remote_user] +tc-remote-user-unset # defaults to C.DEFAULT_REMOTE_USER +tc-remote-user-explicit-ssh ansible_ssh_user=ansi-ssh-user +tc-remote-user-explicit-user ansible_user=ansi-user +tc-remote-user-explicit-both ansible_user=a.b.c ansible_ssh_user=c.b.a + +[tc_become] tc-become-unset tc-become-set -# become_method() +[tc_become_method] tc-become-method-unset tc-become-method-su ansible_become_method=su -# become_user() +[tc_become_user] tc-become-user-unset tc-become-user-set ansible_become_user=ansi-become-user -# become_pass() +[tc_become_pass] tc-become-pass-unset tc-become-pass-password ansible_become_password=apassword tc-become-pass-pass ansible_become_pass=apass -tc-become-pass-both ansible_become_password=a.b.c ansible_become_pass=c.b.a +tc-become-pass-both ansible_become_pass=bpass ansible_become_password=bpassword + +[tc_host_key_checking] +tc-hkc-unset +tc-hkc-host-key-checking ansible_host_key_checking=true +tc-hkc-ssh-host-key-checking ansible_ssh_host_key_checking=true -# port() +[tc_port] tc-port-unset tc-port-explicit-port ansible_port=1234 tc-port-explicit-ssh ansible_ssh_port=4321 diff --git a/tests/ansible/integration/_expected_ssh_port.yml b/tests/ansible/integration/_expected_ssh_port.yml new file mode 100644 index 000000000..442659a52 --- /dev/null +++ b/tests/ansible/integration/_expected_ssh_port.yml @@ -0,0 +1,18 @@ +# Ansible removed its default SSH port in May 2021, defering to the SSH +# implementation. +# https://github.com/ansible/ansible/commit/45618a6f3856f7332df8afe4adc40d85649a70da + +# Careful templating is needed to preseve the type(s) of expected_ssh_port, +# particularly in combination with the assert_equal action plugin. +# Do: {{ expected_ssh_port }} +# Don't: {{ expected_ssh_port | any_filter }} +# Don't: {% if ...%}{{ expected_ssh_port }}{% else %}...{% endif %} +# https://stackoverflow.com/questions/66102524/ansible-set-fact-type-cast/66104814#66104814 + +- set_fact: + expected_ssh_port: null + when: ansible_version.full is version('2.11.1', '>=', strict=True) + +- set_fact: + expected_ssh_port: 22 + when: ansible_version.full is version('2.11.1', '<', strict=True) diff --git a/tests/ansible/integration/action/all.yml b/tests/ansible/integration/action/all.yml index c43d5cc7d..d823e05e1 100644 --- a/tests/ansible/integration/action/all.yml +++ b/tests/ansible/integration/action/all.yml @@ -1,10 +1,10 @@ -- include: copy.yml -- include: fixup_perms2__copy.yml -- include: low_level_execute_command.yml -- include: make_tmp_path.yml -- include: make_tmp_path__double.yml -- include: remote_expand_user.yml -- include: remote_file_exists.yml -- include: remove_tmp_path.yml -- include: synchronize.yml -- include: transfer_data.yml +- import_playbook: copy.yml +- import_playbook: fixup_perms2__copy.yml +- import_playbook: low_level_execute_command.yml +- import_playbook: make_tmp_path.yml +- import_playbook: make_tmp_path__double.yml +- import_playbook: remote_expand_user.yml +- import_playbook: remote_file_exists.yml +- import_playbook: remove_tmp_path.yml +- import_playbook: synchronize.yml +- import_playbook: transfer_data.yml diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index b34b9831a..f1ad5dc3a 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -1,83 +1,95 @@ # Verify copy module for small and large files, and inline content. +# To exercise https://github.com/mitogen-hq/mitogen/pull/1110 destination +# files must have extensions and loops must use `with_items:`. - name: integration/action/copy.yml hosts: test-targets - any_errors_fatal: true - tasks: - - copy: - dest: /tmp/copy-tiny-file - content: - this is a tiny file. - delegate_to: localhost + vars: + sourced_files: + - src: /tmp/copy-tiny-file + dest: /tmp/copy-tiny-file.out + content: this is a tiny file. + expected_checksum: f29faa9a6f19a700a941bf2aa5b281643c4ec8a0 + - src: /tmp/copy-large-file + dest: /tmp/copy-large-file.out + content: "{{ 'x' * 200000 }}" + expected_checksum: 62951f943c41cdd326e5ce2b53a779e7916a820d - - copy: - dest: /tmp/copy-large-file - # Must be larger than Connection.SMALL_SIZE_LIMIT. - content: "{% for x in range(200000) %}x{% endfor %}" - delegate_to: localhost + inline_files: + - dest: /tmp/copy-tiny-inline-file.out + content: tiny inline content + expected_checksum: b26dd6444595e2bdb342aa0a91721b57478b5029 + - dest: /tmp/copy-large-inline-file.out + content: | + {{ 'y' * 200000 }} + expected_checksum: d675f47e467eae19e49032a2cc39118e12a6ee72 - # end of making files + files: "{{ sourced_files + inline_files }}" + tasks: + - name: Create sourced files + copy: + dest: "{{ item.src }}" + content: "{{ item.content }}" + mode: u=rw,go=r + with_items: "{{ sourced_files }}" + loop_control: + label: "{{ item.src }}" + delegate_to: localhost + run_once: true - - file: + - name: Cleanup lingering destination files + file: + path: "{{ item.dest }}" state: absent - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file.out - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file.out + with_items: "{{ files }}" + loop_control: + label: "{{ item.dest }}" - # end of cleaning out files + - name: Copy sourced files + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: u=rw,go=r + with_items: "{{ sourced_files }}" + loop_control: + label: "{{ item.dest }}" - - copy: - dest: /tmp/copy-large-file.out - src: /tmp/copy-large-file - - - copy: - dest: /tmp/copy-tiny-file.out - src: /tmp/copy-tiny-file - - - copy: - dest: /tmp/copy-tiny-inline-file.out - content: "tiny inline content" - - - copy: - dest: /tmp/copy-large-inline-file.out - content: | - {% for x in range(200000) %}y{% endfor %} + - name: Copy inline files + copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: u=rw,go=r + with_items: "{{ inline_files }}" + loop_control: + label: "{{ item.dest }}" # stat results - - stat: - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file.out - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file.out + - name: Stat copied files + stat: + path: "{{ item.dest }}" + with_items: "{{ files }}" + loop_control: + label: "{{ item.dest }}" register: stat - assert: that: - - stat.results[0].stat.checksum == "f29faa9a6f19a700a941bf2aa5b281643c4ec8a0" - - stat.results[1].stat.checksum == "62951f943c41cdd326e5ce2b53a779e7916a820d" - - stat.results[2].stat.checksum == "b26dd6444595e2bdb342aa0a91721b57478b5029" - - stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72" + - item.stat.checksum == item.item.expected_checksum + quiet: true # Avoid spamming stdout with 400 kB of item.item.content + fail_msg: | + item={{ item }} + with_items: "{{ stat.results }}" + loop_control: + label: "{{ item.stat.path }}" - - file: + - name: Cleanup destination files + file: + path: "{{ item.dest }}" state: absent - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file - - /tmp/copy-tiny-file.out - - /tmp/copy-no-mode - - /tmp/copy-no-mode.out - - /tmp/copy-with-mode - - /tmp/copy-with-mode.out - - /tmp/copy-large-file - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file - - /tmp/copy-large-inline-file.out - - # end of cleaning out files (again) + with_items: "{{ files }}" + loop_control: + label: "{{ item.dest }}" + tags: + - copy + - issue_1110 diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml index 1331f9bb6..7821111e8 100644 --- a/tests/ansible/integration/action/fixup_perms2__copy.yml +++ b/tests/ansible/integration/action/fixup_perms2__copy.yml @@ -5,12 +5,7 @@ - name: integration/action/fixup_perms2__copy.yml hosts: test-targets - any_errors_fatal: true tasks: - # - # copy module (no mode). - # - - name: "Copy files (no mode)" copy: content: "" @@ -21,10 +16,8 @@ - assert: that: - out.stat.mode in ("0644", "0664") - - # - # copy module (explicit mode). - # + fail_msg: | + out={{ out }} - name: "Copy files from content: arg" copy: @@ -37,23 +30,24 @@ - assert: that: - out.stat.mode == "0400" + fail_msg: | + out={{ out }} - # - # copy module (existing disk files, no mode). - # - - - file: + - name: Cleanup local weird mode file + file: path: /tmp/weird-mode.out state: absent - - name: Create local test file. + - name: Create local weird mode file delegate_to: localhost + run_once: true copy: content: "weird mode" dest: "/tmp/weird-mode" mode: "1462" - - copy: + - name: Copy file with weird mode + copy: src: "/tmp/weird-mode" dest: "/tmp/weird-mode.out" @@ -63,12 +57,11 @@ - assert: that: - out.stat.mode in ("0644", "0664") + fail_msg: | + out={{ out }} - # - # copy module (existing disk files, preserve mode). - # - - - copy: + - name: Copy file with weird mode, preserving mode + copy: src: "/tmp/weird-mode" dest: "/tmp/weird-mode" mode: preserve @@ -79,12 +72,11 @@ - assert: that: - out.stat.mode == "1462" + fail_msg: | + out={{ out }} - # - # copy module (existing disk files, explicit mode). - # - - - copy: + - name: Copy file with weird mode, explicit mode + copy: src: "/tmp/weird-mode" dest: "/tmp/weird-mode" mode: "1461" @@ -96,8 +88,11 @@ - assert: that: - out.stat.mode == "1461" + fail_msg: | + out={{ out }} - - file: + - name: Cleanup + file: state: absent path: "{{item}}" with_items: @@ -109,3 +104,5 @@ - /tmp/copy-with-mode.out # end of cleaning out files + tags: + - fixup_perms2__copy diff --git a/tests/ansible/integration/action/low_level_execute_command.yml b/tests/ansible/integration/action/low_level_execute_command.yml index 7c14cb225..f42e05614 100644 --- a/tests/ansible/integration/action/low_level_execute_command.yml +++ b/tests/ansible/integration/action/low_level_execute_command.yml @@ -2,7 +2,6 @@ - name: integration/action/low_level_execute_command.yml hosts: test-targets - any_errors_fatal: true tasks: # "echo -en" to test we actually hit bash shell too. @@ -16,6 +15,8 @@ - 'raw.rc == 0' - 'raw.stdout_lines[-1]|to_text == "2"' - 'raw.stdout[-1]|to_text == "2"' + fail_msg: | + raw={{ raw }} - name: Run raw module with sudo become: true @@ -39,3 +40,7 @@ ["root\r\n"], ["root"], ) + fail_msg: | + raw={{ raw }} + tags: + - low_level_execute_command diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index 0a018d4c5..4e706ed5a 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -9,10 +9,8 @@ - name: integration/action/make_tmp_path.yml hosts: test-targets - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml # # non-root @@ -44,11 +42,17 @@ assert: that: - good_temp_path == good_temp_path2 + fail_msg: | + good_temp_path={{ good_temp_path }} + good_temp_path2={{ good_temp_path2 }} - name: "Verify different subdir for both tasks" assert: that: - tmp_path.path != tmp_path2.path + fail_msg: | + tmp_path={{ tmp_path }} + tmp_path2={{ tmp_path2 }} # # Verify subdirectory removal. @@ -69,6 +73,9 @@ that: - not stat1.stat.exists - not stat2.stat.exists + fail_msg: | + stat1={{ stat1 }} + stat2={{ stat2 }} # # Verify good directory persistence. @@ -83,6 +90,8 @@ assert: that: - stat.stat.exists + fail_msg: | + stat={{ stat }} # # Write some junk into the temp path. @@ -105,6 +114,8 @@ - assert: that: - not out.stat.exists + fail_msg: | + out={{ out }} # # root @@ -123,22 +134,26 @@ that: - tmp_path2.path != tmp_path_root.path - tmp_path2.path|dirname != tmp_path_root.path|dirname + fail_msg: | + tmp_path_root={{ tmp_path_root }} + tmp_path2={{ tmp_path2 }} # # readonly homedir # - # TODO: https://github.com/dw/mitogen/issues/692 - # - name: "Try writing to temp directory for the readonly_homedir user" - # become: true - # become_user: mitogen__readonly_homedir - # custom_python_run_script: - # script: | - # from ansible.module_utils.basic import get_module_path - # path = get_module_path() + '/foo.txt' - # result['path'] = path - # open(path, 'w').write("bar") - # register: tmp_path + - name: Try writing to temp directory for the readonly_homedir user + become: true + become_user: mitogen__readonly_homedir + custom_python_run_script: + script: | + from ansible.module_utils.basic import get_module_path + path = get_module_path() + '/foo.txt' + result['path'] = path + open(path, 'w').write("bar") + register: tmp_path + when: + - become_unpriv_available # # modules get the same base dir @@ -153,3 +168,8 @@ that: - out.module_path.startswith(good_temp_path2) - out.module_tmpdir.startswith(good_temp_path2) + fail_msg: | + out={{ out }} + tags: + - make_tmp_path + - mitogen_only diff --git a/tests/ansible/integration/action/make_tmp_path__double.yml b/tests/ansible/integration/action/make_tmp_path__double.yml index 8b24d3223..fd66c13d4 100644 --- a/tests/ansible/integration/action/make_tmp_path__double.yml +++ b/tests/ansible/integration/action/make_tmp_path__double.yml @@ -1,7 +1,8 @@ # issue #554: double calls to make_tmp_path() fail with assertion error. Ensure # they succeed and are cleaned up correctly. -- hosts: target +- name: integration/action/make_tmp_path__double.yml + hosts: test-targets tasks: - mitogen_action_script: script: | @@ -18,3 +19,5 @@ script: | assert not self._remote_file_exists("{{ out.t1 }}") assert not self._remote_file_exists("{{ out.t2 }}") + tags: + - make_tmp_path_double diff --git a/tests/ansible/integration/action/remote_expand_user.yml b/tests/ansible/integration/action/remote_expand_user.yml index 37fc5ebeb..d7cefffe4 100644 --- a/tests/ansible/integration/action/remote_expand_user.yml +++ b/tests/ansible/integration/action/remote_expand_user.yml @@ -3,7 +3,6 @@ - name: integration/action/remote_expand_user.yml hosts: test-targets - any_errors_fatal: true tasks: - name: "Find out root's homedir." # Runs first because it blats regular Ansible facts with junk, so @@ -26,7 +25,9 @@ sudoable: false register: out - assert: - that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo' + that: out.result == user_facts.ansible_facts.ansible_user_dir ~ '/foo' + fail_msg: | + out={{ out }} - name: "Expand ~/foo with become active. ~ is become_user's home." action_passthrough: @@ -48,7 +49,9 @@ sudoable: false register: out - assert: - that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo' + that: out.result == user_facts.ansible_facts.ansible_user_dir ~ '/foo' + fail_msg: | + out={{ out }} - name: "Expanding $HOME/foo has no effect." action_passthrough: @@ -59,6 +62,8 @@ register: out - assert: that: out.result == '$HOME/foo' + fail_msg: | + out={{ out }} # ------------------------ @@ -70,7 +75,9 @@ sudoable: true register: out - assert: - that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo' + that: out.result == user_facts.ansible_facts.ansible_user_dir ~ '/foo' + fail_msg: | + out={{ out }} - name: "sudoable; Expand ~/foo with become active. ~ is become_user's home." action_passthrough: @@ -93,7 +100,9 @@ sudoable: true register: out - assert: - that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo' + that: out.result == user_facts.ansible_facts.ansible_user_dir ~ '/foo' + fail_msg: | + out={{ out }} - name: "sudoable; Expanding $HOME/foo has no effect." action_passthrough: @@ -104,3 +113,7 @@ register: out - assert: that: out.result == '$HOME/foo' + fail_msg: | + out={{ out }} + tags: + - remote_expand_user diff --git a/tests/ansible/integration/action/remote_file_exists.yml b/tests/ansible/integration/action/remote_file_exists.yml index 20a825c88..465a3af79 100644 --- a/tests/ansible/integration/action/remote_file_exists.yml +++ b/tests/ansible/integration/action/remote_file_exists.yml @@ -1,36 +1,36 @@ - name: integration/action/remote_file_exists.yml hosts: test-targets - any_errors_fatal: true tasks: - - - file: + - name: Ensure does-not-exist doesnt + file: path: /tmp/does-not-exist state: absent - - action_passthrough: method: _remote_file_exists args: ['/tmp/does-not-exist'] register: out - - assert: that: out.result == False + fail_msg: | + out={{ out }} - # --- - - - copy: + - name: Ensure does-exist does + copy: dest: /tmp/does-exist content: "I think, therefore I am" - - action_passthrough: method: _remote_file_exists args: ['/tmp/does-exist'] register: out - - assert: that: out.result == True + fail_msg: | + out={{ out }} - - file: + - name: Cleanup + file: path: /tmp/does-exist state: absent - + tags: + - remote_file_exists diff --git a/tests/ansible/integration/action/remove_tmp_path.yml b/tests/ansible/integration/action/remove_tmp_path.yml index 7a0c6c251..9807477be 100644 --- a/tests/ansible/integration/action/remove_tmp_path.yml +++ b/tests/ansible/integration/action/remove_tmp_path.yml @@ -4,14 +4,14 @@ # - name: integration/action/remove_tmp_path.yml hosts: test-targets - any_errors_fatal: true tasks: # # Use the copy module to cause a temporary directory to be created, and # return a result with a 'src' attribute pointing into that directory. # - - copy: + - name: Ensure remove_tmp_path_test + copy: dest: /tmp/remove_tmp_path_test content: "{{ 123123 | random }}" register: out @@ -23,6 +23,8 @@ - assert: that: - not out2.stat.exists + fail_msg: | + out={{ out }} - stat: path: "{{out.src|dirname}}" @@ -31,7 +33,11 @@ - assert: that: - not out2.stat.exists + fail_msg: | + out={{ out }} - file: path: /tmp/remove_tmp_path_test state: absent + tags: + - remove_tmp_path diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml index 31cfe5539..58f54cc82 100644 --- a/tests/ansible/integration/action/synchronize.yml +++ b/tests/ansible/integration/action/synchronize.yml @@ -2,7 +2,6 @@ - name: integration/action/synchronize.yml hosts: test-targets - any_errors_fatal: true vars: ansible_user: mitogen__has_sudo_pubkey ansible_become_pass: has_sudo_pubkey_password @@ -13,32 +12,39 @@ ansible_password: '' tasks: # must copy git file to set proper file mode. - - copy: + - name: Copy synchronize-action-key + copy: dest: /tmp/synchronize-action-key src: ../../../data/docker/mitogen__has_sudo_pubkey.key mode: u=rw,go= delegate_to: localhost - - file: + - name: Cleanup sync-test + file: path: /tmp/sync-test state: absent delegate_to: localhost + run_once: true - - file: + - name: Create sync-test + file: path: /tmp/sync-test state: directory delegate_to: localhost + run_once: true - - copy: + - name: Create syn-test item + copy: dest: /tmp/sync-test/item content: "item!" delegate_to: localhost + run_once: true - # TODO: https://github.com/dw/mitogen/issues/692 - # - file: - # path: /tmp/sync-test.out - # state: absent - # become: true + - name: Ensure clean slate + become: true + file: + path: /tmp/sync-test.out + state: absent # exception: File "/tmp/venv/lib/python2.7/site-packages/ansible/plugins/action/__init__.py", line 129, in cleanup # exception: self._remove_tmp_path(self._connection._shell.tmpdir) @@ -60,14 +66,18 @@ - assert: that: outout == "item!" + fail_msg: | + outout={{ outout }} when: False - # TODO: https://github.com/dw/mitogen/issues/692 - # - file: - # path: "{{item}}" - # state: absent - # become: true - # with_items: - # - /tmp/synchronize-action-key - # - /tmp/sync-test - # - /tmp/sync-test.out + - name: Cleanup + become: true + file: + path: "{{ item }}" + state: absent + with_items: + - /tmp/synchronize-action-key + - /tmp/sync-test + - /tmp/sync-test.out + tags: + - synchronize diff --git a/tests/ansible/integration/action/transfer_data.yml b/tests/ansible/integration/action/transfer_data.yml index bbd39309e..ab994683c 100644 --- a/tests/ansible/integration/action/transfer_data.yml +++ b/tests/ansible/integration/action/transfer_data.yml @@ -1,46 +1,49 @@ - name: integration/action/transfer_data.yml hosts: test-targets - any_errors_fatal: true tasks: - - - file: + - name: Cleanup transfer data + file: path: /tmp/transfer-data state: absent - # Ensure it JSON-encodes dicts. - - action_passthrough: + - name: Create JSON transfer data + action_passthrough: method: _transfer_data kwargs: remote_path: /tmp/transfer-data data: { "I am JSON": true } - - - slurp: + - name: Slurp JSON transfer data + slurp: src: /tmp/transfer-data register: out - - assert: that: | out.content|b64decode == '{"I am JSON": true}' + fail_msg: | + out={{ out }} - - # Ensure it handles strings. - - action_passthrough: + - name: Create text transfer data + action_passthrough: method: _transfer_data kwargs: remote_path: /tmp/transfer-data data: "I am text." - - - slurp: + - name: Slurp text transfer data + slurp: src: /tmp/transfer-data register: out - - assert: that: out.content|b64decode == 'I am text.' + fail_msg: | + out={{ out }} - - file: + - name: Cleanup transfer data + file: path: /tmp/transfer-data state: absent + tags: + - transfer_data diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index 8c059fc4e..ac196584e 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -3,21 +3,39 @@ # This playbook imports all tests that are known to work at present. # -- include: action/all.yml -- include: async/all.yml -- include: become/all.yml -- include: connection/all.yml -- include: connection_delegation/all.yml -- include: connection_loader/all.yml -- include: context_service/all.yml -- include: glibc_caches/all.yml -- include: interpreter_discovery/all.yml -- include: local/all.yml -- include: module_utils/all.yml -- include: playbook_semantics/all.yml -- include: process/all.yml -- include: runner/all.yml -- include: ssh/all.yml -- include: strategy/all.yml -- include: stub_connections/all.yml -- include: transport_config/all.yml +- import_playbook: action/all.yml + tags: action +- import_playbook: async/all.yml + tags: async +- import_playbook: become/all.yml + tags: become +- import_playbook: connection/all.yml + tags: connection +- import_playbook: connection_delegation/all.yml + tags: connection_delegation +- import_playbook: connection_loader/all.yml + tags: connection_loader +- import_playbook: context_service/all.yml + tags: context_service +- import_playbook: glibc_caches/all.yml + tags: glibc_caches +- import_playbook: interpreter_discovery/all.yml + tags: interpreter_discovery +- import_playbook: local/all.yml + tags: local +- import_playbook: module_utils/all.yml + tags: module_utils +- import_playbook: playbook_semantics/all.yml + tags: playbook_semantics +- import_playbook: process/all.yml + tags: process +- import_playbook: runner/all.yml + tags: runner +- import_playbook: ssh/all.yml + tags: ssh +- import_playbook: strategy/all.yml + tags: strategy +- import_playbook: stub_connections/all.yml + tags: stub_connections +- import_playbook: transport_config/all.yml + tags: transport_config diff --git a/tests/ansible/integration/async/all.yml b/tests/ansible/integration/async/all.yml index f14537ede..61d2d35ca 100644 --- a/tests/ansible/integration/async/all.yml +++ b/tests/ansible/integration/async/all.yml @@ -1,9 +1,9 @@ -- include: multiple_items_loop.yml -- include: result_binary_producing_json.yml -- include: result_binary_producing_junk.yml -- include: result_shell_echo_hi.yml -- include: runner_new_process.yml -- include: runner_one_job.yml -- include: runner_timeout_then_polling.yml -- include: runner_two_simultaneous_jobs.yml -- include: runner_with_polling_and_timeout.yml +- import_playbook: multiple_items_loop.yml +- import_playbook: result_binary_producing_json.yml +- import_playbook: result_binary_producing_junk.yml +- import_playbook: result_shell_echo_hi.yml +- import_playbook: runner_new_process.yml +- import_playbook: runner_one_job.yml +- import_playbook: runner_timeout_then_polling.yml +- import_playbook: runner_two_simultaneous_jobs.yml +- import_playbook: runner_with_polling_and_timeout.yml diff --git a/tests/ansible/integration/async/multiple_items_loop.yml b/tests/ansible/integration/async/multiple_items_loop.yml index 9a9b1192a..e7cc1bac0 100644 --- a/tests/ansible/integration/async/multiple_items_loop.yml +++ b/tests/ansible/integration/async/multiple_items_loop.yml @@ -2,7 +2,6 @@ - name: integration/async/multiple_items_loop.yml hosts: test-targets - any_errors_fatal: true tasks: - name: start long running ops @@ -34,3 +33,7 @@ - out.results[1].stdout == 'hi-from-job-2' - out.results[1].rc == 0 - out.results[1].delta > '0:00:05' + fail_msg: | + out={{ out }} + tags: + - multiple_items_loop diff --git a/tests/ansible/integration/async/result_binary_producing_json.yml b/tests/ansible/integration/async/result_binary_producing_json.yml index f81d0bb26..fc81ba9de 100644 --- a/tests/ansible/integration/async/result_binary_producing_json.yml +++ b/tests/ansible/integration/async/result_binary_producing_json.yml @@ -2,7 +2,6 @@ - name: integration/async/result_binary_producing_json.yml gather_facts: true hosts: test-targets - any_errors_fatal: true tasks: - block: @@ -28,6 +27,8 @@ (job.started == 1) and (job.changed == True) and (job.finished == 0) + fail_msg: | + job={{ job }} - name: busy-poll up to 100000 times async_status: @@ -37,7 +38,8 @@ retries: 100000 delay: 0 - - slurp: + - name: Slurp async busy-poll + slurp: src: "{{ansible_user_dir}}/.ansible_async/{{job.ansible_job_id}}" register: result @@ -51,5 +53,9 @@ - async_out.failed == False - async_out.msg == "Hello, world." - 'async_out.stderr == "binary_producing_json: oh noes\n"' + fail_msg: | + async_out={{ async_out }} vars: async_out: "{{result.content|b64decode|from_json}}" + tags: + - result_binary_producing_json diff --git a/tests/ansible/integration/async/result_binary_producing_junk.yml b/tests/ansible/integration/async/result_binary_producing_junk.yml index 87877db72..10c867695 100644 --- a/tests/ansible/integration/async/result_binary_producing_junk.yml +++ b/tests/ansible/integration/async/result_binary_producing_junk.yml @@ -2,7 +2,6 @@ - name: integration/async/result_binary_producing_junk.yml gather_facts: true hosts: test-targets - any_errors_fatal: true tasks: - block: @@ -39,5 +38,9 @@ - async_out.msg.startswith("Traceback") - '"ValueError: No start of json char found\n" in async_out.msg' - 'async_out.stderr == "binary_producing_junk: oh noes\n"' + fail_msg: | + async_out={{ async_out }} vars: async_out: "{{result.content|b64decode|from_json}}" + tags: + - result_binary_producing_junk diff --git a/tests/ansible/integration/async/result_shell_echo_hi.yml b/tests/ansible/integration/async/result_shell_echo_hi.yml index e1068587a..d3f9d42cb 100644 --- a/tests/ansible/integration/async/result_shell_echo_hi.yml +++ b/tests/ansible/integration/async/result_shell_echo_hi.yml @@ -2,17 +2,19 @@ - name: integration/async/result_shell_echo_hi.yml gather_facts: true hosts: test-targets - any_errors_fatal: true tasks: - - shell: echo hi; echo there >&2 + - name: Async shell + shell: echo hi; echo there >&2 async: 100 poll: 0 register: job - - shell: sleep 1 + - name: Sleepy shell + shell: sleep 1 - - slurp: + - name: Slurp async shell + slurp: src: "{{ansible_user_dir}}/.ansible_async/{{job.ansible_job_id}}" register: result @@ -30,17 +32,29 @@ - async_out.invocation.module_args.creates == None - async_out.invocation.module_args.executable == None - async_out.invocation.module_args.removes == None - - async_out.invocation.module_args.warn == True + # | Ansible <= 3 | ansible-core <= 2.10 | present | True | + # | Ansible 4 - 6 | ansible-core 2.11 - 2.13 | deprecated | False | + # | Ansible >= 7 | ansible-core >= 2.14 | absent | n/a | + - (ansible_version.full is version("2.14", ">=", strict=True) and async_out.invocation.module_args.warn is not defined) + or (ansible_version.full is version("2.11", ">=", strict=True) and async_out.invocation.module_args.warn == False) + or (async_out.invocation.module_args.warn == True) - async_out.rc == 0 - async_out.start.startswith("20") - async_out.stderr == "there" - async_out.stdout == "hi" + fail_msg: | + async_out={{ async_out }} vars: async_out: "{{result.content|b64decode|from_json}}" - assert: that: - async_out.invocation.module_args.stdin == None - when: ansible_version.full > '2.4' + fail_msg: | + async_out={{ async_out }} + when: + - ansible_version.full is version('2.4', '>=', strict=True) vars: async_out: "{{result.content|b64decode|from_json}}" + tags: + - result_shell_echo_hi diff --git a/tests/ansible/integration/async/runner_new_process.yml b/tests/ansible/integration/async/runner_new_process.yml index 7b0bf628a..0ed0798a1 100644 --- a/tests/ansible/integration/async/runner_new_process.yml +++ b/tests/ansible/integration/async/runner_new_process.yml @@ -2,7 +2,6 @@ - name: integration/async/runner_new_process.yml hosts: test-targets - any_errors_fatal: true tasks: - name: get process ID. @@ -16,6 +15,9 @@ - assert: that: - sync_proc1.pid == sync_proc2.pid + fail_msg: | + sync_proc1={{ sync_proc1 }} + sync_proc2={{ sync_proc2 }} when: is_mitogen - name: get async process ID. @@ -48,7 +50,13 @@ - assert: that: + # FIXME should this be async_proc1, and async_proc2? - sync_proc1.pid == sync_proc2.pid - async_result1.pid != sync_proc1.pid - async_result1.pid != async_result2.pid + fail_msg: | + async_result1={{ async_result1 }} + async_result2={{ async_result2 }} when: is_mitogen + tags: + - runner_new_process diff --git a/tests/ansible/integration/async/runner_one_job.yml b/tests/ansible/integration/async/runner_one_job.yml index bea6ed9c2..95a32e2b5 100644 --- a/tests/ansible/integration/async/runner_one_job.yml +++ b/tests/ansible/integration/async/runner_one_job.yml @@ -3,7 +3,6 @@ - name: integration/async/runner_one_job.yml hosts: test-targets - any_errors_fatal: true tasks: # Verify output of a single async job. @@ -24,6 +23,8 @@ (job1.started == 1) and (job1.changed == True) and (job1.finished == 0) + fail_msg: | + job1={{ job1 }} - name: busy-poll up to 100000 times async_status: @@ -40,14 +41,16 @@ - result1.changed == True # ansible/b72e989e1837ccad8dcdc926c43ccbc4d8cdfe44 - | - (ansible_version.full is version('2.8', ">=") and + (ansible_version.full is version('2.8', ">=", strict=True) and result1.cmd == "echo alldone;\nsleep 1;\n") or - (ansible_version.full is version('2.8', '<') and + (ansible_version.full is version('2.8', '<', strict=True) and result1.cmd == "echo alldone;\n sleep 1;") - result1.delta|length == 14 - result1.start|length == 26 - result1.finished == 1 - result1.rc == 0 + fail_msg: | + result1={{ result1 }} - assert: that: @@ -55,9 +58,17 @@ - result1.stderr_lines == [] - result1.stdout == "alldone" - result1.stdout_lines == ["alldone"] - when: ansible_version.full is version('2.8', '>') # ansible#51393 + fail_msg: | + result1={{ result1 }} + when: + - ansible_version.full is version('2.8', '>', strict=True) # ansible#51393 - assert: that: - result1.failed == False - when: ansible_version.full is version('2.4', '>') + fail_msg: | + result1={{ result1 }} + when: + - ansible_version.full is version('2.4', '>', strict=True) + tags: + - runner_one_job diff --git a/tests/ansible/integration/async/runner_timeout_then_polling.yml b/tests/ansible/integration/async/runner_timeout_then_polling.yml index 5490e7115..783d30a07 100644 --- a/tests/ansible/integration/async/runner_timeout_then_polling.yml +++ b/tests/ansible/integration/async/runner_timeout_then_polling.yml @@ -2,8 +2,8 @@ - name: integration/async/runner_timeout_then_polling.yml hosts: test-targets - any_errors_fatal: true tasks: + - include_tasks: ../_mitogen_only.yml # Verify async-with-timeout-then-poll behaviour. # This is semi-broken in upstream Ansible, it does not bother to update the @@ -14,7 +14,6 @@ async: 1 poll: 0 register: job - when: is_mitogen - name: busy-poll up to 500 times async_status: @@ -23,7 +22,6 @@ until: result.finished retries: 500 delay: 0 - when: is_mitogen ignore_errors: true - assert: @@ -31,4 +29,8 @@ - result.failed == 1 - result.finished == 1 - result.msg == "Job reached maximum time limit of 1 seconds." - when: is_mitogen + fail_msg: | + result={{ result }} + tags: + - mitogen_only + - runner_timeout_then_polling diff --git a/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml b/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml index fdde04633..4d236c3e3 100644 --- a/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml +++ b/tests/ansible/integration/async/runner_two_simultaneous_jobs.yml @@ -1,12 +1,12 @@ - name: integration/async/runner_two_simultaneous_jobs.yml hosts: test-targets - any_errors_fatal: true tasks: # Start 2 duplicate jobs, verify they run concurrently. - - file: + - name: Cleanup semaphore file + file: path: /tmp/flurp state: absent @@ -56,8 +56,16 @@ that: - result1.rc == 0 - result2.rc == 0 + fail_msg: | + result1={{ result1 }} + result2={{ result2 }} - assert: that: - result2.stdout == 'im_alive' - when: ansible_version.full > '2.8' # ansible#51393 + fail_msg: | + result2={{ result2 }} + when: + - ansible_version.full is version('2.8', '>=', strict=True) # ansible#51393 + tags: + - runner_two_simultaneous_jobs diff --git a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml index dcfa186f4..2975e7acd 100644 --- a/tests/ansible/integration/async/runner_with_polling_and_timeout.yml +++ b/tests/ansible/integration/async/runner_with_polling_and_timeout.yml @@ -2,7 +2,6 @@ - name: integration/async/runner_with_polling_and_timeout.yml hosts: test-targets - any_errors_fatal: true tasks: # Verify async-with-polling-and-timeout behaviour. @@ -22,4 +21,7 @@ job1.msg == "async task did not complete within the requested time" or job1.msg == "async task did not complete within the requested time - 1s" or job1.msg == "Job reached maximum time limit of 1 seconds." - + fail_msg: | + job1={{ job1 }} + tags: + - runner_with_polling_and_timeout diff --git a/tests/ansible/integration/become/all.yml b/tests/ansible/integration/become/all.yml index 5fa030d17..1b507e166 100644 --- a/tests/ansible/integration/become/all.yml +++ b/tests/ansible/integration/become/all.yml @@ -1,7 +1,11 @@ -- include: su_password.yml -- include: sudo_flags_failure.yml -- include: sudo_nonexistent.yml -- include: sudo_nopassword.yml -- include: sudo_password.yml -- include: sudo_requiretty.yml +- import_playbook: su_password.yml +- import_playbook: sudo_flags_failure.yml +- import_playbook: sudo_nonexistent.yml +- import_playbook: sudo_nopassword.yml +- import_playbook: sudo_password.yml +- import_playbook: sudo_requiretty.yml +- import_playbook: templated_by_inv.yml +- import_playbook: templated_by_play_keywords.yml +- import_playbook: templated_by_play_vars.yml +- import_playbook: templated_by_task_keywords.yml diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index f6eb0b47c..19a15c7e4 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -1,10 +1,8 @@ # Verify passwordful su behaviour - # Ansible can't handle this on OS X. I don't care why. - name: integration/become/su_password.yml hosts: test-targets become_method: su - any_errors_fatal: true tasks: - name: Ensure su password absent but required. @@ -22,6 +20,8 @@ ('password is required' in out.msg) or ('password is required' in out.module_stderr) ) + fail_msg: | + out={{ out }} when: is_mitogen @@ -41,18 +41,48 @@ ('Incorrect su password' in out.msg) or ('su password is incorrect' in out.msg) ) + fail_msg: | + out={{ out }} when: is_mitogen - - name: Ensure password su succeeds. + - name: Ensure password su with chdir succeeds shell: whoami + args: + chdir: ~mitogen__user1 become: true become_user: mitogen__user1 register: out vars: ansible_become_pass: user1_password - when: is_mitogen + when: + - become_unpriv_available - assert: that: - out.stdout == 'mitogen__user1' - when: is_mitogen + fail_msg: | + out={{ out }} + when: + - become_unpriv_available + + - name: Ensure password su without chdir succeeds + shell: whoami + become: true + become_user: mitogen__user1 + register: out + vars: + ansible_become_pass: user1_password + when: + - become_unpriv_available + + - assert: + that: + - out.stdout == 'mitogen__user1' + fail_msg: | + out={{ out }} + when: + - become_unpriv_available + + tags: + - su + - su_password diff --git a/tests/ansible/integration/become/sudo_flags_failure.yml b/tests/ansible/integration/become/sudo_flags_failure.yml index 39fbb4b84..e7c012027 100644 --- a/tests/ansible/integration/become/sudo_flags_failure.yml +++ b/tests/ansible/integration/become/sudo_flags_failure.yml @@ -1,6 +1,5 @@ - name: integration/become/sudo_flags_failure.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Verify behaviour for bad sudo flags. @@ -15,8 +14,14 @@ assert: that: - out.failed - - | - ('sudo: no such option: --derps' in out.msg) or - ("sudo: invalid option -- '-'" in out.module_stderr) or - ("sudo: unrecognized option `--derps'" in out.module_stderr) or - ("sudo: unrecognized option '--derps'" in out.module_stderr) + - >- + 'sudo: no such option: --derps' in out.msg + or out.module_stdout is match("sudo: invalid option -- '-'") + or out.module_stderr is match("sudo: invalid option -- '-'") + or out.module_stdout is match("sudo: unrecognized option [`']--derps'") + or out.module_stderr is match("sudo: unrecognized option [`']--derps'") + fail_msg: | + out={{ out }} + tags: + - sudo + - sudo_flags_failure diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index 09ab91bba..208f0c9b8 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -1,6 +1,5 @@ - name: integration/become/sudo_nonexistent.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Verify behaviour for non-existent accounts. @@ -9,6 +8,8 @@ become_user: slartibartfast ignore_errors: true register: out + when: + - become_unpriv_available - name: Verify raw module output. assert: @@ -17,5 +18,12 @@ # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release) # removed user/group error messages, as defence against CVE-2019-14287. - >- - ('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)) + (out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast') or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10') + fail_msg: | + out={{ out }} + when: + - become_unpriv_available + tags: + - sudo + - sudo_nonexistent diff --git a/tests/ansible/integration/become/sudo_nopassword.yml b/tests/ansible/integration/become/sudo_nopassword.yml index 036444232..0e84ce5ee 100644 --- a/tests/ansible/integration/become/sudo_nopassword.yml +++ b/tests/ansible/integration/become/sudo_nopassword.yml @@ -1,8 +1,7 @@ # Verify passwordless sudo behaviour in various cases. -- name: integration/become/sudo_basic.yml +- name: integration/become/sudo_nopassword.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Verify we aren't root @@ -12,6 +11,8 @@ - assert: that: - out.stdout != 'root' + fail_msg: | + out={{ out }} - name: Ensure passwordless sudo to root succeeds. shell: whoami @@ -22,3 +23,8 @@ - assert: that: - out.stdout == 'root' + fail_msg: | + out={{ out }} + tags: + - sudo + - sudo_nopassword diff --git a/tests/ansible/integration/become/sudo_password.yml b/tests/ansible/integration/become/sudo_password.yml index 145c8d69c..150cb1f0f 100644 --- a/tests/ansible/integration/become/sudo_password.yml +++ b/tests/ansible/integration/become/sudo_password.yml @@ -2,15 +2,18 @@ - name: integration/become/sudo_password.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Ensure sudo password absent but required. - shell: whoami become: true become_user: mitogen__pw_required + command: + cmd: whoami register: out + changed_when: false ignore_errors: true + when: + - become_unpriv_available - assert: that: | @@ -19,15 +22,23 @@ ('Missing sudo password' in out.msg) or ('password is required' in out.module_stderr) ) + fail_msg: | + out={{ out }} + when: + - become_unpriv_available - name: Ensure password sudo incorrect. - shell: whoami become: true become_user: mitogen__pw_required + command: + cmd: whoami register: out + changed_when: false vars: ansible_become_pass: nopes ignore_errors: true + when: + - become_unpriv_available - assert: that: | @@ -35,16 +46,29 @@ ('Incorrect sudo password' in out.msg) or ('sudo password is incorrect' in out.msg) ) + fail_msg: | + out={{ out }} + when: + - become_unpriv_available - # TODO: https://github.com/dw/mitogen/issues/692 - # - name: Ensure password sudo succeeds. - # shell: whoami - # become: true - # become_user: mitogen__pw_required - # register: out - # vars: - # ansible_become_pass: pw_required_password + - block: + - name: Ensure password sudo succeeds + become: true + become_user: mitogen__pw_required + vars: + ansible_become_pass: pw_required_password + command: + cmd: whoami + register: sudo_password_success_whoami + changed_when: false - # - assert: - # that: - # - out.stdout == 'mitogen__pw_required' + - assert: + that: + - sudo_password_success_whoami.stdout == 'mitogen__pw_required' + fail_msg: | + sudo_password_success_whoami={{ sudo_password_success_whoami }} + when: + - become_unpriv_available + tags: + - sudo + - sudo_password diff --git a/tests/ansible/integration/become/sudo_requiretty.yml b/tests/ansible/integration/become/sudo_requiretty.yml index dd62d9a09..7849035b9 100644 --- a/tests/ansible/integration/become/sudo_requiretty.yml +++ b/tests/ansible/integration/become/sudo_requiretty.yml @@ -2,36 +2,48 @@ - name: integration/become/sudo_requiretty.yml hosts: test-targets - any_errors_fatal: true tasks: + # AIUI Vanilla Ansible cannot do sudo when requiretty configured + - include_tasks: ../_mitogen_only.yml - # TODO: https://github.com/dw/mitogen/issues/692 - # - name: Verify we can login to a non-passworded requiretty account - # shell: whoami - # become: true - # become_user: mitogen__require_tty - # register: out - # when: is_mitogen + - name: Verify we can login to a non-passworded requiretty account + become: true + become_user: mitogen__require_tty + command: + cmd: whoami + changed_when: false + register: sudo_require_tty_whoami + when: + - become_unpriv_available - # - assert: - # that: - # - out.stdout == 'mitogen__require_tty' - # when: is_mitogen + - assert: + that: + - sudo_require_tty_whoami.stdout == 'mitogen__require_tty' + fail_msg: | + sudo_require_tty_whoami={{ sudo_require_tty_whoami }} + when: + - become_unpriv_available + - name: Verify we can login to a passworded requiretty account + become: true + become_user: mitogen__require_tty_pw_required + vars: + ansible_become_pass: require_tty_pw_required_password + command: + cmd: whoami + changed_when: false + register: sudo_require_tty_password_whoami + when: + - become_unpriv_available - # --------------- - - # TODO: https://github.com/dw/mitogen/issues/692 - # - name: Verify we can login to a passworded requiretty account - # shell: whoami - # become: true - # become_user: mitogen__require_tty_pw_required - # vars: - # ansible_become_pass: require_tty_pw_required_password - # register: out - # when: is_mitogen - - # - assert: - # that: - # - out.stdout == 'mitogen__require_tty_pw_required' - # when: is_mitogen + - assert: + that: + - sudo_require_tty_password_whoami.stdout == 'mitogen__require_tty_pw_required' + fail_msg: | + sudo_require_tty_password_whoami={{ sudo_require_tty_password_whoami }} + when: + - become_unpriv_available + tags: + - mitogen_only + - sudo + - sudo_requiretty diff --git a/tests/ansible/integration/become/templated_by_inv.yml b/tests/ansible/integration/become/templated_by_inv.yml new file mode 100644 index 000000000..a2e9c457a --- /dev/null +++ b/tests/ansible/integration/become/templated_by_inv.yml @@ -0,0 +1,31 @@ +- name: integration/become/templated_by_inv.yml + hosts: tt_become_by_inv + gather_facts: false + tasks: + - name: Gather facts (avoiding any unprivileged become) + vars: + ansible_become: false + setup: + + - meta: reset_connection + + - name: Templated become in inventory + vars: + expected_become_users: + tt-become: root + tt-become-exe: root + tt-become-flags: root + tt-become-method: root + tt-become-pass: mitogen__pw_required + tt-become-user: root + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_inv_whoami + failed_when: + - become_templated_by_inv_whoami is failed + or become_templated_by_inv_whoami.stdout != expected_become_users[inventory_hostname] + when: + - ansible_become_user in ['root'] + or become_unpriv_available diff --git a/tests/ansible/integration/become/templated_by_play_keywords.yml b/tests/ansible/integration/become/templated_by_play_keywords.yml new file mode 100644 index 000000000..6f6ea2495 --- /dev/null +++ b/tests/ansible/integration/become/templated_by_play_keywords.yml @@ -0,0 +1,50 @@ +- name: integration/become/templated_by_play_keywords.yml + hosts: tt_become_bare + gather_facts: false + become: "{{ 'true' | trim }}" + become_exe: "{{ 'sudo' | trim }}" + become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + become_method: "{{ 'sudo' | trim }}" + become_user: "{{ 'root' | trim }}" + tasks: + - meta: reset_connection + + - name: Templated become by play keywords, no password + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_keywords_whoami + failed_when: + - become_templated_by_play_keywords_whoami is failed + or become_templated_by_play_keywords_whoami.stdout != 'root' + +- name: integration/become/templated_by_play_keywords.yml + hosts: tt_become_bare + gather_facts: false + become: "{{ 'true' | trim }}" + become_exe: "{{ 'sudo' | trim }}" + become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + become_method: "{{ 'sudo' | trim }}" + become_user: "{{ 'mitogen__pw_required' | trim }}" + vars: + ansible_become_pass: "{{ 'pw_required_password' | trim }}" + tasks: + - name: Gather facts (avoiding any unprivileged become) + vars: + ansible_become: false + setup: + + - meta: reset_connection + + - name: Templated become by play keywords, password + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_keywords_password_whoami + failed_when: + - become_templated_by_play_keywords_password_whoami is failed + or become_templated_by_play_keywords_password_whoami.stdout != 'mitogen__pw_required' + when: + - become_unpriv_available diff --git a/tests/ansible/integration/become/templated_by_play_vars.yml b/tests/ansible/integration/become/templated_by_play_vars.yml new file mode 100644 index 000000000..ba7c143ea --- /dev/null +++ b/tests/ansible/integration/become/templated_by_play_vars.yml @@ -0,0 +1,49 @@ +- name: integration/become/templated_by_play_vars.yml + hosts: tt_become_bare + gather_facts: false + vars: + ansible_become: true + ansible_become_exe: "{{ 'sudo' | trim }}" + ansible_become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + ansible_become_method: "{{ 'sudo' | trim }}" + ansible_become_user: "{{ 'root' | trim }}" + tasks: + - name: Templated become by play vars, no password + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_vars_whoami + failed_when: + - become_templated_by_play_vars_whoami is failed + or become_templated_by_play_vars_whoami.stdout != 'root' + +- name: integration/become/templated_by_play_vars.yml + hosts: tt_become_bare + gather_facts: false + vars: + ansible_become: true + ansible_become_exe: "{{ 'sudo' | trim }}" + ansible_become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + ansible_become_method: "{{ 'sudo' | trim }}" + ansible_become_pass: "{{ 'pw_required_password' | trim }}" + ansible_become_user: "{{ 'mitogen__pw_required' | trim }}" + tasks: + - name: Gather facts (avoiding any unprivileged become) + vars: + ansible_become: false + setup: + + - meta: reset_connection + + - name: Templated become by play vars, password + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_vars_password_whoami + failed_when: + - become_templated_by_play_vars_password_whoami is failed + or become_templated_by_play_vars_password_whoami.stdout != 'mitogen__pw_required' + when: + - become_unpriv_available diff --git a/tests/ansible/integration/become/templated_by_task_keywords.yml b/tests/ansible/integration/become/templated_by_task_keywords.yml new file mode 100644 index 000000000..ca61e3891 --- /dev/null +++ b/tests/ansible/integration/become/templated_by_task_keywords.yml @@ -0,0 +1,80 @@ +- name: integration/become/templated_by_task_keywords.yml + hosts: tt_become_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + become: "{{ 'true' | trim }}" + become_exe: "{{ 'sudo' | trim }}" + become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + become_method: "{{ 'sudo' | trim }}" + become_user: "{{ 'root' | trim }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Templated become by task keywords, with delegate_to + become: "{{ 'true' | trim }}" + become_exe: "{{ 'sudo' | trim }}" + become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + become_method: "{{ 'sudo' | trim }}" + become_user: "{{ 'root' | trim }}" + delegate_to: "{{ groups.tt_become_bare[0] }}" + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_task_with_delegate_to_whoami + failed_when: + - become_templated_by_task_with_delegate_to_whoami is failed + or become_templated_by_task_with_delegate_to_whoami.stdout != 'root' + + +- name: integration/become/templated_by_task_keywords.yml + hosts: tt_become_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + become: "{{ 'true' | trim }}" + become_exe: "{{ 'sudo' | trim }}" + become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + become_method: "{{ 'sudo' | trim }}" + become_user: "{{ 'mitogen__pw_required' | trim }}" + vars: + ansible_become_pass: "{{ 'pw_required_password' | trim }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Gather facts (avoiding any unprivileged become) + delegate_to: "{{ groups.tt_become_bare[0] }}" + vars: + ansible_become: false + setup: + + - name: Templated become by task keywords, with delegate_to + become: "{{ 'true' | trim }}" + become_exe: "{{ 'sudo' | trim }}" + become_flags: "{{ '--set-home --stdin --non-interactive' | trim }}" + become_method: "{{ 'sudo' | trim }}" + become_user: "{{ 'mitogen__pw_required' | trim }}" + delegate_to: "{{ groups.tt_become_bare[0] }}" + vars: + ansible_become_pass: "{{ 'pw_required_password' | trim }}" + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_task_with_delegate_to_password_whoami + failed_when: + - become_templated_by_task_with_delegate_to_password_whoami is failed + or become_templated_by_task_with_delegate_to_password_whoami.stdout != 'mitogen__pw_required' + when: + - become_unpriv_available diff --git a/tests/ansible/integration/connection/_cleanup_file.yml b/tests/ansible/integration/connection/_cleanup_file.yml new file mode 100644 index 000000000..c1b862558 --- /dev/null +++ b/tests/ansible/integration/connection/_cleanup_file.yml @@ -0,0 +1,17 @@ +- name: Cleanup local file + file: + path: /tmp/{{ file_name }} + state: absent + delegate_to: localhost + run_once: true + tags: + - cleanup_local + - cleanup + +- name: Cleanup target file + file: + path: /tmp/{{ file_name }}.out + state: absent + tags: + - cleanup_target + - cleanup diff --git a/tests/ansible/integration/connection/_put_file.yml b/tests/ansible/integration/connection/_put_file.yml index 5b661d9f1..5d3ad6253 100644 --- a/tests/ansible/integration/connection/_put_file.yml +++ b/tests/ansible/integration/connection/_put_file.yml @@ -1,19 +1,24 @@ ---- - -- shell: dd if=/dev/urandom of=/tmp/{{file_name}} bs=1024 count={{file_size}} - args: +- name: Create {{ file_name }} + command: + cmd: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size_kib }} creates: /tmp/{{file_name}} delegate_to: localhost + run_once: true -- copy: +- name: Copy {{ file_name }} + copy: dest: /tmp/{{file_name}}.out src: /tmp/{{file_name}} + mode: "{{ file_mode }}" -- stat: path=/tmp/{{file_name}} +- name: Stat created {{ file_name }} + stat: path=/tmp/{{ file_name }} register: original delegate_to: localhost + run_once: true -- stat: path=/tmp/{{file_name}}.out +- name: Stat copied {{ file_name }} + stat: path=/tmp/{{ file_name }}.out register: copied - assert: @@ -21,3 +26,6 @@ - original.stat.checksum == copied.stat.checksum # Upstream does not preserve timestamps at al. #- (not is_mitogen) or (original.stat.mtime|int == copied.stat.mtime|int) + fail_msg: | + original={{ original }} + copied={{ copied }} diff --git a/tests/ansible/integration/connection/all.yml b/tests/ansible/integration/connection/all.yml index 348857f54..b707b0eb1 100644 --- a/tests/ansible/integration/connection/all.yml +++ b/tests/ansible/integration/connection/all.yml @@ -1,11 +1,11 @@ --- -- include: become_same_user.yml -- include: disconnect_during_module.yml -- include: disconnect_resets_connection.yml -- include: exec_command.yml -- include: home_dir.yml -- include: put_large_file.yml -- include: put_small_file.yml -- include: reset.yml -- include: reset_become.yml +- import_playbook: become_same_user.yml +- import_playbook: disconnect_during_module.yml +- import_playbook: disconnect_resets_connection.yml +- import_playbook: exec_command.yml +- import_playbook: home_dir.yml +- import_playbook: put_large_file.yml +- import_playbook: put_small_file.yml +- import_playbook: reset.yml +- import_playbook: reset_become.yml diff --git a/tests/ansible/integration/connection/become_same_user.yml b/tests/ansible/integration/connection/become_same_user.yml index d73eca86a..fd8511e3c 100644 --- a/tests/ansible/integration/connection/become_same_user.yml +++ b/tests/ansible/integration/connection/become_same_user.yml @@ -4,22 +4,22 @@ - name: integration/connection/become_same_user.yml hosts: bsu-joe gather_facts: no - any_errors_fatal: true tasks: + - include_tasks: ../_mitogen_only.yml # bsu-joe's login user is joe, so become should be ignored. - mitogen_get_stack: become: true become_user: joe register: out - when: is_mitogen - assert: that: - out.result[0].method == "ssh" - out.result[0].kwargs.username == "joe" - out.result|length == 1 # no sudo - when: is_mitogen + fail_msg: | + out={{ out }} # Now try with a different account. @@ -27,7 +27,6 @@ become: true become_user: james register: out - when: is_mitogen - assert: that: @@ -36,4 +35,8 @@ - out.result[1].method == "sudo" - out.result[1].kwargs.username == "james" - out.result|length == 2 # no sudo - when: is_mitogen + fail_msg: | + out={{ out }} + tags: + - become_same_user + - mitogen_only diff --git a/tests/ansible/integration/connection/disconnect_during_module.yml b/tests/ansible/integration/connection/disconnect_during_module.yml index e628e68ea..3846d0370 100644 --- a/tests/ansible/integration/connection/disconnect_during_module.yml +++ b/tests/ansible/integration/connection/disconnect_during_module.yml @@ -6,13 +6,17 @@ gather_facts: no any_errors_fatal: false tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - - delegate_to: localhost + - name: Run _disconnect_during_module.yml + delegate_to: localhost + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook - -i "{{MITOGEN_INVENTORY_FILE}}" + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} integration/connection/_disconnect_during_module.yml args: chdir: ../.. @@ -25,3 +29,9 @@ that: - out.rc == 4 - "'Mitogen was disconnected from the remote environment while a call was in-progress.' in out.stdout" + fail_msg: | + out={{ out }} + tags: + - disconnect + - disconnect_during_module + - mitogen_only diff --git a/tests/ansible/integration/connection/disconnect_resets_connection.yml b/tests/ansible/integration/connection/disconnect_resets_connection.yml index 5f02a8d5d..d9879a71f 100644 --- a/tests/ansible/integration/connection/disconnect_resets_connection.yml +++ b/tests/ansible/integration/connection/disconnect_resets_connection.yml @@ -12,10 +12,8 @@ - name: integration/connection/disconnect_resets_connection.yml hosts: test-targets gather_facts: no - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - mitogen_action_script: script: | @@ -45,3 +43,7 @@ except AnsibleConnectionFailure: e = sys.exc_info()[1] assert str(e).startswith('Mitogen was disconnected') + tags: + - disconnect + - disconnect_resets_connection + - mitogen_only diff --git a/tests/ansible/integration/connection/exec_command.yml b/tests/ansible/integration/connection/exec_command.yml index 105505d1e..3ac065061 100644 --- a/tests/ansible/integration/connection/exec_command.yml +++ b/tests/ansible/integration/connection/exec_command.yml @@ -4,7 +4,6 @@ - name: integration/connection/exec_command.yml hosts: test-targets gather_facts: no - any_errors_fatal: true tasks: - connection_passthrough: method: exec_command @@ -17,3 +16,7 @@ - out.result[0] == 0 - out.result[1].decode() == "hello, world\r\n" - out.result[2].decode().startswith("Shared connection to ") + fail_msg: | + out={{ out }} + tags: + - exec_command diff --git a/tests/ansible/integration/connection/home_dir.yml b/tests/ansible/integration/connection/home_dir.yml index 10154450b..07ce9dd1e 100644 --- a/tests/ansible/integration/connection/home_dir.yml +++ b/tests/ansible/integration/connection/home_dir.yml @@ -2,20 +2,19 @@ - name: integration/connection/home_dir.yml hosts: test-targets - any_errors_fatal: true tasks: + - include_tasks: ../_mitogen_only.yml + - name: "Find out root's homedir." # Runs first because it blats regular Ansible facts with junk, so # non-become run fixes that up. setup: become: true register: root_facts - when: is_mitogen - name: "Find regular homedir" setup: register: user_facts - when: is_mitogen - name: "Verify Connection.homedir correct when become:false" mitogen_action_script: @@ -25,7 +24,6 @@ "connection homedir": self._connection.homedir, "homedir from facts": "{{user_facts.ansible_facts.ansible_user_dir}}" } - when: is_mitogen - name: "Verify Connection.homedir correct when become:true" become: true @@ -36,4 +34,6 @@ "connection homedir": self._connection.homedir, "homedir from facts": "{{root_facts.ansible_facts.ansible_user_dir}}" } - when: is_mitogen + tags: + - home_dir + - mitogen_only diff --git a/tests/ansible/integration/connection/put_large_file.yml b/tests/ansible/integration/connection/put_large_file.yml index 392731df7..e6f5f6454 100644 --- a/tests/ansible/integration/connection/put_large_file.yml +++ b/tests/ansible/integration/connection/put_large_file.yml @@ -4,9 +4,13 @@ - name: integration/connection/put_large_file.yml hosts: test-targets gather_facts: no - any_errors_fatal: true vars: file_name: large-file - file_size: 512 + file_size_kib: 512 + file_mode: u=rw,go= tasks: - - include: _put_file.yml + - include_tasks: _put_file.yml + - include_tasks: _cleanup_file.yml + tags: + - put_file + - put_large_file diff --git a/tests/ansible/integration/connection/put_small_file.yml b/tests/ansible/integration/connection/put_small_file.yml index d9423f755..06c41673c 100644 --- a/tests/ansible/integration/connection/put_small_file.yml +++ b/tests/ansible/integration/connection/put_small_file.yml @@ -4,9 +4,13 @@ - name: integration/connection/put_small_file.yml hosts: test-targets gather_facts: no - any_errors_fatal: true vars: file_name: small-file - file_size: 123 + file_size_kib: 123 + file_mode: u=rw,go= tasks: - - include: _put_file.yml + - include_tasks: _put_file.yml + - include_tasks: _cleanup_file.yml + tags: + - put_file + - put_small_file diff --git a/tests/ansible/integration/connection/reset.yml b/tests/ansible/integration/connection/reset.yml index 768cd2d54..2d7a75d3d 100644 --- a/tests/ansible/integration/connection/reset.yml +++ b/tests/ansible/integration/connection/reset.yml @@ -6,14 +6,15 @@ - name: integration/connection/reset.yml hosts: test-targets tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - debug: msg="reset.yml skipped on Ansible<2.5.6" - when: ansible_version.full < '2.5.6' + when: + - ansible_version.full is version('2.5.6', '<', strict=True) - meta: end_play - when: ansible_version.full < '2.5.6' + when: + - ansible_version.full is version('2.5.6', '<', strict=True) - custom_python_detect_environment: register: out @@ -43,3 +44,11 @@ # sudo PID has changed. - out_become.ppid != out_become2.ppid + fail_msg: | + out={{ out }} + out2={{ out2 }} + out_become={{ out_become }} + out_become2={{ out_become2 }} + tags: + - mitogen_only + - reset diff --git a/tests/ansible/integration/connection/reset_become.yml b/tests/ansible/integration/connection/reset_become.yml index 5a411e82b..2548df17f 100644 --- a/tests/ansible/integration/connection/reset_become.yml +++ b/tests/ansible/integration/connection/reset_become.yml @@ -1,15 +1,18 @@ # issue #633: Connection.reset() should ignore "become", and apply to the login # account. -- hosts: test-targets +- name: integration/connection/reset_become.yml + hosts: test-targets become: true gather_facts: false tasks: - debug: msg="reset_become.yml skipped on Ansible<2.5.6" - when: ansible_version.full < '2.5.6' + when: + - ansible_version.full is version('2.5.6', '<', strict=True) - meta: end_play - when: ansible_version.full < '2.5.6' + when: + - ansible_version.full is version('2.5.6', '<', strict=True) - name: save pid of the become acct custom_python_detect_environment: @@ -24,6 +27,9 @@ assert: that: - become_acct.pid != login_acct.pid + fail_msg: | + become_acct={{ become_acct }} + login_acct={{ login_acct }} - name: reset the connection meta: reset_connection @@ -36,6 +42,9 @@ assert: that: - become_acct.pid != new_become_acct.pid + fail_msg: | + become_acct={{ become_acct }} + new_become_acct={{ new_become_acct }} - name: save new pid of login acct become: false @@ -46,3 +55,8 @@ assert: that: - login_acct.pid != new_login_acct.pid + fail_msg: | + login_acct={{ login_acct }} + new_login_acct={{ new_login_acct }} + tags: + - reset_become diff --git a/tests/ansible/integration/connection_delegation/all.yml b/tests/ansible/integration/connection_delegation/all.yml index c9b096871..cb55bdc75 100644 --- a/tests/ansible/integration/connection_delegation/all.yml +++ b/tests/ansible/integration/connection_delegation/all.yml @@ -1,5 +1,5 @@ -- include: delegate_to_template.yml -- include: local_action.yml -- include: osa_container_standalone.yml -- include: osa_delegate_to_self.yml -- include: stack_construction.yml +- import_playbook: delegate_to_template.yml +- import_playbook: local_action.yml +#- import_playbook: osa_container_standalone.yml +#- import_playbook: osa_delegate_to_self.yml +- import_playbook: stack_construction.yml diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index bfde1265a..60a67b82f 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -11,16 +11,15 @@ - name: integration/connection_delegation/delegate_to_template.yml vars: physical_host: "cd-normal-alias" - physical_hosts: ["cd-normal-alias", "cd-normal-normal"] hosts: test-targets gather_facts: no - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - meta: end_play - when: ansible_version.full < '2.4' + when: + - ansible_version.full is version('2.4', '<', strict=True) - mitogen_get_stack: delegate_to: "{{ physical_host }}" @@ -33,7 +32,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, @@ -41,17 +40,15 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -63,25 +60,23 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-normal-alias', 'identities_only': False, 'identity_file': null, 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, - 'python_path': ["/usr/bin/python"], + 'port': '{{ expected_ssh_port }}', + 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -90,3 +85,6 @@ 'method': 'ssh', } ] + tags: + - delegate_to_template + - mitogen_only diff --git a/tests/ansible/integration/connection_delegation/local_action.yml b/tests/ansible/integration/connection_delegation/local_action.yml index 05fc3db90..1f5cd7671 100644 --- a/tests/ansible/integration/connection_delegation/local_action.yml +++ b/tests/ansible/integration/connection_delegation/local_action.yml @@ -1,10 +1,9 @@ - # issue #251: local_action with mitogen_via= builds wrong stack. -- hosts: cd-newuser-normal-normal +- name: integration/connection_delegation/local_action.yml + hosts: cd-newuser-normal-normal tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - local_action: mitogen_get_stack become: true @@ -22,7 +21,7 @@ { 'enable_lru': true, 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'python_path': ["{{ansible_playbook_python}}"], 'remote_name': null, 'password': null, @@ -33,3 +32,6 @@ 'method': 'sudo', } ] + tags: + - local_action + - mitogen_only diff --git a/tests/ansible/integration/connection_delegation/osa_container_standalone.yml b/tests/ansible/integration/connection_delegation/osa_container_standalone.yml index cf5eceaa6..df295fd04 100644 --- a/tests/ansible/integration/connection_delegation/osa_container_standalone.yml +++ b/tests/ansible/integration/connection_delegation/osa_container_standalone.yml @@ -4,8 +4,7 @@ hosts: dtc-container-1 gather_facts: false tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - mitogen_get_stack: register: out @@ -20,10 +19,14 @@ 'kind': 'lxc', 'lxc_info_path': null, 'machinectl_path': null, - 'python_path': ['/usr/bin/python'], + 'python_path': ['python3000'], 'remote_name': null, 'username': null, }, 'method': 'setns', }, ] + tags: + - mitogen_only + - osa + - osa_container_standalone diff --git a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml index 4a1fa6810..53addbfe4 100644 --- a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml +++ b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml @@ -6,8 +6,7 @@ target: osa-container-1 gather_facts: false tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - mitogen_get_stack: delegate_to: "{{target}}" @@ -24,9 +23,13 @@ 'lxc_info_path': null, 'lxc_path': null, 'machinectl_path': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'username': 'ansible-cfg-remote-user', }, 'method': 'setns', }, ] + tags: + - mitogen_only + - osa + - osa_delegate_to_self diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index ed298599a..b0475275f 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -19,20 +19,17 @@ - name: integration/connection_delegation/stack_construction.yml hosts: cd-normal tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml # used later for local_action test. - local_action: custom_python_detect_environment register: local_env + tags: + - stack_construction - hosts: cd-normal - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml - mitogen_get_stack: register: out - assert_equal: @@ -40,23 +37,25 @@ right: [ { "kwargs": { - "connect_timeout": 10, + "connect_timeout": 30, "doas_path": null, "password": null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, "username": "normal-user", }, "method": "doas", } ] + tags: + - mitogen_only + - stack_construction - hosts: cd-normal tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: delegate_to: cd-alias register: out @@ -67,25 +66,23 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, - "python_path": ["/usr/bin/python"], + 'port': '{{ expected_ssh_port }}', + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -94,13 +91,15 @@ 'method': 'ssh', }, ] + tags: + - mitogen_only + - stack_construction - hosts: cd-alias tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -110,25 +109,23 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, - "python_path": ["/usr/bin/python"], + 'port': '{{ expected_ssh_port }}', + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -137,13 +134,15 @@ 'method': 'ssh', }, ] + tags: + - mitogen_only + - stack_construction - hosts: cd-normal-normal tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -151,10 +150,10 @@ right: [ { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -164,25 +163,23 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-normal-normal', 'identities_only': False, 'identity_file': null, 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, - "python_path": ["/usr/bin/python"], + 'port': '{{ expected_ssh_port }}', + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -191,13 +188,15 @@ 'method': 'ssh', }, ] + tags: + - mitogen_only + - stack_construction - hosts: cd-normal-alias tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -207,7 +206,7 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, @@ -215,17 +214,15 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -237,25 +234,23 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-normal-alias', 'identities_only': False, 'identity_file': null, 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, - "python_path": ["/usr/bin/python"], + 'port': '{{ expected_ssh_port }}', + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -264,13 +259,14 @@ 'method': 'ssh', }, ] + tags: + - stack_construction - hosts: cd-newuser-normal-normal tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -278,10 +274,10 @@ right: [ { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -291,25 +287,23 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'cd-newuser-normal-normal', 'identities_only': False, 'identity_file': null, 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, - "python_path": ["/usr/bin/python"], + 'port': '{{ expected_ssh_port }}', + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -318,13 +312,15 @@ 'method': 'ssh', }, ] + tags: + - mitogen_only + - stack_construction - hosts: cd-newuser-normal-normal tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: delegate_to: cd-alias register: out @@ -335,25 +331,23 @@ 'kwargs': { 'check_host_keys': 'ignore', 'compression': True, - 'connect_timeout': 10, + 'connect_timeout': 30, 'hostname': 'alias-host', 'identities_only': False, 'identity_file': null, 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, - "python_path": ["/usr/bin/python"], + 'port': '{{ expected_ssh_port }}', + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -362,13 +356,14 @@ 'method': 'ssh', }, ] + tags: + - mitogen_only + - stack_construction - hosts: cd-newuser-normal-normal tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml - local_action: mitogen_get_stack register: out - assert_equal: @@ -381,13 +376,14 @@ 'method': 'local', }, ] + tags: + - mitogen_only + - stack_construction - hosts: cd-newuser-doas-normal tasks: - - meta: end_play - when: not is_mitogen - + - include_tasks: ../_mitogen_only.yml - mitogen_get_stack: register: out - assert_equal: @@ -395,10 +391,10 @@ right: [ { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -406,13 +402,16 @@ }, { 'kwargs': { - 'connect_timeout': 10, + 'connect_timeout': 30, 'doas_path': null, 'password': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'remote_name': null, 'username': 'newuser-doas-normal-user', }, 'method': 'doas', }, - ] + ] + tags: + - mitogen_only + - stack_construction diff --git a/tests/ansible/integration/connection_loader/all.yml b/tests/ansible/integration/connection_loader/all.yml index 76ffe8f4f..7a44bb2f4 100644 --- a/tests/ansible/integration/connection_loader/all.yml +++ b/tests/ansible/integration/connection_loader/all.yml @@ -1,3 +1,3 @@ -- include: local_blemished.yml -- include: paramiko_unblemished.yml -- include: ssh_blemished.yml +- import_playbook: local_blemished.yml +- import_playbook: paramiko_unblemished.yml +- import_playbook: ssh_blemished.yml diff --git a/tests/ansible/integration/connection_loader/local_blemished.yml b/tests/ansible/integration/connection_loader/local_blemished.yml index d0fcabba6..56aefcbb2 100644 --- a/tests/ansible/integration/connection_loader/local_blemished.yml +++ b/tests/ansible/integration/connection_loader/local_blemished.yml @@ -2,7 +2,6 @@ - name: integration/connection_loader/local_blemished.yml hosts: test-targets - any_errors_fatal: true tasks: - determine_strategy: @@ -12,3 +11,8 @@ - assert: that: (not not out.mitogen_loaded) == (not not is_mitogen) + fail_msg: | + out={{ out }} + tags: + - local + - local_blemished diff --git a/tests/ansible/integration/connection_loader/paramiko_unblemished.yml b/tests/ansible/integration/connection_loader/paramiko_unblemished.yml index a48bd3cad..dc9d346fd 100644 --- a/tests/ansible/integration/connection_loader/paramiko_unblemished.yml +++ b/tests/ansible/integration/connection_loader/paramiko_unblemished.yml @@ -3,7 +3,6 @@ - name: integration/connection_loader/paramiko_unblemished.yml hosts: test-targets - any_errors_fatal: true tasks: - debug: msg: "skipped for now" @@ -15,4 +14,9 @@ - assert: that: not out.mitogen_loaded + fail_msg: | + out={{ out }} when: False + tags: + - paramiko + - paramiko_unblemished diff --git a/tests/ansible/integration/connection_loader/ssh_blemished.yml b/tests/ansible/integration/connection_loader/ssh_blemished.yml index 04ada1e33..0c7cfd3fa 100644 --- a/tests/ansible/integration/connection_loader/ssh_blemished.yml +++ b/tests/ansible/integration/connection_loader/ssh_blemished.yml @@ -2,7 +2,6 @@ - name: integration/connection_loader__ssh_blemished.yml hosts: test-targets - any_errors_fatal: true tasks: - determine_strategy: @@ -12,3 +11,8 @@ - assert: that: (not not out.mitogen_loaded) == (not not is_mitogen) + fail_msg: | + out={{ out }} + tags: + - ssh + - ssh_blemished diff --git a/tests/ansible/integration/context_service/all.yml b/tests/ansible/integration/context_service/all.yml index 7770629a5..edcf8eff4 100644 --- a/tests/ansible/integration/context_service/all.yml +++ b/tests/ansible/integration/context_service/all.yml @@ -1,4 +1,4 @@ -- include: disconnect_cleanup.yml -- include: lru_one_target.yml -- include: reconnection.yml -- include: remote_name.yml +- import_playbook: disconnect_cleanup.yml +- import_playbook: lru_one_target.yml +- import_playbook: reconnection.yml +- import_playbook: remote_name.yml diff --git a/tests/ansible/integration/context_service/disconnect_cleanup.yml b/tests/ansible/integration/context_service/disconnect_cleanup.yml index d7345932a..22ba12eba 100644 --- a/tests/ansible/integration/context_service/disconnect_cleanup.yml +++ b/tests/ansible/integration/context_service/disconnect_cleanup.yml @@ -3,13 +3,12 @@ - name: integration/context_service/disconnect_cleanup.yml hosts: test-targets[0] - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - meta: end_play - when: ansible_version.full < '2.5.6' + when: + - ansible_version.full is version('2.5.6', '<', strict=True) # Start with a clean slate. - mitogen_shutdown_all: @@ -48,3 +47,6 @@ # - assert: # that: out.dump|length == play_hosts|length # just the ssh account + tags: + - disconnect_cleanup + - mitogen_only diff --git a/tests/ansible/integration/context_service/lru_one_target.yml b/tests/ansible/integration/context_service/lru_one_target.yml index 4ab5e134c..570581e30 100644 --- a/tests/ansible/integration/context_service/lru_one_target.yml +++ b/tests/ansible/integration/context_service/lru_one_target.yml @@ -2,7 +2,6 @@ - name: integration/context_service/lru_one_target.yml hosts: test-targets - any_errors_fatal: true vars: max_interps: "{{lookup('env', 'MITOGEN_MAX_INTERPRETERS')}}" ubound: "{{max_interps|int + 1}}" @@ -40,3 +39,5 @@ # that: # - first_run.results[-1].pid != second_run.results[-1].pid # when: is_mitogen + tags: + - lru_one_target diff --git a/tests/ansible/integration/context_service/reconnection.yml b/tests/ansible/integration/context_service/reconnection.yml index eed1dfdb2..6fb605a22 100644 --- a/tests/ansible/integration/context_service/reconnection.yml +++ b/tests/ansible/integration/context_service/reconnection.yml @@ -3,7 +3,6 @@ - name: integration/context_service/reconnection.yml hosts: test-targets - any_errors_fatal: true tasks: - mitogen_shutdown_all: @@ -14,7 +13,8 @@ custom_python_detect_environment: register: old_become_env - - become: true + - name: Kill ssh process + become: true shell: | bash -c "( sleep 3; kill -9 {{ssh_account_env.pid}}; ) & disown" @@ -31,3 +31,8 @@ - assert: that: - old_become_env.pid != new_become_env.pid + fail_msg: | + old_become_env={{ old_become_env }} + new_become_env={{ new_become_env }} + tags: + - reconnection diff --git a/tests/ansible/integration/context_service/remote_name.yml b/tests/ansible/integration/context_service/remote_name.yml index d7116ec10..dbc02fab3 100644 --- a/tests/ansible/integration/context_service/remote_name.yml +++ b/tests/ansible/integration/context_service/remote_name.yml @@ -2,24 +2,26 @@ - name: integration/context_service/remote_name.yml hosts: test-targets[0] - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml # Too much hassle to make this work for OSX - meta: end_play when: ansible_system != 'Linux' - - shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' + - name: Get cmdline + shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' register: out - debug: var=out - assert: that: - out.stdout is match('.*python([0-9.]+)?\(mitogen:[a-z]+@[^:]+:[0-9]+\)') + fail_msg: | + out={{ out }} - - shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' + - name: Get cmdline, with mitogen_mask_remote_name + shell: 'cat /proc/$PPID/cmdline | tr \\0 \\n' register: out vars: mitogen_mask_remote_name: true @@ -28,4 +30,8 @@ - assert: that: - out.stdout is match('.*python([0-9.]+)?\(mitogen:ansible\)') - + fail_msg: | + out={{ out }} + tags: + - mitogen_only + - remote_name diff --git a/tests/ansible/integration/glibc_caches/all.yml b/tests/ansible/integration/glibc_caches/all.yml index 8cff4ea89..7d524540f 100644 --- a/tests/ansible/integration/glibc_caches/all.yml +++ b/tests/ansible/integration/glibc_caches/all.yml @@ -1,2 +1,2 @@ -- include: resolv_conf.yml +- import_playbook: resolv_conf.yml diff --git a/tests/ansible/integration/glibc_caches/resolv_conf.yml b/tests/ansible/integration/glibc_caches/resolv_conf.yml index da78c3086..4468884a5 100644 --- a/tests/ansible/integration/glibc_caches/resolv_conf.yml +++ b/tests/ansible/integration/glibc_caches/resolv_conf.yml @@ -8,42 +8,48 @@ vars: ansible_become_pass: has_sudo_pubkey_password tasks: - - mitogen_test_gethostbyname: name: www.google.com register: out - when: | - ansible_virtualization_type == "docker" and - ansible_python_version > "2.5" - - - shell: cp /etc/resolv.conf /tmp/resolv.conf - when: | - ansible_virtualization_type == "docker" and - ansible_python_version > "2.5" - - - shell: echo > /etc/resolv.conf - when: | - ansible_virtualization_type == "docker" and - ansible_python_version > "2.5" + when: + - ansible_facts.virtualization_type == "docker" + - ansible_facts.python.version_info[:2] >= [2, 5] + + - name: Backup resolv.conf + shell: cp /etc/resolv.conf /tmp/resolv.conf + when: + - ansible_facts.virtualization_type == "docker" + - ansible_facts.python.version_info[:2] >= [2, 5] + + - name: Truncate resolv.conf + shell: echo > /etc/resolv.conf + when: + - ansible_facts.virtualization_type == "docker" + - ansible_facts.python.version_info[:2] >= [2, 5] - mitogen_test_gethostbyname: name: www.google.com register: out ignore_errors: true - when: | - ansible_virtualization_type == "docker" and - ansible_python_version > "2.5" + when: + - ansible_facts.virtualization_type == "docker" + - ansible_facts.python.version_info[:2] >= [2, 5] - - shell: cat /tmp/resolv.conf > /etc/resolv.conf - when: | - ansible_virtualization_type == "docker" and - ansible_python_version > "2.5" + - name: Restore resolv.conf + shell: cat /tmp/resolv.conf > /etc/resolv.conf + when: + - ansible_facts.virtualization_type == "docker" + - ansible_facts.python.version_info[:2] >= [2, 5] - assert: that: - out.failed - '"Name or service not known" in out.msg or "Temporary failure in name resolution" in out.msg' - when: | - ansible_virtualization_type == "docker" and - ansible_python_version > "2.5" + fail_msg: | + out={{ out }} + when: + - ansible_facts.virtualization_type == "docker" + - ansible_facts.python.version_info[:2] >= [2, 5] + tags: + - resolv_conf diff --git a/tests/ansible/integration/interpreter_discovery/all.yml b/tests/ansible/integration/interpreter_discovery/all.yml index 403fd7615..56fbc622e 100644 --- a/tests/ansible/integration/interpreter_discovery/all.yml +++ b/tests/ansible/integration/interpreter_discovery/all.yml @@ -1,2 +1,2 @@ -- include: complex_args.yml -- include: ansible_2_8_tests.yml +- import_playbook: complex_args.yml +- import_playbook: ansible_2_8_tests.yml diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index a4aec22f0..eddca199b 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -3,8 +3,56 @@ - name: integration/interpreter_discovery/ansible_2_8_tests.yml hosts: test-targets - any_errors_fatal: true gather_facts: true + vars: + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python + 'NA': /usr/bin/python # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python3.9 + 'NA': /usr/bin/python3.9 # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python3.9 # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17: + debian: + '10': /usr/bin/python3.7 + '11': /usr/bin/python3.9 + 'bullseye/sid': /usr/bin/python3.9 + ubuntu: + '20': /usr/bin/python3.8 + + discovered_interpreter_expected: >- + {%- if ansible_version.full is version('2.12', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12[distro][distro_major] }} + {%- elif ansible_version.full is version('2.17', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16[distro][distro_major] }} + {%- else -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17[distro][distro_major] }} + {%- endif -%} tasks: - name: can only run these tests on ansible >= 2.8.0 block: @@ -19,9 +67,9 @@ - name: snag some facts to validate for later set_fact: - distro: '{{ ansible_distribution | default("unknown") | lower }}' - distro_version: '{{ ansible_distribution_version | default("unknown") }}' - os_family: '{{ ansible_os_family | default("unknown") }}' + distro: '{{ ansible_facts.distribution | lower }}' + distro_major: '{{ ansible_facts.distribution_major_version }}' + system: '{{ ansible_facts.system }}' - name: test that python discovery is working and that fact persistence makes it only run once block: @@ -38,6 +86,7 @@ vars: ansible_python_interpreter: auto test_echo_module: + facts_copy: "{{ ansible_facts }}" register: echoout # can't test this assertion: @@ -45,10 +94,24 @@ # because Mitogen's ansible_python_interpreter is a connection-layer configurable that # "must be extracted during each task execution to form the complete connection-layer configuration". # Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered - - assert: + - name: assert discovered python matches invoked python + assert: that: - auto_out.ansible_facts.discovered_interpreter_python is defined - - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python + - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen + - echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen + fail_msg: + - "auto_out: {{ auto_out }}" + - "echoout: {{ echoout }}" + when: + # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always + # reports the same path. I can't reach via symlinks. + # >>> sys.executable + # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python + - is_mitogen + or echoout.running_python.sys.version_info.major != 2 + or not (echoout.running_python.sys.platform == "darwin" + and echoout.running_python.platform.release.major == 20) - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result @@ -62,14 +125,35 @@ ping: register: legacy - - name: check for dep warning (only on platforms where auto result is not /usr/bin/python and legacy is) + - name: check for dep warning (only on platforms where auto result is not /usr/bin/python and legacy is) for ansible 2.8-2.11 + # from ansible 2.12 on this changed + # - https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_5.html#python-interpreter-discovery + # - https://docs.ansible.com/ansible/latest/reference_appendices/interpreter_discovery.html + # default discovery method is now auto and will default to python3 + # and the message changed from a deprecation warning to a real warning that can not be suppressed by + # using deprecation_warnings=False assert: that: - legacy.deprecations | default([]) | length > 0 + fail_msg: | + legacy={{ legacy }} # only check for a dep warning if legacy returned /usr/bin/python and auto didn't - when: legacy.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and - auto_out.ansible_facts.discovered_interpreter_python != '/usr/bin/python' + when: + - legacy.ansible_facts.discovered_interpreter_python == '/usr/bin/python' + - auto_out.ansible_facts.discovered_interpreter_python != '/usr/bin/python' + - ansible_version.full is version_compare('2.12.0', '<', strict=True) + - name: check for warning (only on platforms where auto result is not /usr/bin/python and legacy is) from ansible 2.12 on + assert: + that: + - legacy.warnings | default([]) | length > 0 + fail_msg: | + legacy={{ legacy }} + # only check for a warning if legacy returned /usr/bin/python and auto didn't + when: + - legacy.ansible_facts.discovered_interpreter_python == '/usr/bin/python' + - auto_out.ansible_facts.discovered_interpreter_python != '/usr/bin/python' + - ansible_version.full is version_compare('2.12.0', '>=', strict=True) - name: test that auto_silent never warns and got the same answer as auto block: @@ -86,6 +170,8 @@ that: - auto_silent_out.warnings is not defined - auto_silent_out.ansible_facts.discovered_interpreter_python == auto_out.ansible_facts.discovered_interpreter_python + fail_msg: | + auto_silent_out={{ auto_silent_out }} - name: test that auto_legacy_silent never warns and got the same answer as auto_legacy @@ -103,11 +189,14 @@ that: - legacy_silent.warnings is not defined - legacy_silent.ansible_facts.discovered_interpreter_python == legacy.ansible_facts.discovered_interpreter_python + fail_msg: | + legacy_silent={{ legacy_silent }} - name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter block: - test_echo_module: - facts: + facts_copy: "{{ ansible_facts }}" + facts_to_override: ansible_discovered_interpreter_bogus: from module discovered_interpreter_bogus: from_module ansible_bogus_interpreter: from_module @@ -123,36 +212,23 @@ - ansible_facts['ansible_bogus_interpreter'] | default('nope') == 'nope' - ansible_facts['discovered_interpreter_bogus'] | default('nope') == 'nope' - - name: fedora assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' - when: distro == 'fedora' and distro_version is version('23', '>=') - - - name: rhel assertions - assert: - that: - # rhel 6/7 - - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and distro_version is version('8','<')) or distro_version is version('8','>=') - # rhel 8+ - - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python' and distro_version is version('8','>=')) or distro_version is version('8','<') - when: distro in ('redhat', 'centos') - - - name: ubuntu assertions - assert: - that: - # ubuntu < 16 - - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and distro_version is version('16.04','<')) or distro_version is version('16.04','>=') - # ubuntu >= 16 - - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' and distro_version is version('16.04','>=')) or distro_version is version('16.04','<') - when: distro == 'ubuntu' - - - name: mac assertions + - name: Check discovered interpreter matches expected assert: that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - when: os_family == 'Darwin' + - auto_out.ansible_facts.discovered_interpreter_python == discovered_interpreter_expected + fail_msg: | + distro={{ distro }} + distro_major= {{ distro_major }} + system={{ system }} + auto_out={{ auto_out }} + discovered_interpreter_expected={{ discovered_interpreter_expected }} + ansible_version.full={{ ansible_version.full }} + when: + - system in ['Linux'] always: - meta: clear_facts - when: ansible_version.full is version_compare('2.8.0', '>=') + when: + - ansible_version.full is version_compare('2.8.0', '>=', strict=True) + tags: + - ansible_2_8_tests diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index 6c53e9e52..f9770876c 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -3,9 +3,19 @@ - name: integration/interpreter_discovery/complex_args.yml hosts: test-targets - any_errors_fatal: true gather_facts: true + environment: + http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}" + https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" + no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) + - name: create temp file to source file: path: /tmp/fake @@ -22,35 +32,27 @@ # special_python: source /tmp/fake && python - name: set python using sourced file set_fact: - special_python: source /tmp/fake || true && python + special_python: "source /tmp/fake || true && {{ ansible_facts.python.executable }}" - name: run get_url with specially-sourced python - get_url: - url: https://google.com - dest: "/tmp/" - mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no + uri: + # Plain http for wider Ansible & Python version compatibility + url: http://www.gstatic.com/generate_204 + status_code: [204] vars: ansible_python_interpreter: "{{ special_python }}" - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" - name: run get_url with specially-sourced python including jinja - get_url: - url: https://google.com - dest: "/tmp/" - mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no + uri: + # Plain http for wider Ansible & Python version compatibility + url: http://www.gstatic.com/generate_204 + status_code: [204] vars: ansible_python_interpreter: > {% if "1" == "1" %} {{ special_python }} {% else %} - python + python {% endif %} - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" + tags: + - complex_args diff --git a/tests/ansible/integration/local/all.yml b/tests/ansible/integration/local/all.yml index 5f8b4dd49..383a9108a 100644 --- a/tests/ansible/integration/local/all.yml +++ b/tests/ansible/integration/local/all.yml @@ -1,4 +1,4 @@ -- include: cwd_preserved.yml -- include: env_preserved.yml +- import_playbook: cwd_preserved.yml +- import_playbook: env_preserved.yml diff --git a/tests/ansible/integration/local/cwd_preserved.yml b/tests/ansible/integration/local/cwd_preserved.yml index e5c0f7a4d..0b1ddef40 100644 --- a/tests/ansible/integration/local/cwd_preserved.yml +++ b/tests/ansible/integration/local/cwd_preserved.yml @@ -5,10 +5,10 @@ # https://github.com/ansible/ansible/issues/14489 - name: integration/local/cwd_preserved.yml - any_errors_fatal: true hosts: test-targets tasks: - - connection: local + - name: Get local cwd + connection: local command: pwd register: pwd @@ -19,4 +19,7 @@ - assert: that: stat.stat.exists - + fail_msg: | + stat={{ stat }} + tags: + - cwd_prseserved diff --git a/tests/ansible/integration/local/env_preserved.yml b/tests/ansible/integration/local/env_preserved.yml index 1375391e6..e074a4904 100644 --- a/tests/ansible/integration/local/env_preserved.yml +++ b/tests/ansible/integration/local/env_preserved.yml @@ -1,8 +1,10 @@ - # Execution environment should be that of WorkerProcess -- # https://github.com/dw/mitogen/issues/297 -- hosts: localhost +- name: integration/local/env_preserved.yml + hosts: localhost connection: local tasks: - shell: "env | grep EVIL_VARS_PLUGIN" + tags: + - env_preserved diff --git a/tests/ansible/integration/module_utils/adjacent_to_playbook.yml b/tests/ansible/integration/module_utils/adjacent_to_playbook.yml index 63bd90b23..42debea8c 100644 --- a/tests/ansible/integration/module_utils/adjacent_to_playbook.yml +++ b/tests/ansible/integration/module_utils/adjacent_to_playbook.yml @@ -3,7 +3,6 @@ - name: integration/module_utils/adjacent_to_playbook.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_external_module: @@ -13,4 +12,7 @@ that: - out.external1_path == "ansible/integration/module_utils/module_utils/external1.py" - out.external2_path == "ansible/lib/module_utils/external2.py" - + fail_msg: | + out={{ out }} + tags: + - adjacent_to_playbook diff --git a/tests/ansible/integration/module_utils/adjacent_to_role.yml b/tests/ansible/integration/module_utils/adjacent_to_role.yml index 3fd3b1e67..93499790a 100644 --- a/tests/ansible/integration/module_utils/adjacent_to_role.yml +++ b/tests/ansible/integration/module_utils/adjacent_to_role.yml @@ -1,8 +1,9 @@ # external2 is loaded from config path. # external1 is loaded from integration/module_utils/roles/modrole/module_utils/.. -- name: integration/module_utils/adjacent_to_playbook.yml +- name: integration/module_utils/adjacent_to_role.yml hosts: test-targets - any_errors_fatal: true roles: - modrole + tags: + - adjacent_to_playbook diff --git a/tests/ansible/integration/module_utils/all.yml b/tests/ansible/integration/module_utils/all.yml index b68e2ee3a..c8b8f2fbd 100644 --- a/tests/ansible/integration/module_utils/all.yml +++ b/tests/ansible/integration/module_utils/all.yml @@ -1,6 +1,6 @@ -#- include: from_config_path.yml -#- include: from_config_path_pkg.yml -#- include: adjacent_to_playbook.yml -- include: adjacent_to_role.yml -#- include: overrides_builtin.yml +#- import_playbook: from_config_path.yml +#- import_playbook: from_config_path_pkg.yml +#- import_playbook: adjacent_to_playbook.yml +- import_playbook: adjacent_to_role.yml +#- import_playbook: overrides_builtin.yml diff --git a/tests/ansible/integration/module_utils/from_config_path.yml b/tests/ansible/integration/module_utils/from_config_path.yml index e469fe32c..e6817e2ee 100644 --- a/tests/ansible/integration/module_utils/from_config_path.yml +++ b/tests/ansible/integration/module_utils/from_config_path.yml @@ -2,7 +2,6 @@ - name: integration/module_utils/from_config_path.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_external_module: @@ -12,4 +11,7 @@ that: - out.external1_path == "ansible/lib/module_utils/external1.py" - out.external2_path == "ansible/lib/module_utils/external2.py" - + fail_msg: | + out={{ out }} + tags: + - from_config_path diff --git a/tests/ansible/integration/module_utils/from_config_path_pkg.yml b/tests/ansible/integration/module_utils/from_config_path_pkg.yml index 5db3d124e..2742f0c46 100644 --- a/tests/ansible/integration/module_utils/from_config_path_pkg.yml +++ b/tests/ansible/integration/module_utils/from_config_path_pkg.yml @@ -2,7 +2,6 @@ - name: integration/module_utils/from_config_path.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_external_pkg: @@ -11,4 +10,7 @@ - assert: that: - out.extmod_path == "ansible/lib/module_utils/externalpkg/extmod.py" - + fail_msg: | + out={{ out }} + tags: + - from_config_path diff --git a/tests/ansible/integration/module_utils/overrides_builtin.yml b/tests/ansible/integration/module_utils/overrides_builtin.yml index 635876f16..9d841e2ed 100644 --- a/tests/ansible/integration/module_utils/overrides_builtin.yml +++ b/tests/ansible/integration/module_utils/overrides_builtin.yml @@ -1,6 +1,7 @@ - name: integration/module_utils/overrides_builtin.yml hosts: test-targets - any_errors_fatal: true roles: - overrides_modrole + tags: + - overrides_builtin diff --git a/tests/ansible/integration/module_utils/roles/modrole/module_utils/external2.py b/tests/ansible/integration/module_utils/roles/modrole/module_utils/external2.py index a00278a0d..4ea8bc90b 100644 --- a/tests/ansible/integration/module_utils/roles/modrole/module_utils/external2.py +++ b/tests/ansible/integration/module_utils/roles/modrole/module_utils/external2.py @@ -1,3 +1,2 @@ - def path(): return "integration/module_utils/roles/modrole/module_utils/external2.py" diff --git a/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml b/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml index 2c7c3372d..5450028b3 100644 --- a/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml +++ b/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml @@ -7,3 +7,5 @@ that: - out.external3_path == "integration/module_utils/roles/modrole/module_utils/external3.py" - out.external2_path == "integration/module_utils/roles/modrole/module_utils/external2.py" + fail_msg: | + out={{ out }} diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py index f0e9439b2..688fc45ab 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py @@ -1,11 +1,11 @@ #!/usr/bin/python import json -from ansible.module_utils.basic import path +import ansible.module_utils.basic def main(): print(json.dumps({ - 'path': path() + 'path': ansible.module_utils.basic.path() })) if __name__ == '__main__': diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml b/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml index 6ef4703a3..fa20447f2 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml @@ -6,3 +6,5 @@ - assert: that: - out.path == "ansible/integration/module_utils/roles/override_modrole/module_utils/known_hosts.py" + fail_msg: | + out={{ out }} diff --git a/tests/ansible/integration/playbook_semantics/all.yml b/tests/ansible/integration/playbook_semantics/all.yml index ec7a9a07a..6c8dd0658 100644 --- a/tests/ansible/integration/playbook_semantics/all.yml +++ b/tests/ansible/integration/playbook_semantics/all.yml @@ -1,4 +1,4 @@ -- include: become_flags.yml -- include: delegate_to.yml -- include: environment.yml -- include: with_items.yml +- import_playbook: become_flags.yml +- import_playbook: delegate_to.yml +- import_playbook: environment.yml +- import_playbook: with_items.yml diff --git a/tests/ansible/integration/playbook_semantics/become_flags.yml b/tests/ansible/integration/playbook_semantics/become_flags.yml index f2ab0b5d6..466fbbf5c 100644 --- a/tests/ansible/integration/playbook_semantics/become_flags.yml +++ b/tests/ansible/integration/playbook_semantics/become_flags.yml @@ -4,7 +4,6 @@ - name: integration/playbook_semantics/become_flags.yml hosts: test-targets - any_errors_fatal: true tasks: - name: "without -E" @@ -14,9 +13,12 @@ - assert: that: "out.stdout == ''" + fail_msg: | + out={{ out }} + tags: + - become_flags - hosts: test-targets - any_errors_fatal: true become_flags: -E tasks: - name: "with -E" @@ -28,3 +30,7 @@ - assert: that: "out2.stdout == '2'" + fail_msg: | + out={{ out }} + tags: + - become_flags diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml index 23b7168ce..61b4d03df 100644 --- a/tests/ansible/integration/playbook_semantics/delegate_to.yml +++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml @@ -1,24 +1,27 @@ - name: integration/playbook_semantics/delegate_to.yml hosts: test-targets - any_errors_fatal: true + vars: + local_path: "/tmp/delegate_to.{{ inventory_hostname }}.txt" tasks: # # delegate_to, no sudo # - name: "delegate_to, no sudo" copy: - dest: /tmp/delegate_to.yml.txt + dest: "{{ local_path }}" content: "Hello, world." - register: out + mode: u=rw,go=r delegate_to: localhost - name: "delegate_to, no sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'" + that: + - lookup('file', local_path) == 'Hello, world.' + fail_msg: "{{ lookup('file', local_path) }}" - name: "delegate_to, no sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent delegate_to: localhost @@ -28,18 +31,20 @@ # - name: "connection:local, no sudo" copy: - dest: /tmp/delegate_to.yml.txt + dest: "{{ local_path }}" content: "Hello, world." - register: out + mode: u=rw,go=r connection: local - name: "connection:local, no sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'" + that: + - lookup('file', local_path) == 'Hello, world.' + fail_msg: "{{ lookup('file', local_path) }}" - name: "connection:local, no sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent connection: local @@ -48,7 +53,10 @@ # delegate_to, sudo # - name: "delegate_to, sudo" - shell: whoami > /tmp/delegate_to.yml.txt + shell: | + whoami > "{{ local_path }}" + args: + creates: "{{ local_path }}" delegate_to: localhost become: true tags: @@ -56,13 +64,15 @@ - name: "delegate_to, sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + that: + - lookup('file', local_path) == 'root' + fail_msg: "{{ lookup('file', local_path) }}" tags: - requires_local_sudo - name: "delegate_to, sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent delegate_to: localhost become: true @@ -74,7 +84,10 @@ # connection:local, sudo # - name: "connection:local, sudo" - shell: whoami > /tmp/delegate_to.yml.txt + shell: | + whoami > "{{ local_path }}" + args: + creates: "{{ local_path }}" connection: local become: true tags: @@ -82,15 +95,19 @@ - name: "connection:local, sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + that: + - lookup('file', local_path) == 'root' + fail_msg: "{{ lookup('file', local_path) }}" tags: - requires_local_sudo - name: "connection:local, sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent connection: local become: true tags: - requires_local_sudo + tags: + - delegate_to diff --git a/tests/ansible/integration/playbook_semantics/environment.yml b/tests/ansible/integration/playbook_semantics/environment.yml index 1ac7f71d6..31c8f495a 100644 --- a/tests/ansible/integration/playbook_semantics/environment.yml +++ b/tests/ansible/integration/playbook_semantics/environment.yml @@ -2,12 +2,16 @@ - name: integration/playbook_semantics/environment.yml hosts: test-targets - any_errors_fatal: true tasks: - - shell: echo $SOME_ENV + - name: Echo $SOME_ENV + shell: echo $SOME_ENV environment: SOME_ENV: 123 register: result - assert: that: "result.stdout == '123'" + fail_msg: | + result={{ result }} + tags: + - environment diff --git a/tests/ansible/integration/playbook_semantics/with_items.yml b/tests/ansible/integration/playbook_semantics/with_items.yml index 9e64c1ba3..a2ef015cc 100644 --- a/tests/ansible/integration/playbook_semantics/with_items.yml +++ b/tests/ansible/integration/playbook_semantics/with_items.yml @@ -3,29 +3,36 @@ - name: integration/playbook_semantics/with_items.yml hosts: test-targets - any_errors_fatal: true + gather_facts: true tasks: + - block: + - name: Spin up a few interpreters + become: true + vars: + ansible_become_user: "mitogen__user{{ item }}" + command: + cmd: whoami + with_sequence: start=1 end=3 + register: first_run + changed_when: false - # TODO: https://github.com/dw/mitogen/issues/692 - # - name: Spin up a few interpreters - # shell: whoami - # become: true - # vars: - # ansible_become_user: "mitogen__user{{item}}" - # with_sequence: start=1 end=3 - # register: first_run + - name: Reuse them + become: true + vars: + ansible_become_user: "mitogen__user{{ item }}" + command: + cmd: whoami + with_sequence: start=1 end=3 + register: second_run + changed_when: false - # - name: Reuse them - # shell: whoami - # become: true - # vars: - # ansible_become_user: "mitogen__user{{item}}" - # with_sequence: start=1 end=3 - # register: second_run - - # - name: Verify first and second run matches expected username. - # assert: - # that: - # - first_run.results[item|int].stdout == ("mitogen__user%d" % (item|int + 1)) - # - first_run.results[item|int].stdout == second_run.results[item|int].stdout - # with_sequence: start=0 end=2 + - name: Verify first and second run matches expected username. + vars: + user_expected: "mitogen__user{{ item | int + 1 }}" + assert: + that: + - first_run.results[item | int].stdout == user_expected + - second_run.results[item | int].stdout == user_expected + with_sequence: start=0 end=2 + when: + - become_unpriv_available diff --git a/tests/ansible/integration/process/all.yml b/tests/ansible/integration/process/all.yml index a309113ad..c50d6bd56 100644 --- a/tests/ansible/integration/process/all.yml +++ b/tests/ansible/integration/process/all.yml @@ -1 +1 @@ -- include: "unix_socket_cleanup.yml" +- import_playbook: "unix_socket_cleanup.yml" diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index 11a0efe13..eb6720d31 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -1,5 +1,5 @@ - -- hosts: test-targets[0] +- name: integration/process/unix_socket_cleanup.yml + hosts: test-targets[0] tasks: - mitogen_action_script: script: | @@ -9,8 +9,13 @@ - shell: > ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -c local -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" + ansible -m shell -c local -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets args: chdir: ../.. register: out @@ -26,3 +31,5 @@ - assert_equal: left: socks right: socks2 + tags: + - custom_python_new_style_module diff --git a/tests/ansible/integration/runner/_etc_environment_global.yml b/tests/ansible/integration/runner/_etc_environment_global.yml index 2d22b9520..5446b427e 100644 --- a/tests/ansible/integration/runner/_etc_environment_global.yml +++ b/tests/ansible/integration/runner/_etc_environment_global.yml @@ -1,17 +1,22 @@ # /etc/environment -- file: +- name: Remove /etc/environment + file: path: /etc/environment state: absent become: true -- shell: echo $MAGIC_ETC_ENV +- name: Check MAGIC_ETC_ENV without + shell: echo $MAGIC_ETC_ENV register: echo - assert: that: echo.stdout == "" + fail_msg: | + echo={{ echo }} -- copy: +- name: Create /etc/environment + copy: dest: /etc/environment content: | MAGIC_ETC_ENV=555 @@ -22,13 +27,17 @@ #- mitogen_shutdown_all: #when: not is_mitogen -- shell: echo $MAGIC_ETC_ENV +- name: Check MAGIC_ETC_ENV with + shell: echo $MAGIC_ETC_ENV register: echo - assert: that: echo.stdout == "555" + fail_msg: | + echo={{ echo }} -- file: +- name: Cleanup /etc/environment + file: path: /etc/environment state: absent become: true @@ -38,8 +47,11 @@ - mitogen_shutdown_all: when: not is_mitogen -- shell: echo $MAGIC_ETC_ENV +- name: Check MAGIC_ETC_ENV without+shutdown + shell: echo $MAGIC_ETC_ENV register: echo - assert: that: echo.stdout == "" + fail_msg: | + echo={{ echo }} diff --git a/tests/ansible/integration/runner/_etc_environment_user.yml b/tests/ansible/integration/runner/_etc_environment_user.yml index ca1dc5cc7..9a4abe97e 100644 --- a/tests/ansible/integration/runner/_etc_environment_user.yml +++ b/tests/ansible/integration/runner/_etc_environment_user.yml @@ -1,6 +1,7 @@ # ~/.pam_environment -- file: +- name: Remove pam_environment + file: path: ~/.pam_environment state: absent @@ -9,8 +10,11 @@ - assert: that: echo.stdout == "" + fail_msg: | + echo={{ echo }} -- copy: +- name: Copy pam_environment + copy: dest: ~/.pam_environment content: | MAGIC_PAM_ENV=321 @@ -20,8 +24,11 @@ - assert: that: echo.stdout == "321" + fail_msg: | + echo={{ echo }} -- file: +- name: Cleanup pam_environment + file: path: ~/.pam_environment state: absent @@ -30,3 +37,5 @@ - assert: that: echo.stdout == "" + fail_msg: | + echo={{ echo }} diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml index 19586547a..5de9fb270 100644 --- a/tests/ansible/integration/runner/all.yml +++ b/tests/ansible/integration/runner/all.yml @@ -1,24 +1,24 @@ -- include: atexit.yml -- include: builtin_command_module.yml -- include: crashy_new_style_module.yml -- include: custom_bash_hashbang_argument.yml -- include: custom_bash_old_style_module.yml -- include: custom_bash_want_json_module.yml -- include: custom_binary_producing_json.yml -- include: custom_binary_producing_junk.yml -- include: custom_binary_single_null.yml -- include: custom_perl_json_args_module.yml -- include: custom_perl_want_json_module.yml -- include: custom_python_json_args_module.yml -- include: custom_python_new_style_missing_interpreter.yml -- include: custom_python_new_style_module.yml -- include: custom_python_prehistoric_module.yml -- include: custom_python_want_json_module.yml -- include: custom_script_interpreter.yml -- include: environment_isolation.yml +- import_playbook: atexit.yml +- import_playbook: builtin_command_module.yml +- import_playbook: crashy_new_style_module.yml +- import_playbook: custom_bash_hashbang_argument.yml +- import_playbook: custom_bash_old_style_module.yml +- import_playbook: custom_bash_want_json_module.yml +- import_playbook: custom_binary_producing_json.yml +- import_playbook: custom_binary_producing_junk.yml +- import_playbook: custom_binary_single_null.yml +- import_playbook: custom_perl_json_args_module.yml +- import_playbook: custom_perl_want_json_module.yml +- import_playbook: custom_python_json_args_module.yml +- import_playbook: custom_python_new_style_missing_interpreter.yml +- import_playbook: custom_python_new_style_module.yml +- import_playbook: custom_python_prehistoric_module.yml +- import_playbook: custom_python_want_json_module.yml +- import_playbook: custom_script_interpreter.yml +- import_playbook: environment_isolation.yml # I hate this test. I hope it dies, it has caused nothing but misery and suffering -#- include: etc_environment.yml -- include: forking_active.yml -- include: forking_correct_parent.yml -- include: forking_inactive.yml -- include: missing_module.yml +#- import_playbook: etc_environment.yml +- import_playbook: forking_active.yml +- import_playbook: forking_correct_parent.yml +- import_playbook: forking_inactive.yml +- import_playbook: missing_module.yml diff --git a/tests/ansible/integration/runner/atexit.yml b/tests/ansible/integration/runner/atexit.yml index 65d27d596..144cfd85c 100644 --- a/tests/ansible/integration/runner/atexit.yml +++ b/tests/ansible/integration/runner/atexit.yml @@ -9,23 +9,23 @@ vars: path: /tmp/atexit-should-delete-this tasks: - - # - # Verify a run with a healthy atexit handler. Broken handlers cause an - # exception to be raised. - # - - - custom_python_run_script: + - name: Verify a run with a healthy atexit handler + custom_python_run_script: script: | - import atexit, shutil + import atexit, os, shutil path = '{{path}}' os.mkdir(path, int('777', 8)) atexit.register(shutil.rmtree, path) - - stat: + - name: Stat atexit file + stat: path: "{{path}}" register: out - assert: that: - not out.stat.exists + fail_msg: | + out={{ out }} + tags: + - atexit diff --git a/tests/ansible/integration/runner/builtin_command_module.yml b/tests/ansible/integration/runner/builtin_command_module.yml index 0bc5bd34d..0de436488 100644 --- a/tests/ansible/integration/runner/builtin_command_module.yml +++ b/tests/ansible/integration/runner/builtin_command_module.yml @@ -1,10 +1,10 @@ - name: integration/runner/builtin_command_module.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - - command: hostname + - name: Run hostname + command: hostname with_sequence: start=1 end={{end|default(1)}} register: out @@ -16,3 +16,7 @@ out.results[0].item == '1' and out.results[0].rc == 0 and (out.results[0].stdout == ansible_nodename) + fail_msg: | + out={{ out }} + tags: + - builtin_command_module diff --git a/tests/ansible/integration/runner/crashy_new_style_module.yml b/tests/ansible/integration/runner/crashy_new_style_module.yml index 73bac1f9b..3fb1a722f 100644 --- a/tests/ansible/integration/runner/crashy_new_style_module.yml +++ b/tests/ansible/integration/runner/crashy_new_style_module.yml @@ -8,18 +8,21 @@ register: out ignore_errors: true - - assert: + - name: Check error report + vars: + msg_pattern: "MODULE FAILURE(?:\nSee stdout/stderr for the exact error)?" + # (?s) -> . matches any character, even newlines + tb_pattern: "(?s)Traceback \\(most recent call last\\).+NameError: name 'kaboom' is not defined" + assert: that: - not out.changed - out.rc == 1 - # ansible/62d8c8fde6a76d9c567ded381e9b34dad69afcd6 - - | - (ansible_version.full is version('2.7', '<') and out.msg == "MODULE FAILURE") or - (ansible_version.full is version('2.7', '>=') and - out.msg == ( - "MODULE FAILURE\n" + - "See stdout/stderr for the exact error" - )) - - out.module_stdout == "" - - "'Traceback (most recent call last)' in out.module_stderr" - - "\"NameError: name 'kaboom' is not defined\" in out.module_stderr" + # https://github.com/ansible/ansible/commit/62d8c8fde6a76d9c567ded381e9b34dad69afcd6 + - out.msg is match(msg_pattern) + - (out.module_stdout == "" and out.module_stderr is search(tb_pattern)) + or + (out.module_stdout is search(tb_pattern) and out.module_stderr is match("Shared connection to localhost closed.")) + fail_msg: | + out={{ out }} + tags: + - crashy_new_style_module diff --git a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml index f02b8419c..34a60e612 100644 --- a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml +++ b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml @@ -1,8 +1,13 @@ # https://github.com/dw/mitogen/issues/291 - name: integration/runner/custom_bash_hashbang_argument.yml hosts: test-targets - any_errors_fatal: true tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) - custom_bash_old_style_module: foo: true @@ -17,3 +22,7 @@ (not out.results[0].changed) and out.results[0].msg == 'Here is my input' and out.results[0].run_via_env == "yes" + fail_msg: | + out={{ out }} + tags: + - custom_bash_hashbang_argument diff --git a/tests/ansible/integration/runner/custom_bash_old_style_module.yml b/tests/ansible/integration/runner/custom_bash_old_style_module.yml index ff9636657..ce7a8a882 100644 --- a/tests/ansible/integration/runner/custom_bash_old_style_module.yml +++ b/tests/ansible/integration/runner/custom_bash_old_style_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_bash_old_style_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_bash_old_style_module: @@ -13,3 +12,7 @@ (not out.changed) and (not out.results[0].changed) and out.results[0].msg == 'Here is my input' + fail_msg: | + out={{ out }} + tags: + - custom_bash_old_style_module diff --git a/tests/ansible/integration/runner/custom_bash_want_json_module.yml b/tests/ansible/integration/runner/custom_bash_want_json_module.yml index 075c95b2b..5328c0ccf 100644 --- a/tests/ansible/integration/runner/custom_bash_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_bash_want_json_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_bash_want_json_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_bash_want_json_module: foo: true @@ -12,3 +11,7 @@ (not out.changed) and (not out.results[0].changed) and out.results[0].msg == 'Here is my input' + fail_msg: | + out={{ out }} + tags: + - custom_bash_want_json_module diff --git a/tests/ansible/integration/runner/custom_binary_producing_json.yml b/tests/ansible/integration/runner/custom_binary_producing_json.yml index a3b8a2243..183982aa2 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_json.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_json.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_binary_producing_json.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - block: @@ -24,3 +23,7 @@ out.changed and out.results[0].changed and out.results[0].msg == 'Hello, world.' + fail_msg: | + out={{ out }} + tags: + - custom_binary_producing_json diff --git a/tests/ansible/integration/runner/custom_binary_producing_junk.yml b/tests/ansible/integration/runner/custom_binary_producing_junk.yml index b9cfb6b4e..2a05fb750 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_junk.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_junk.yml @@ -19,10 +19,11 @@ register: out_linux - set_fact: out={{out_linux}} when: ansible_system == "Linux" + tags: + - custom_binary_producing_junk - hosts: test-targets - any_errors_fatal: true tasks: - assert: that: @@ -30,3 +31,7 @@ - out.results[0].failed - out.results[0].msg.startswith('MODULE FAILURE') - out.results[0].rc == 0 + fail_msg: | + out={{ out }} + tags: + - custom_binary_producing_junk diff --git a/tests/ansible/integration/runner/custom_binary_single_null.yml b/tests/ansible/integration/runner/custom_binary_single_null.yml index ce96bf134..cfd401f83 100644 --- a/tests/ansible/integration/runner/custom_binary_single_null.yml +++ b/tests/ansible/integration/runner/custom_binary_single_null.yml @@ -6,9 +6,10 @@ with_sequence: start=1 end={{end|default(1)}} ignore_errors: true register: out + tags: + - custom_binary_single_null - hosts: test-targets - any_errors_fatal: true tasks: - assert: that: @@ -27,7 +28,10 @@ 'custom_binary_single_null: cannot execute binary file: Exec format error\r\n', )) or (ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_version == '16.04') - + fail_msg: | + out={{ out }} + tags: + - custom_binary_single_null # Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the # return value and always returned 0. diff --git a/tests/ansible/integration/runner/custom_perl_json_args_module.yml b/tests/ansible/integration/runner/custom_perl_json_args_module.yml index f705cfe41..a34b6b758 100644 --- a/tests/ansible/integration/runner/custom_perl_json_args_module.yml +++ b/tests/ansible/integration/runner/custom_perl_json_args_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_perl_json_args_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_perl_json_args_module: foo: true @@ -11,9 +10,16 @@ that: - out.results[0].input.foo - out.results[0].message == 'I am a perl script! Here is my input.' + fail_msg: | + out={{ out }} - - when: ansible_version.full > '2.4' - assert: + - assert: that: - (not out.changed) - (not out.results[0].changed) + fail_msg: | + out={{ out }} + when: + - ansible_version.full is version('2.4', '>=', strict=True) + tags: + - custom_perl_json_args_module diff --git a/tests/ansible/integration/runner/custom_perl_want_json_module.yml b/tests/ansible/integration/runner/custom_perl_want_json_module.yml index 245271648..28ad7f7f4 100644 --- a/tests/ansible/integration/runner/custom_perl_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_perl_want_json_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_perl_want_json_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_perl_want_json_module: foo: true @@ -11,9 +10,16 @@ that: - out.results[0].input.foo - out.results[0].message == 'I am a want JSON perl script! Here is my input.' + fail_msg: | + out={{ out }} - - when: ansible_version.full > '2.4' - assert: + - assert: that: - (not out.changed) - (not out.results[0].changed) + fail_msg: | + out={{ out }} + when: + - ansible_version.full is version('2.4', '>=', strict=True) + tags: + - custom_perl_want_json_module diff --git a/tests/ansible/integration/runner/custom_python_json_args_module.yml b/tests/ansible/integration/runner/custom_python_json_args_module.yml index 338f9180a..b8b0cc8dd 100644 --- a/tests/ansible/integration/runner/custom_python_json_args_module.yml +++ b/tests/ansible/integration/runner/custom_python_json_args_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_python_json_args_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_json_args_module: foo: true @@ -13,3 +12,7 @@ (not out.results[0].changed) and out.results[0].input[0].foo and out.results[0].msg == 'Here is my input' + fail_msg: | + out={{ out }} + tags: + - custom_python_json_args_module diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index 77f2cb5cf..ebf23c3d4 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -1,8 +1,11 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets - any_errors_fatal: true tasks: + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. + - include_tasks: ../_mitogen_only.yml + - custom_python_new_style_missing_interpreter: foo: true with_sequence: start=0 end={{end|default(1)}} @@ -15,3 +18,8 @@ # Random breaking interface change since 2.7.x #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo" - "out.results[0].msg == 'Here is my input'" + fail_msg: | + out={{ out }} + tags: + - custom_python_new_style_module + - mitogen_only diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index 2ec896b7c..9fe2af9c9 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -1,10 +1,9 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets - any_errors_fatal: true tasks: - # without Mitogen Ansible 2.10 hangs on this play - - meta: end_play - when: not is_mitogen + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. + - include_tasks: ../_mitogen_only.yml - custom_python_new_style_module: foo: true @@ -18,6 +17,8 @@ # Random breaking interface change since 2.7.x #- "out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo" - "out.results[0].msg == 'Here is my input'" + fail_msg: | + out={{ out }} # Verify sys.argv is not Unicode. - custom_python_detect_environment: @@ -26,3 +27,8 @@ - assert: that: - out.argv_types_correct + fail_msg: | + out={{ out }} + tags: + - custom_python_new_style_module + - mitogen_only diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml index 458f3d2ba..d44565d82 100644 --- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml +++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml @@ -1,10 +1,21 @@ +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, if any - reload() is a Python 2.x builtin function. # issue #555 - name: integration/runner/custom_python_prehistoric_module.yml hosts: test-targets - any_errors_fatal: true tasks: + - include_tasks: ../_mitogen_only.yml + - custom_python_prehistoric_module: register: out - - assert: that=out.ok + - assert: + that: out.ok + fail_msg: | + out={{ out }} + tags: + - custom_python_prehistoric_module + - mitogen_only diff --git a/tests/ansible/integration/runner/custom_python_want_json_module.yml b/tests/ansible/integration/runner/custom_python_want_json_module.yml index f6d8c3553..fa4bc72fe 100644 --- a/tests/ansible/integration/runner/custom_python_want_json_module.yml +++ b/tests/ansible/integration/runner/custom_python_want_json_module.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_python_want_json_module.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_python_want_json_module: foo: true @@ -13,3 +12,7 @@ (not out.results[0].changed) and out.results[0].input[0].foo and out.results[0].msg == 'Here is my input' + fail_msg: | + out={{ out }} + tags: + - custom_python_want_json_module diff --git a/tests/ansible/integration/runner/custom_script_interpreter.yml b/tests/ansible/integration/runner/custom_script_interpreter.yml index 4c6b3ef5d..56f2fbe59 100644 --- a/tests/ansible/integration/runner/custom_script_interpreter.yml +++ b/tests/ansible/integration/runner/custom_script_interpreter.yml @@ -1,6 +1,5 @@ - name: integration/runner/custom_script_interpreter.yml hosts: test-targets - any_errors_fatal: true tasks: - custom_bash_old_style_module: @@ -15,4 +14,7 @@ (not out.changed) and (not out.results[0].changed) and out.results[0].msg == 'Here is my input' - + fail_msg: | + out={{ out }} + tags: + - custom_script_interpreter diff --git a/tests/ansible/integration/runner/environment_isolation.yml b/tests/ansible/integration/runner/environment_isolation.yml index 08f0924f7..fcdf1de21 100644 --- a/tests/ansible/integration/runner/environment_isolation.yml +++ b/tests/ansible/integration/runner/environment_isolation.yml @@ -1,50 +1,45 @@ # issue #309: ensure process environment is restored after a module runs. - - name: integration/runner/environment_isolation.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - - # --- - # Verify custom env setting is cleared out. - # --- - - # Verify sane state first. - - custom_python_detect_environment: + - name: Verify custom env setting is cleared, control + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined - - - shell: echo 'hi' + fail_msg: | + out={{ out }} + - name: Verify custom env setting is cleared, with evil_key + shell: echo 'hi' environment: evil_key: evil - - # Verify environment was cleaned up. - - custom_python_detect_environment: + - name: Verify custom env setting is cleared, without evil_key + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined + fail_msg: | + out={{ out }} - - # --- - # Verify non-explicit module env mutations are cleared out. - # --- - - # Verify sane state first. - - custom_python_detect_environment: + - name: Verify non-explicit module env mutations are cleared, control + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined - - - custom_python_modify_environ: + fail_msg: | + out={{ out }} + - name: Verify non-explicit module env mutations are cleared, mutate evil_key + custom_python_modify_environ: key: evil_key val: evil - - # Verify environment was cleaned up. - - custom_python_detect_environment: + - name: Verify non-explicit module env mutations are cleared, without evil_key + custom_python_detect_environment: register: out - assert: that: not out.env.evil_key is defined - + fail_msg: | + out={{ out }} + tags: + - environment_isolation diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml index df15bbdba..79e5bfa65 100644 --- a/tests/ansible/integration/runner/etc_environment.yml +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -4,12 +4,13 @@ - name: integration/runner/etc_environment.yml hosts: test-targets[0] - any_errors_fatal: true gather_facts: true tasks: - - include: _etc_environment_user.yml + - include_tasks: _etc_environment_user.yml when: ansible_system == "Linux" and is_mitogen - include_tasks: _etc_environment_global.yml # Don't destroy laptops. when: ansible_virtualization_type == "docker" + tags: + - etc_environment diff --git a/tests/ansible/integration/runner/forking_active.yml b/tests/ansible/integration/runner/forking_active.yml index e3e63b713..819142bd7 100644 --- a/tests/ansible/integration/runner/forking_active.yml +++ b/tests/ansible/integration/runner/forking_active.yml @@ -1,7 +1,7 @@ - name: integration/runner/forking_active.yml hosts: test-targets - any_errors_fatal: true tasks: + - include_tasks: ../_mitogen_only.yml # Verify mitogen_task_isolation=fork triggers forking. @@ -14,18 +14,22 @@ register: fork_proc1 vars: mitogen_task_isolation: fork - when: is_mitogen - name: get force-forked process ID again. custom_python_detect_environment: register: fork_proc2 vars: mitogen_task_isolation: fork - when: is_mitogen - assert: that: - fork_proc1.pid != sync_proc1.pid - fork_proc1.pid != fork_proc2.pid - when: is_mitogen + fail_msg: | + fork_proc1={{ fork_proc1 }} + sync_proc1={{ sync_proc1 }} + fork_proc2={{ fork_proc2 }} + tags: + - forking_active + - mitogen_only diff --git a/tests/ansible/integration/runner/forking_correct_parent.yml b/tests/ansible/integration/runner/forking_correct_parent.yml index c70db4e38..ffe1241b4 100644 --- a/tests/ansible/integration/runner/forking_correct_parent.yml +++ b/tests/ansible/integration/runner/forking_correct_parent.yml @@ -1,8 +1,8 @@ - name: integration/runner/forking_correct_parent.yml hosts: test-targets - any_errors_fatal: true tasks: + - include_tasks: ../_mitogen_only.yml # Verify mitogen_task_isolation=fork forks from "virginal fork parent", not # shared interpreter, but only if forking is enabled (e.g. that's never true @@ -15,29 +15,40 @@ self._connection.init_child_result['fork_context'] is not None ) register: forkmode - when: is_mitogen - name: get regular process ID. custom_python_detect_environment: register: regular_proc - when: is_mitogen - name: get force-forked process ID again. custom_python_detect_environment: register: fork_proc vars: mitogen_task_isolation: fork - when: is_mitogen - assert: that: - fork_proc.pid != regular_proc.pid - when: is_mitogen + fail_msg: | + fork_proc={{ fork_proc }} + regular_proc={{ regular_proc }} - assert: that: fork_proc.ppid != regular_proc.pid - when: is_mitogen and forkmode.uses_fork + fail_msg: | + fork_proc={{ fork_proc }} + regular_proc={{ regular_proc }} + when: + - forkmode.uses_fork - assert: that: fork_proc.ppid == regular_proc.pid - when: is_mitogen and not forkmode.uses_fork + fail_msg: | + fork_proc={{ fork_proc }} + regular_proc={{ regular_proc }} + when: + - not forkmode.uses_fork + + tags: + - forking_correct_parent + - mitogen_only diff --git a/tests/ansible/integration/runner/forking_inactive.yml b/tests/ansible/integration/runner/forking_inactive.yml index b84cec7ec..38fd5045b 100644 --- a/tests/ansible/integration/runner/forking_inactive.yml +++ b/tests/ansible/integration/runner/forking_inactive.yml @@ -2,22 +2,24 @@ - name: integration/runner/forking_inactive.yml hosts: test-targets - any_errors_fatal: true tasks: + - include_tasks: ../_mitogen_only.yml - name: get process ID. custom_python_detect_environment: register: sync_proc1 - when: is_mitogen - name: get process ID again. custom_python_detect_environment: register: sync_proc2 - when: is_mitogen - assert: that: - sync_proc1.pid == sync_proc2.pid - when: is_mitogen - + fail_msg: | + sync_proc1={{ sync_proc1 }} + sync_proc2={{ sync_proc2 }} + tags: + - forking_inactive + - mitogen_only diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index 107f5c209..e074c5241 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -1,19 +1,30 @@ - - name: integration/runner/missing_module.yml hosts: test-targets[0] connection: local tasks: - - connection: local + - name: Run missing_module + connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | - ansible -vvv - -i "{{MITOGEN_INVENTORY_FILE}}" + ansible + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} test-targets -m missing_module args: chdir: ../.. register: out + changed_when: false + check_mode: false ignore_errors: true - assert: that: | 'The module missing_module was not found in configured module paths' in out.stdout + fail_msg: | + out={{ out }} + tags: + - local + - missing_module diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index 28495f49e..20031704f 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -1,3 +1,10 @@ -- include: config.yml -- include: timeouts.yml -- include: variables.yml +- import_playbook: args_by_inv.yml +- import_playbook: args_by_play_taskvar.yml +- import_playbook: config.yml +- import_playbook: password.yml +- import_playbook: timeouts.yml +- import_playbook: templated_by_inv.yml +- import_playbook: templated_by_play_keyword.yml +- import_playbook: templated_by_play_taskvar.yml +- import_playbook: templated_by_task_keyword.yml +- import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/args_by_inv.yml b/tests/ansible/integration/ssh/args_by_inv.yml new file mode 100644 index 000000000..58fd1e282 --- /dev/null +++ b/tests/ansible/integration/ssh/args_by_inv.yml @@ -0,0 +1,45 @@ +- name: integration/ssh/args_by_inv.yml + hosts: issue905 + gather_facts: false + tasks: + # Test that ansible_ssh_common_args are templated; ansible_ssh_args & + # ansible_ssh_extra_args aren't directly tested, we assume they're similar. + # TODO Replace LocalCommand canary with SetEnv canary, to simplify test. + # Requires modification of sshd_config files to add AcceptEnv ... + - name: Test templating of ansible_ssh_common_args et al + block: + - name: Ensure no lingering canary files + file: + path: "{{ ssh_args_canary_file }}" + state: absent + delegate_to: localhost + + - name: Reset connections to force new ssh execution + meta: reset_connection + + - name: Perform SSH connection, to trigger side effect + ping: + + # LocalCommand="touch {{ ssh_args_canary_file }}" in ssh_*_args + - name: Stat for canary file created by side effect + stat: + path: "{{ ssh_args_canary_file }}" + delegate_to: localhost + register: ssh_args_canary_stat + + - assert: + that: + - ssh_args_canary_stat.stat.exists == true + quiet: true + success_msg: "Canary found: {{ ssh_args_canary_file }}" + fail_msg: | + ssh_args_canary_file={{ ssh_args_canary_file }} + ssh_args_canary_stat={{ ssh_args_canary_stat }} + always: + - name: Cleanup canary files + file: + path: "{{ ssh_args_canary_file }}" + state: absent + delegate_to: localhost + tags: + - issue_905 diff --git a/tests/ansible/integration/ssh/args_by_play_taskvar.yml b/tests/ansible/integration/ssh/args_by_play_taskvar.yml new file mode 100644 index 000000000..073265b2a --- /dev/null +++ b/tests/ansible/integration/ssh/args_by_play_taskvar.yml @@ -0,0 +1,53 @@ +- name: integration/ssh/args_by_play_taskvar.yml + hosts: tt_targets_bare + gather_facts: false + vars: + ansible_host: "{{ hostvars[groups['test-targets'][0]].host | default('localhost') }}" + ansible_password: "{{ 'has_sudo_nopw_password' | trim }}" + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ansible_ssh_common_args: >- + -o PermitLocalCommand=yes + -o LocalCommand="touch {{ ssh_args_canary_file }}" + ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + ssh_args_canary_file: "/tmp/ssh_args_by_play_taskvar_{{ inventory_hostname }}" + tasks: + # Test that ansible_ssh_common_args are templated; ansible_ssh_args & + # ansible_ssh_extra_args aren't directly tested, we assume they're similar. + # TODO Replace LocalCommand canary with SetEnv canary, to simplify test. + # Requires modification of sshd_config files to add AcceptEnv ... + - name: Test templating of ansible_ssh_common_args et al, by play taskvars + block: + - name: Ensure no lingering canary files + file: + path: "{{ ssh_args_canary_file }}" + state: absent + delegate_to: localhost + + - name: Reset connections to force new ssh execution + meta: reset_connection + + - name: Perform SSH connection, to trigger side effect + ping: + + - name: Stat for canary file created by side effect + stat: + path: "{{ ssh_args_canary_file }}" + delegate_to: localhost + register: ssh_args_by_play_taskvar_canary_stat + + - assert: + that: + - ssh_args_by_play_taskvar_canary_stat.stat.exists == true + quiet: true + success_msg: "Canary found: {{ ssh_args_canary_file }}" + fail_msg: | + ssh_args_canary_file={{ ssh_args_canary_file }} + ssh_args_by_play_taskvar_canary_stat={{ ssh_args_by_play_taskvar_canary_stat }} + always: + - name: Cleanup canary files + file: + path: "{{ ssh_args_canary_file }}" + state: absent + delegate_to: localhost + tags: + - issue_905 diff --git a/tests/ansible/integration/ssh/config.yml b/tests/ansible/integration/ssh/config.yml index 07ad1c210..eb804f765 100644 --- a/tests/ansible/integration/ssh/config.yml +++ b/tests/ansible/integration/ssh/config.yml @@ -6,8 +6,7 @@ vars: ansible_private_key_file: ~/fakekey tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - mitogen_get_stack: register: out @@ -17,3 +16,8 @@ out.result[0].kwargs.identity_file == ( lookup('env', 'HOME') + '/fakekey' ) + fail_msg: | + out={{ out }} + tags: + - config + - mitogen_only diff --git a/tests/ansible/integration/ssh/password.yml b/tests/ansible/integration/ssh/password.yml new file mode 100644 index 000000000..21ab6f15d --- /dev/null +++ b/tests/ansible/integration/ssh/password.yml @@ -0,0 +1,68 @@ +- name: integration/ssh/password.yml + hosts: test-targets[0] + gather_facts: false + vars: + ansible_user: mitogen__user1 + tasks: + - meta: reset_connection + - name: ansible_password + vars: + ansible_password: user1_password + ping: + + - meta: reset_connection + - name: ansible_ssh_pass + vars: + ansible_ssh_pass: user1_password + ping: + + - meta: reset_connection + - name: ansible_ssh_password + vars: + ansible_ssh_password: user1_password + ping: + + - meta: reset_connection + - name: absent password should fail + ping: + ignore_errors: true + ignore_unreachable: true + register: ssh_no_password_result + - assert: + that: + - ssh_no_password_result.unreachable == True + fail_msg: | + ssh_no_password_result={{ ssh_no_password_result }} + + - meta: reset_connection + - name: ansible_ssh_pass should override ansible_password + ping: + vars: + ansible_password: wrong + ansible_ssh_pass: user1_password + + - meta: reset_connection + - name: Highest priority password variable should override all others + vars: + ansible_password: wrong + ansible_ssh_pass: wrong + ansible_ssh_password: user1_password + ping: + + # Tests that ansible_ssh_password has priority over others + # and that a wrong password causes a target to be marked unreachable. + - meta: reset_connection + - name: Lower priority password variables should not override + vars: + ansible_password: user1_password + ansible_ssh_pass: user1_password + ansible_ssh_password: wrong + ping: + ignore_errors: true + ignore_unreachable: true + register: ssh_wrong_password_result + - assert: + that: + - ssh_wrong_password_result.unreachable == True + fail_msg: | + ssh_wrong_password_result={{ ssh_wrong_password_result }} diff --git a/tests/ansible/integration/ssh/templated_by_inv.yml b/tests/ansible/integration/ssh/templated_by_inv.yml new file mode 100644 index 000000000..686518fd4 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_inv.yml @@ -0,0 +1,7 @@ +- name: integration/ssh/templated_by_inv.yml + hosts: tt_targets_inventory + gather_facts: false + tasks: + - meta: reset_connection + - name: Templated variables in inventory + ping: diff --git a/tests/ansible/integration/ssh/templated_by_play_keyword.yml b/tests/ansible/integration/ssh/templated_by_play_keyword.yml new file mode 100644 index 000000000..c6ff1674e --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_play_keyword.yml @@ -0,0 +1,12 @@ +- name: integration/ssh/templated_by_play_keyword.yml + hosts: tt_targets_bare + gather_facts: false + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_host: "{{ hostvars[groups['test-targets'][0]].host | default('localhost') }}" + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - meta: reset_connection + - name: Templated variables in play keywords + ping: diff --git a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml new file mode 100644 index 000000000..c5c2e5443 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml @@ -0,0 +1,38 @@ +- name: integration/ssh/templated_by_play_taskvar.yml + hosts: tt_targets_bare + gather_facts: false + vars: + ansible_host: "{{ hostvars[groups['test-targets'][0]].host | default('localhost') }}" + ansible_host_key_checking: "{{ 'false' | trim }}" + ansible_password: "{{ 'has_sudo_nopw_password' | trim }}" + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ansible_ssh_executable: "{{ 'ssh' | trim }}" + ansible_timeout: "{{ 5 | int }}" + ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + + tasks: + - meta: reset_connection + - name: Templated variables in play, password authentication + ping: + +- name: integration/ssh/templated_by_play_taskvar.yml + hosts: tt_targets_bare + gather_facts: false + vars: + ansible_host: "{{ hostvars[groups['test-targets'][0]].host | default('localhost') }}" + ansible_host_key_checking: "{{ 'false' | trim }}" + ansible_private_key_file: "{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key" + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ansible_ssh_executable: "{{ 'ssh' | trim }}" + ansible_timeout: "{{ 5 | int }}" + ansible_user: "{{ 'mitogen__has_sudo_pubkey' | trim }}" + + tasks: + - meta: end_play + when: + # https://github.com/ansible/ansible/issues/84238 + - not is_mitogen + - ansible_version.full is version('2.19', '<', strict=True) + - meta: reset_connection + - name: Templated variables in play, key authentication + ping: diff --git a/tests/ansible/integration/ssh/templated_by_task_keyword.yml b/tests/ansible/integration/ssh/templated_by_task_keyword.yml new file mode 100644 index 000000000..dc16205a9 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_task_keyword.yml @@ -0,0 +1,26 @@ +- name: integration/ssh/templated_by_task_keyword.yml + hosts: tt_targets_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_host: "{{ hostvars[groups['test-targets'][0]].host | default('localhost') }}" + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Templated by task keywords, with delegate_to + delegate_to: "{{ groups.tt_targets_bare[0] }}" + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_host: "{{ hostvars[groups['test-targets'][0]].host | default('localhost') }}" + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ping: diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index 92fd93077..ec5aed05d 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -1,22 +1,38 @@ # Ensure 'ssh' connections time out correctly. +# mitogen__slow_user performs a long sleep in ~/.profile. +# Mitogen counts this time towards the connection timeout. Ansible doesn't. +# ansible_python_interpreter=python3000 is an optimisation, to avoid waiting +# on the timeout multiple times (e.g. interpreter discovery). - name: integration/ssh/timeouts.yml hosts: test-targets + gather_facts: false tasks: - - connection: local + - include_tasks: ../_mitogen_only.yml + + - name: Cause Ansible connection timeout + connection: local + environment: + ANSIBLE_SSH_TIMEOUT: 10 + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | - ansible -vvv - -i "{{MITOGEN_INVENTORY_FILE}}" - test-targets - -m custom_python_detect_environment + ansible + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + "{{ inventory_hostname }}" + -m ping -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out + changed_when: false + check_mode: false ignore_errors: true - when: is_mitogen - - assert: + - name: Verify connection timeout occurred + assert: that: - | '"changed": false' in out.stdout @@ -24,4 +40,8 @@ '"unreachable": true' in out.stdout - | '"msg": "Connection timed out."' in out.stdout - when: is_mitogen + fail_msg: | + out={{ out }} + tags: + - mitogen_only + - timeouts diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 71536391b..51783881b 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -13,123 +13,48 @@ -o "ControlPath /tmp/mitogen-ansible-test-{{18446744073709551615|random}}" tasks: - - name: ansible_ssh_user - # Remaining tests just use "ansible_user". + - name: ansible_user, ansible_ssh_private_key_file shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets - -e ansible_ssh_user=mitogen__has_sudo - -e ansible_ssh_pass=has_sudo_password - args: - chdir: ../.. - register: out - when: is_mitogen - - - shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets - -e ansible_ssh_user=mitogen__has_sudo - -e ansible_ssh_pass=wrong_password - args: - chdir: ../.. - register: out - ignore_errors: true - when: is_mitogen - - - assert: - that: out.rc == 4 # unreachable - when: is_mitogen - - - - name: ansible_ssh_pass - shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_ssh_pass=has_sudo_password - args: - chdir: ../.. - register: out - when: is_mitogen - - - shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_ssh_pass=wrong_password - args: - chdir: ../.. - register: out - ignore_errors: true - when: is_mitogen - - - assert: - that: out.rc == 4 # unreachable - when: is_mitogen - - - - name: ansible_password - shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_password=has_sudo_password - args: - chdir: ../.. - register: out - when: is_mitogen - - - shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_password=wrong_password - args: - chdir: ../.. - register: out - ignore_errors: true - when: is_mitogen - - - assert: - that: out.rc == 4 # unreachable - when: is_mitogen - - - - name: ansible_ssh_private_key_file - shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key - args: - chdir: ../.. - - - name: ansible_ssh_private_key_file - shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + ANSIBLE_ANY_ERRORS_FATAL=false + ANSIBLE_STRATEGY=mitogen_linear + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo_pubkey -e ansible_ssh_private_key_file=../data/docker/mitogen__has_sudo_pubkey.key args: chdir: ../.. register: out - when: is_mitogen + changed_when: false - - shell: > - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" - ansible -m shell -a whoami -i "{{MITOGEN_INVENTORY_FILE}}" test-targets + - name: ansible_user, wrong ansible_ssh_private_key_file + shell: > + ANSIBLE_ANY_ERRORS_FATAL=false + ANSIBLE_STRATEGY=mitogen_linear + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" + ansible -m shell -a whoami + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_private_key_file=/dev/null + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out + changed_when: false ignore_errors: true - when: is_mitogen - assert: - that: out.rc == 4 # unreachable - when: is_mitogen + that: + - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS + fail_msg: | + out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml b/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml index 1ec76fd1f..e388b7dd2 100644 --- a/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml +++ b/tests/ansible/integration/strategy/_mixed_mitogen_vanilla.yml @@ -3,7 +3,6 @@ - name: integration/strategy/_mixed_mitogen_vanilla.yml (mitogen_linear) hosts: test-targets[0] - any_errors_fatal: true strategy: mitogen_linear run_once: true tasks: @@ -11,37 +10,51 @@ register: out - assert: that: out.mitogen_loaded + fail_msg: | + out={{ out }} - determine_strategy: - assert: that: strategy == 'ansible.plugins.strategy.mitogen_linear.StrategyModule' - + fail_msg: | + strategy={{ strategy }} + tags: + - mitogen_linear - name: integration/strategy/_mixed_mitogen_vanilla.yml (linear) hosts: test-targets[0] - any_errors_fatal: true strategy: linear tasks: - custom_python_detect_environment: register: out - assert: that: not out.mitogen_loaded + fail_msg: | + out={{ out }} - determine_strategy: - assert: that: strategy == 'ansible.plugins.strategy.linear.StrategyModule' - + fail_msg: | + strategy={{ strategy }} + tags: + - linear - name: integration/strategy/_mixed_mitogen_vanilla.yml (mitogen_linear) hosts: test-targets[0] - any_errors_fatal: true strategy: mitogen_linear tasks: - custom_python_detect_environment: register: out - assert: that: out.mitogen_loaded + fail_msg: | + out={{ out }} - determine_strategy: - assert: that: strategy == 'ansible.plugins.strategy.mitogen_linear.StrategyModule' + fail_msg: | + strategy={{ strategy }} + tags: + - mitogen_linear diff --git a/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml index babcab3f9..2dbb22eda 100644 --- a/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/_mixed_vanilla_mitogen.yml @@ -3,43 +3,57 @@ - name: integration/strategy/_mixed_vanilla_mitogen.yml (linear) hosts: test-targets[0] - any_errors_fatal: true strategy: linear tasks: - custom_python_detect_environment: register: out - assert: that: not out.mitogen_loaded + fail_msg: | + out={{ out }} - determine_strategy: - assert: that: strategy == 'ansible.plugins.strategy.linear.StrategyModule' + fail_msg: | + strategy={{ strategy }} + tags: + - linear - name: integration/strategy/_mixed_vanilla_mitogen.yml (mitogen_linear) hosts: test-targets[0] - any_errors_fatal: true strategy: mitogen_linear tasks: - custom_python_detect_environment: register: out - assert: that: out.mitogen_loaded + fail_msg: | + out={{ out }} - determine_strategy: - assert: that: strategy == 'ansible.plugins.strategy.mitogen_linear.StrategyModule' - + fail_msg: | + strategy={{ strategy }} + tags: + - mitogen_linear - name: integration/strategy/_mixed_vanilla_mitogen.yml (linear) hosts: test-targets[0] - any_errors_fatal: true strategy: linear tasks: - custom_python_detect_environment: register: out - assert: that: not out.mitogen_loaded + fail_msg: | + out={{ out }} - determine_strategy: - assert: that: strategy == 'ansible.plugins.strategy.linear.StrategyModule' + fail_msg: | + strategy={{ strategy }} + tags: + - linear diff --git a/tests/ansible/integration/strategy/all.yml b/tests/ansible/integration/strategy/all.yml index b519e1bc1..3304817c0 100644 --- a/tests/ansible/integration/strategy/all.yml +++ b/tests/ansible/integration/strategy/all.yml @@ -1 +1 @@ -- include: mixed_vanilla_mitogen.yml +- import_playbook: mixed_vanilla_mitogen.yml diff --git a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml index 206f80bdc..4220ed4cd 100644 --- a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml @@ -1,24 +1,31 @@ - name: integration/strategy/mixed_vanilla_mitogen.yml (linear->mitogen->linear) hosts: test-targets[0] - any_errors_fatal: true tasks: - connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook - -i "{{MITOGEN_INVENTORY_FILE}}" - -vvv + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} integration/strategy/_mixed_mitogen_vanilla.yml args: chdir: ../.. register: out - connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook - -i "{{MITOGEN_INVENTORY_FILE}}" - -vvv + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} integration/strategy/_mixed_vanilla_mitogen.yml args: chdir: ../.. register: out + tags: + - mixed_vanilla_mitogen diff --git a/tests/ansible/integration/stub_connections/_end_play_if_not_sudo_linux.yml b/tests/ansible/integration/stub_connections/_end_play_if_not_sudo_linux.yml index 55997a729..a53f75edf 100644 --- a/tests/ansible/integration/stub_connections/_end_play_if_not_sudo_linux.yml +++ b/tests/ansible/integration/stub_connections/_end_play_if_not_sudo_linux.yml @@ -9,7 +9,7 @@ - command: sudo -n whoami args: - warn: false + warn: "{{ False if ansible_version.full is version('2.10', '<=', strict=True) else omit }}" ignore_errors: true register: sudo_available diff --git a/tests/ansible/integration/stub_connections/all.yml b/tests/ansible/integration/stub_connections/all.yml index e1810138e..a9744ab7f 100644 --- a/tests/ansible/integration/stub_connections/all.yml +++ b/tests/ansible/integration/stub_connections/all.yml @@ -1,7 +1,7 @@ -- include: kubectl.yml -- include: lxc.yml -- include: lxd.yml -- include: mitogen_doas.yml -- include: mitogen_sudo.yml -- include: setns_lxc.yml -- include: setns_lxd.yml +- import_playbook: kubectl.yml +- import_playbook: lxc.yml +- import_playbook: lxd.yml +- import_playbook: mitogen_doas.yml +- import_playbook: mitogen_sudo.yml +- import_playbook: setns_lxc.yml +- import_playbook: setns_lxd.yml diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index 867a8c178..4ee00e393 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -2,21 +2,24 @@ - name: integration/stub_connections/kubectl.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - meta: end_play - when: ansible_version.full < '2.5' + when: + - ansible_version.full is version('2.5', '<', strict=True) - custom_python_detect_environment: vars: ansible_connection: kubectl - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_kubectl_path: stub-kubectl.py register: out - assert: that: - out.env.THIS_IS_STUB_KUBECTL == '1' + fail_msg: | + out={{ out }} + tags: + - kubectl + - mitogen_only diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index 1dbe2a487..eba808fc8 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -2,18 +2,20 @@ - name: integration/stub_connections/lxc.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - custom_python_detect_environment: vars: ansible_connection: lxc - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_attach_path: stub-lxc-attach.py register: out - assert: that: - out.env.THIS_IS_STUB_LXC_ATTACH == '1' + fail_msg: | + out={{ out }} + tags: + - lxc + - mitogen_only diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index 7839a35f9..c1825d73e 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -2,18 +2,20 @@ - name: integration/stub_connections/lxd.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - custom_python_detect_environment: vars: ansible_connection: lxd - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_path: stub-lxc.py register: out - assert: that: - out.env.THIS_IS_STUB_LXC == '1' + fail_msg: | + out={{ out }} + tags: + - lxd + - mitogen_only diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index 5387744e5..c1bf991a0 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -2,15 +2,12 @@ - name: integration/stub_connections/mitogen_doas.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - custom_python_detect_environment: vars: ansible_connection: mitogen_doas - ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_doas_exe: stub-doas.py ansible_user: someuser register: out @@ -20,3 +17,8 @@ that: - out.env.THIS_IS_STUB_DOAS == '1' - (out.env.ORIGINAL_ARGV|from_json)[1:3] == ['-u', 'someuser'] + fail_msg: | + out={{ out }} + tags: + - mitogen_doas + - mitogen_only diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index e78afebc4..3634dd21d 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -2,15 +2,12 @@ - name: integration/stub_connections/mitogen_sudo.yml hosts: test-targets gather_facts: false - any_errors_fatal: true tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml - custom_python_detect_environment: vars: ansible_connection: mitogen_sudo - ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_user: root ansible_become_exe: stub-sudo.py ansible_become_flags: -H --type=sometype --role=somerole @@ -18,6 +15,12 @@ - assert: that: out.env.THIS_IS_STUB_SUDO == '1' + fail_msg: | + out={{ out }} + - assert_equal: left: (out.env.ORIGINAL_ARGV|from_json)[1:9] right: ['-u', 'root', '-H', '-r', 'somerole', '-t', 'sometype', '--'] + tags: + - mitogen_only + - mitogen_sudo diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index efef37611..58744a523 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -8,12 +8,13 @@ any_errors_fatal: false connection: local tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml + - include_tasks: _end_play_if_not_sudo_linux.yml - - include: _end_play_if_not_sudo_linux.yml - - - command: | + - name: Run stub-lxc-info.py + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, -c setns @@ -26,8 +27,13 @@ localhost args: chdir: ../.. - warn: false + warn: "{{ False if ansible_version.full is version('2.10', '<=', strict=True) else omit }}" register: result - assert: that: result.rc == 0 + fail_msg: | + result={{ result }} + tags: + - stns_lxc + - mitogen_only diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index adee0b14c..dd9fbe630 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -8,12 +8,13 @@ any_errors_fatal: false connection: local tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: ../_mitogen_only.yml + - include_tasks: _end_play_if_not_sudo_linux.yml - - include: _end_play_if_not_sudo_linux.yml - - - command: | + - name: Run ansible stub-lxc.py + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, -c setns @@ -26,8 +27,13 @@ localhost args: chdir: ../.. - warn: false + warn: "{{ False if ansible_version.full is version('2.10', '<=', strict=True) else omit }}" register: result - assert: that: result.rc == 0 + fail_msg: | + result={{ result }} + tags: + - mitogen_only + - sens_lxd diff --git a/tests/ansible/integration/transport/all.yml b/tests/ansible/integration/transport/all.yml index 534534db8..c0cb2fc7a 100644 --- a/tests/ansible/integration/transport/all.yml +++ b/tests/ansible/integration/transport/all.yml @@ -1,2 +1,4 @@ -- include: kubectl.yml +- include_playbook: kubectl.yml + tags: + - kubectl diff --git a/tests/ansible/integration/transport/kubectl.yml b/tests/ansible/integration/transport/kubectl.yml index d2be9ba5c..b7eae8648 100644 --- a/tests/ansible/integration/transport/kubectl.yml +++ b/tests/ansible/integration/transport/kubectl.yml @@ -74,6 +74,8 @@ register: _ - assert: { that: "'Python 3' in _.stdout" } + fail_msg: | + _={{ _ }} - debug: var=_.stdout,_.stderr run_once: yes @@ -83,6 +85,8 @@ register: _ - assert: { that: "'Python 2' in _.stderr" } + fail_msg: | + _={{ _ }} - debug: var=_.stdout,_.stderr run_once: yes @@ -113,7 +117,10 @@ ansible_kubectl_container: python3 register: _ - - assert: { that: "'Python 3' in _.stdout" } + - assert: + that: "'Python 3' in _.stdout" + fail_msg: | + _={{ _ }} - debug: var=_.stdout,_.stderr run_once: yes @@ -122,7 +129,10 @@ command: python --version register: _ - - assert: { that: "'Python 2' in _.stderr" } + - assert: + that: "'Python 2' in _.stderr" + fail_msg: | + _={{ _ }} - debug: var=_.stdout,_.stderr run_once: yes diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index d4ed832fe..b486549ba 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -1,11 +1,12 @@ -- include: become_method.yml -- include: become_pass.yml -- include: become_user.yml -- include: become.yml -- include: password.yml -- include: port.yml -- include: python_path.yml -- include: remote_addr.yml -- include: remote_user.yml -- include: transport.yml -- include: transport__smart.yml +- import_playbook: become_method.yml +- import_playbook: become_pass.yml +- import_playbook: become_user.yml +- import_playbook: become.yml +- import_playbook: host_key_checking.yml +- import_playbook: password.yml +- import_playbook: port.yml +- import_playbook: python_path.yml +- import_playbook: remote_addr.yml +- import_playbook: remote_user.yml +- import_playbook: transport.yml +- import_playbook: transport__smart.yml diff --git a/tests/ansible/integration/transport_config/become.yml b/tests/ansible/integration/transport_config/become.yml index baa2085e4..968d5e513 100644 --- a/tests/ansible/integration/transport_config/become.yml +++ b/tests/ansible/integration/transport_config/become.yml @@ -5,18 +5,22 @@ - name: integration/transport_config/become.yml hosts: tc-become-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 1 - out.result[0].method == "ssh" - out.result[0].kwargs.username == "ansible-cfg-remote-user" + fail_msg: | + out={{ out }} + tags: + - mitogen_only - hosts: tc-become-unset vars: {mitogen_via: becomeuser@tc-become-set} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -29,6 +33,10 @@ - out.result[2].method == "ssh" - out.result[2].kwargs.hostname == "tc-become-unset" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # Become set. @@ -37,7 +45,7 @@ become: true become_user: becomeuser tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -46,13 +54,17 @@ - out.result[0].kwargs.username == "ansible-cfg-remote-user" - out.result[1].method == "sudo" - out.result[1].kwargs.username == "becomeuser" + fail_msg: | + out={{ out }} + tags: + - mitogen_only - hosts: tc-become-set vars: {mitogen_via: tc-become-unset} become: true become_user: becomeuser tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -66,3 +78,7 @@ - out.result[2].method == "sudo" - out.result[2].kwargs.username == "becomeuser" + fail_msg: | + out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/become_method.yml b/tests/ansible/integration/transport_config/become_method.yml index 5129e5b8d..d62503149 100644 --- a/tests/ansible/integration/transport_config/become_method.yml +++ b/tests/ansible/integration/transport_config/become_method.yml @@ -2,22 +2,26 @@ # No become-method set. -- name: integration/transport_config/become-method.yml +- name: integration/transport_config/become_method.yml hosts: tc-become-method-unset become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 2 - out.result[0].method == "ssh" - out.result[1].method == "sudo" + fail_msg: | + out={{ out }} + tags: + - mitogen_only - hosts: tc-become-method-unset vars: {mitogen_via: becomeuser@tc-become-method-su} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -27,6 +31,10 @@ - out.result[1].kwargs.username == "becomeuser" - out.result[2].method == "ssh" - out.result[2].kwargs.hostname == "tc-become-method-unset" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_method=su @@ -34,7 +42,7 @@ become: true become_user: becomeuser tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -42,13 +50,17 @@ - out.result[0].method == "ssh" - out.result[1].method == "su" - out.result[1].kwargs.username == "becomeuser" + fail_msg: | + out={{ out }} + tags: + - mitogen_only - hosts: tc-become-method-su vars: {mitogen_via: tc-become-method-unset} become: true become_user: becomeuser tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -61,6 +73,10 @@ - out.result[2].method == "su" - out.result[2].kwargs.username == "becomeuser" + fail_msg: | + out={{ out }} + tags: + - mitogen_only @@ -68,7 +84,7 @@ - hosts: tc-become-method-unset vars: {mitogen_via: "doas:doasuser@tc-become-method-su"} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -81,3 +97,7 @@ - out.result[2].method == "ssh" - out.result[2].kwargs.hostname == "tc-become-method-unset" + fail_msg: | + out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml index 6a2188b11..c6b0fe728 100644 --- a/tests/ansible/integration/transport_config/become_pass.yml +++ b/tests/ansible/integration/transport_config/become_pass.yml @@ -2,11 +2,11 @@ # No become-pass set, defaults to "root" -- name: integration/transport_config/become-pass.yml +- name: integration/transport_config/become_pass.yml hosts: tc-become-pass-unset become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -14,13 +14,17 @@ - out.result[0].method == "ssh" - out.result[1].method == "sudo" - out.result[1].kwargs.password == None + fail_msg: | + out={{ out }} + tags: + - mitogen_only # Not set, unbecoming mitogen_via= - hosts: tc-become-pass-unset become: true vars: {mitogen_via: tc-become-pass-password} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -29,13 +33,17 @@ - out.result[1].method == "ssh" - out.result[2].method == "sudo" - out.result[2].kwargs.password == None + fail_msg: | + out={{ out }} + tags: + - mitogen_only # Not set, becoming mitogen_via= - hosts: tc-become-pass-unset become: true vars: {mitogen_via: viapass@tc-become-pass-password} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -46,13 +54,17 @@ - out.result[2].method == "ssh" - out.result[3].method == "sudo" - out.result[3].kwargs.password == None + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_password= set. - hosts: tc-become-pass-password become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -60,6 +72,10 @@ - out.result[0].method == "ssh" - out.result[1].method == "sudo" - out.result[1].kwargs.password == "apassword" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_password=, via= @@ -67,7 +83,7 @@ vars: {mitogen_via: root@tc-become-pass-pass} become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -78,13 +94,17 @@ - out.result[2].method == "ssh" - out.result[3].method == "sudo" - out.result[3].kwargs.password == "apassword" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_pass= - hosts: tc-become-pass-pass become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -92,6 +112,10 @@ - out.result[0].method == "ssh" - out.result[1].method == "sudo" - out.result[1].kwargs.password == "apass" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_pass=, via= @@ -99,7 +123,7 @@ vars: {mitogen_via: root@tc-become-pass-password} become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -110,34 +134,46 @@ - out.result[2].method == "ssh" - out.result[3].method == "sudo" - out.result[3].kwargs.password == "apass" + fail_msg: | + out={{ out }} + tags: + - mitogen_only - -# ansible_become_pass & ansible_become_password set, password used to take precedence -# but it's possible since https://github.com/ansible/ansible/pull/69629/files#r428376864, now it doesn't - hosts: tc-become-pass-both become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 2 - out.result[0].method == "ssh" - out.result[1].method == "sudo" - - out.result[1].kwargs.password == "c.b.a" + # Ansible <= 2.9.1 prioritises ansible_become_password. + # Ansible >= 2.9.2 prioritises ansible_become_pass. + # https://github.com/ansible/ansible/commit/480b106d6535978ae6ecab68b40942ca4fa914a0 + - out.result[1].kwargs.password == "bpass" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # both, mitogen_via - hosts: tc-become-pass-unset vars: {mitogen_via: root@tc-become-pass-both} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 3 - out.result[0].method == "ssh" - out.result[1].method == "sudo" - - out.result[1].kwargs.password == "a.b.c" + - out.result[1].kwargs.password == "bpass" - out.result[2].method == "ssh" + fail_msg: | + out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/become_user.yml b/tests/ansible/integration/transport_config/become_user.yml index 43cbca2a3..59dbe2a9d 100644 --- a/tests/ansible/integration/transport_config/become_user.yml +++ b/tests/ansible/integration/transport_config/become_user.yml @@ -2,11 +2,11 @@ # No become-user set, defaults to "root" -- name: integration/transport_config/become-user.yml +- name: integration/transport_config/become_user.yml hosts: tc-become-user-unset become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -14,13 +14,17 @@ - out.result[0].method == "ssh" - out.result[1].method == "sudo" - out.result[1].kwargs.username == "root" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # Not set, unbecoming mitogen_via= - hosts: tc-become-user-unset become: true vars: {mitogen_via: tc-become-user-set} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -29,13 +33,17 @@ - out.result[1].method == "ssh" - out.result[2].method == "sudo" - out.result[2].kwargs.username == "root" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # Not set, becoming mitogen_via= - hosts: tc-become-user-unset become: true vars: {mitogen_via: viauser@tc-become-user-set} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -46,13 +54,17 @@ - out.result[2].method == "ssh" - out.result[3].method == "sudo" - out.result[3].kwargs.username == "root" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_user= set. - hosts: tc-become-user-set become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -60,6 +72,10 @@ - out.result[0].method == "ssh" - out.result[1].method == "sudo" - out.result[1].kwargs.username == "ansi-become-user" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_user=, unbecoming via= @@ -67,7 +83,7 @@ vars: {mitogen_via: tc-become-user-unset} become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -80,6 +96,10 @@ - out.result[2].method == "sudo" - out.result[2].kwargs.username == "ansi-become-user" + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_become_user=, becoming via= @@ -87,7 +107,7 @@ vars: {mitogen_via: "doas:doasuser@tc-become-user-unset"} become: true tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -103,4 +123,8 @@ - out.result[3].method == "sudo" - out.result[3].kwargs.username == "ansi-become-user" + fail_msg: | + out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/host_key_checking.yml b/tests/ansible/integration/transport_config/host_key_checking.yml new file mode 100644 index 000000000..5f9c7e3a8 --- /dev/null +++ b/tests/ansible/integration/transport_config/host_key_checking.yml @@ -0,0 +1,100 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + +- name: integration/transport_config/host_key_checking.yml + hosts: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + fail_msg: | + out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-unset + vars: + mitogen_via: tc-hkc-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "ignore" + fail_msg: | + out={{ out }} + tags: + - mitogen_only + + +- hosts: tc-hkc-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + fail_msg: | + out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-host-key-checking + vars: + mitogen_via: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "enforce" + fail_msg: | + out={{ out }} + tags: + - mitogen_only + + +- hosts: tc-hkc-ssh-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + fail_msg: | + out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-ssh-host-key-checking + vars: + mitogen_via: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "enforce" + fail_msg: | + out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/password.yml b/tests/ansible/integration/transport_config/password.yml index ac236d66f..c55939f7f 100644 --- a/tests/ansible/integration/transport_config/password.yml +++ b/tests/ansible/integration/transport_config/password.yml @@ -6,16 +6,18 @@ - name: integration/transport_config/password.yml hosts: tc-password-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password right: "" # actually null, but assert_equal limitation + tags: + - mitogen_only - hosts: tc-password-unset vars: {mitogen_via: tc-password-explicit-ssh} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password @@ -23,22 +25,26 @@ - assert_equal: left: out.result[1].kwargs.password right: "" + tags: + - mitogen_only # ansible_ssh_user= - hosts: tc-password-explicit-ssh tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password right: "ansi-ssh-pass" + tags: + - mitogen_only - hosts: tc-password-explicit-ssh vars: {mitogen_via: tc-password-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password @@ -46,22 +52,26 @@ - assert_equal: left: out.result[1].kwargs.password right: "ansi-ssh-pass" + tags: + - mitogen_only # ansible_user= - hosts: tc-password-explicit-pass tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password right: "ansi-pass" + tags: + - mitogen_only - hosts: tc-password-explicit-pass vars: {mitogen_via: tc-password-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password @@ -69,22 +79,26 @@ - assert_equal: left: out.result[1].kwargs.password right: "ansi-pass" + tags: + - mitogen_only # both; ansible_ssh_user= takes precedence according to play_context.py. - hosts: tc-password-explicit-both tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password right: "c.b.a" + tags: + - mitogen_only - hosts: tc-password-explicit-both vars: {mitogen_via: tc-password-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.password @@ -92,3 +106,5 @@ - assert_equal: left: out.result[1].kwargs.password right: "c.b.a" + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/port.yml b/tests/ansible/integration/transport_config/port.yml index 2781081a8..abc680587 100644 --- a/tests/ansible/integration/transport_config/port.yml +++ b/tests/ansible/integration/transport_config/port.yml @@ -5,19 +5,25 @@ - name: integration/transport_config/port.yml hosts: tc-port-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 1 - out.result[0].method == "ssh" - - out.result[0].kwargs.port == None + - out.result[0].kwargs.port == expected_ssh_port + fail_msg: | + expected_ssh_port={{ expected_ssh_port }} + out={{ out }} + tags: + - mitogen_only # Not set, mitogen_via= - hosts: tc-port-explicit-ssh vars: {mitogen_via: tc-port-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -26,46 +32,65 @@ - out.result[0].kwargs.port == None - out.result[1].method == "ssh" - out.result[1].kwargs.port == 4321 + fail_msg: | + out={{ out }} + tags: + - mitogen_only # ansible_ssh_port= - hosts: tc-port-explicit-ssh tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 1 - out.result[0].method == "ssh" - out.result[0].kwargs.port == 4321 + fail_msg: | + out={{ out }} + tags: + - mitogen_only -- hosts: tc-port-explicit-unset +- hosts: tc-port-unset vars: {mitogen_via: tc-port-explicit-ssh} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 2 - out.result[0].method == "ssh" - - out.result[1].kwargs.port == 4321 + - out.result[0].kwargs.port == 4321 - out.result[1].method == "ssh" - - out.result[0].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port + fail_msg: | + expected_ssh_port={{ expected_ssh_port }} + out={{ out }} + tags: + - mitogen_only # ansible_port= - hosts: tc-port-explicit-port tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 1 - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1234 + fail_msg: | + out={{ out }} + tags: + - mitogen_only - hosts: tc-port-unset vars: {mitogen_via: tc-port-explicit-port} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -73,24 +98,34 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1234 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port + fail_msg: | + expected_ssh_port={{ expected_ssh_port }} + out={{ out }} + tags: + - mitogen_only # both, ssh takes precedence - hosts: tc-port-both tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 1 - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1532 + fail_msg: | + out={{ out }} + tags: + - mitogen_only - hosts: tc-port-unset vars: {mitogen_via: tc-port-both} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -98,4 +133,9 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1532 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port + fail_msg: | + expected_ssh_port={{ expected_ssh_port }} + out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml index 0c6069a00..ecc243744 100644 --- a/tests/ansible/integration/transport_config/python_path.yml +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -2,21 +2,31 @@ # Each case is followed by mitogen_via= case to test hostvars method. -# When no ansible_python_interpreter is set, ansible 2.8+ automatically -# tries to detect the desired interpreter, falling back to "/usr/bin/python" if necessary +# If ansible_python_interpreter isn't set, Ansible 2.8+ tries to connect and +# detect the interpreter. If that fails (e.g. connection timeout) +# - Ansible 2.8 - 9 (ansible-core 2.8 - 2.16) assumes "/usr/bin/python" +# - Ansible 10+ (ansible-core 2.17+) marks the target unreachable - name: integration/transport_config/python_path.yml hosts: tc-python-path-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["{{out.discovered_interpreter}}"] + tags: + - mitogen_only - hosts: tc-python-path-hostvar vars: {mitogen_via: tc-python-path-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -24,21 +34,28 @@ - assert_equal: left: out.result[1].kwargs.python_path right: ["/hostvar/path/to/python"] + tags: + - mitogen_only # Non-localhost with explicit ansible_python_interpreter - hosts: tc-python-path-hostvar tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: [/hostvar/path/to/python] + tags: + - mitogen_only - hosts: tc-python-path-unset vars: {mitogen_via: tc-python-path-hostvar} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -46,21 +63,28 @@ - assert_equal: left: out.result[1].kwargs.python_path right: ["{{out.discovered_interpreter}}"] + tags: + - mitogen_only # Implicit localhost gets ansible_python_interpreter=virtualenv interpreter - hosts: localhost tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["{{ansible_playbook_python}}"] + tags: + - mitogen_only - hosts: tc-python-path-unset vars: {mitogen_via: localhost} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -68,22 +92,26 @@ - assert_equal: left: out.result[1].kwargs.python_path right: ["{{out.discovered_interpreter}}"] + tags: + - mitogen_only # explicit local connections get the same treatment as everything else. - hosts: tc-python-path-local-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["{{out.discovered_interpreter}}"] + tags: + - mitogen_only - hosts: localhost vars: {mitogen_via: tc-python-path-local-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -91,21 +119,25 @@ - assert_equal: left: out.result[1].kwargs.python_path right: ["{{ansible_playbook_python}}"] + tags: + - mitogen_only # explicit local connection with explicit interpreter - hosts: tc-python-path-local-explicit tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path right: ["/a/b/c"] + tags: + - mitogen_only - hosts: localhost vars: {mitogen_via: tc-python-path-local-explicit} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -113,3 +145,5 @@ - assert_equal: left: out.result[1].kwargs.python_path right: ["{{ansible_playbook_python}}"] + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/remote_addr.yml b/tests/ansible/integration/transport_config/remote_addr.yml index b98872029..b1a6c5a7d 100644 --- a/tests/ansible/integration/transport_config/remote_addr.yml +++ b/tests/ansible/integration/transport_config/remote_addr.yml @@ -7,16 +7,18 @@ - name: integration/transport_config/remote_addr.yml hosts: tc-remote-addr-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname right: "tc-remote-addr-unset" + tags: + - mitogen_only - hosts: tc-remote-addr-unset vars: {mitogen_via: tc-remote-addr-explicit-ssh} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname @@ -24,22 +26,26 @@ - assert_equal: left: out.result[1].kwargs.hostname right: "tc-remote-addr-unset" + tags: + - mitogen_only # ansible_ssh_host= - hosts: tc-remote-addr-explicit-ssh tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname right: "ansi.ssh.host" + tags: + - mitogen_only - hosts: tc-remote-addr-explicit-ssh vars: {mitogen_via: tc-remote-addr-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname @@ -47,22 +53,26 @@ - assert_equal: left: out.result[1].kwargs.hostname right: "ansi.ssh.host" + tags: + - mitogen_only # ansible_host= - hosts: tc-remote-addr-explicit-host tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname right: "ansi.host" + tags: + - mitogen_only - hosts: tc-remote-addr-explicit-host vars: {mitogen_via: tc-remote-addr-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname @@ -70,22 +80,26 @@ - assert_equal: left: out.result[1].kwargs.hostname right: "ansi.host" + tags: + - mitogen_only # both; ansible_ssh_host= takes precedence according to play_context.py. - hosts: tc-remote-addr-explicit-both tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname right: "a.b.c" + tags: + - mitogen_only - hosts: tc-remote-addr-explicit-both vars: {mitogen_via: tc-remote-addr-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.hostname @@ -93,3 +107,5 @@ - assert_equal: left: out.result[1].kwargs.hostname right: "a.b.c" + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/remote_user.yml b/tests/ansible/integration/transport_config/remote_user.yml index b873fcbef..a5559edbc 100644 --- a/tests/ansible/integration/transport_config/remote_user.yml +++ b/tests/ansible/integration/transport_config/remote_user.yml @@ -7,17 +7,19 @@ - name: integration/transport_config/remote_user.yml hosts: tc-remote-user-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username # We set DEFAULT_REMOTE_USER in our ansible.cfg right: "ansible-cfg-remote-user" + tags: + - mitogen_only - hosts: tc-remote-user-unset vars: {mitogen_via: tc-remote-user-explicit-ssh} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username @@ -25,22 +27,26 @@ - assert_equal: left: out.result[1].kwargs.username right: "ansible-cfg-remote-user" + tags: + - mitogen_only # ansible_ssh_user= - hosts: tc-remote-user-explicit-ssh tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username right: "ansi-ssh-user" + tags: + - mitogen_only - hosts: tc-remote-user-explicit-ssh vars: {mitogen_via: tc-remote-user-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username @@ -48,22 +54,24 @@ - assert_equal: left: out.result[1].kwargs.username right: "ansi-ssh-user" + tags: + - mitogen_only # ansible_user= - hosts: tc-remote-user-explicit-user tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username right: "ansi-user" -- hosts: tc-remote-user-explicit-host +- hosts: tc-remote-user-explicit-user vars: {mitogen_via: tc-remote-user-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username @@ -71,22 +79,26 @@ - assert_equal: left: out.result[1].kwargs.username right: "ansi-user" + tags: + - mitogen_only # both; ansible_ssh_user= takes precedence according to play_context.py. - hosts: tc-remote-user-explicit-both tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username right: "c.b.a" + tags: + - mitogen_only - hosts: tc-remote-user-explicit-both vars: {mitogen_via: tc-remote-user-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.username @@ -94,3 +106,5 @@ - assert_equal: left: out.result[1].kwargs.username right: "c.b.a" + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/transport.yml b/tests/ansible/integration/transport_config/transport.yml index efedc8d44..1bac788b1 100644 --- a/tests/ansible/integration/transport_config/transport.yml +++ b/tests/ansible/integration/transport_config/transport.yml @@ -6,16 +6,18 @@ - name: integration/transport_config/transport.yml hosts: tc-transport-unset tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method right: "ssh" + tags: + - mitogen_only - hosts: tc-transport-local vars: {mitogen_via: tc-transport-unset} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method @@ -23,22 +25,26 @@ - assert_equal: left: out.result[1].method right: "local" + tags: + - mitogen_only # ansible_connection=local - hosts: tc-transport-local tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method right: "local" + tags: + - mitogen_only - hosts: tc-transport-unset vars: {mitogen_via: tc-transport-local} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method @@ -46,3 +52,5 @@ - assert_equal: left: out.result[1].method right: "ssh" + tags: + - mitogen_only diff --git a/tests/ansible/integration/transport_config/transport__smart.yml b/tests/ansible/integration/transport_config/transport__smart.yml index a4c1959fc..d2adca45f 100644 --- a/tests/ansible/integration/transport_config/transport__smart.yml +++ b/tests/ansible/integration/transport_config/transport__smart.yml @@ -4,19 +4,21 @@ # When no ansible_connection= is set, transport comes via ansible.cfg ("smart" # is parsed away to either paramiko or ssh). -- name: integration/transport_config/transport.yml +- name: integration/transport_config/transport__smart.yml hosts: tc-transport-smart tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method right: "ssh" + tags: + - mitogen_only - hosts: tc-transport-local vars: {mitogen_via: tc-transport-smart} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method @@ -24,22 +26,26 @@ - assert_equal: left: out.result[1].method right: "local" + tags: + - mitogen_only # ansible_connection=local - hosts: tc-transport-local tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method right: "local" + tags: + - mitogen_only - hosts: tc-transport-smart vars: {mitogen_via: tc-transport-local} tasks: - - include: ../_mitogen_only.yml + - include_tasks: ../_mitogen_only.yml - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].method @@ -47,3 +53,5 @@ - assert_equal: left: out.result[1].method right: "ssh" + tags: + - mitogen_only diff --git a/tests/ansible/lib/action/action_passthrough.py b/tests/ansible/lib/action/action_passthrough.py index 2748a9329..04e95f471 100644 --- a/tests/ansible/lib/action/action_passthrough.py +++ b/tests/ansible/lib/action/action_passthrough.py @@ -1,8 +1,5 @@ - import traceback -import sys -from ansible.plugins.strategy import StrategyBase from ansible.plugins.action import ActionBase diff --git a/tests/ansible/lib/action/assert_equal.py b/tests/ansible/lib/action/assert_equal.py index 84ec76065..72264cf6e 100644 --- a/tests/ansible/lib/action/assert_equal.py +++ b/tests/ansible/lib/action/assert_equal.py @@ -7,23 +7,26 @@ __metaclass__ = type import inspect -import unittest2 +import unittest import ansible.template -from ansible.errors import AnsibleError from ansible.plugins.action import ActionBase -from ansible.module_utils.six import string_types TEMPLATE_KWARGS = {} -_argspec = inspect.getargspec(ansible.template.Templar.template) +try: + # inspect.getfullargspec() Added: 3.0 + _argspec = inspect.getfullargspec(ansible.template.Templar.template) +except AttributeError: + # inspect.getargspec() Added: 2.1 Deprecated: 3.0 Removed: 3.11 + _argspec = inspect.getargspec(ansible.template.Templar.template) if 'bare_deprecated' in _argspec.args: TEMPLATE_KWARGS['bare_deprecated'] = False -class TestCase(unittest2.TestCase): +class TestCase(unittest.TestCase): def runTest(self): pass diff --git a/tests/ansible/lib/action/connection_passthrough.py b/tests/ansible/lib/action/connection_passthrough.py index 1e9211e4d..b639061b8 100644 --- a/tests/ansible/lib/action/connection_passthrough.py +++ b/tests/ansible/lib/action/connection_passthrough.py @@ -1,8 +1,5 @@ - import traceback -import sys -from ansible.plugins.strategy import StrategyBase from ansible.plugins.action import ActionBase diff --git a/tests/ansible/lib/action/determine_strategy.py b/tests/ansible/lib/action/determine_strategy.py index b4b067c11..b76783e92 100644 --- a/tests/ansible/lib/action/determine_strategy.py +++ b/tests/ansible/lib/action/determine_strategy.py @@ -1,4 +1,3 @@ - import sys from ansible.plugins.strategy import StrategyBase diff --git a/tests/ansible/lib/action/mitogen_shutdown_all.py b/tests/ansible/lib/action/mitogen_shutdown_all.py index 59191450b..14ab0656d 100644 --- a/tests/ansible/lib/action/mitogen_shutdown_all.py +++ b/tests/ansible/lib/action/mitogen_shutdown_all.py @@ -5,9 +5,7 @@ import ansible_mitogen.connection import ansible_mitogen.services -import mitogen.service -from ansible.plugins.strategy import StrategyBase from ansible.plugins.action import ActionBase diff --git a/tests/ansible/lib/callback/fork_histogram.py b/tests/ansible/lib/callback/fork_histogram.py index 15260cb5f..de5f4712e 100644 --- a/tests/ansible/lib/callback/fork_histogram.py +++ b/tests/ansible/lib/callback/fork_histogram.py @@ -1,4 +1,3 @@ - # Monkey-patch os.fork() to produce a latency histogram on run completion. # Requires 'hdrhsitograms' PyPI module. diff --git a/tests/ansible/lib/callback/nice_stdout.py b/tests/ansible/lib/callback/nice_stdout.py index 7c90a4993..5345083e3 100644 --- a/tests/ansible/lib/callback/nice_stdout.py +++ b/tests/ansible/lib/callback/nice_stdout.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import io import os -import sys from ansible import constants as C from ansible.module_utils import six diff --git a/tests/ansible/lib/callback/profile_tasks.py b/tests/ansible/lib/callback/profile_tasks.py deleted file mode 100644 index 89d956ac6..000000000 --- a/tests/ansible/lib/callback/profile_tasks.py +++ /dev/null @@ -1,82 +0,0 @@ -# profile_tasks.py: an Ansible plugin for timing tasks - -# Copyright (C) 2014 Jharrod LaFon -# https://github.com/jlafon/ansible-profile/ -# Included with permission - - -# The MIT License (MIT) -# -# Copyright (c) 2014 Jharrod LaFon -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from ansible.plugins.callback import CallbackBase -import time - - -class CallbackModule(CallbackBase): - """ - A plugin for timing tasks - """ - def __init__(self): - super(CallbackModule, self).__init__() - self.stats = {} - self.current = None - - def playbook_on_task_start(self, name, is_conditional): - """ - Logs the start of each task - """ - if self.current is not None: - # Record the running time of the last executed task - self.stats[self.current] = time.time() - self.stats[self.current] - - # Record the start time of the current task - self.current = name - self.stats[self.current] = time.time() - - called = False - - def playbook_on_stats(self, stats): - """ - Prints the timings - """ - if CallbackModule.called: - return - CallbackModule.called = True - - # Record the timing of the very last task - if self.current is not None: - self.stats[self.current] = time.time() - self.stats[self.current] - - # Sort the tasks by their running time - results = sorted(self.stats.items(), - key=lambda value: value[1], reverse=True) - - # Just keep the top 10 - results = results[:10] - - # Print the timings - for name, elapsed in results: - print("{0:-<70}{1:->9}".format( - '{0} '.format(name), - ' {0:.02f}s'.format(elapsed))) - diff --git a/tests/ansible/lib/filters/mitogen_tests.py b/tests/ansible/lib/filters/mitogen_tests.py index e76146581..a9037ccc1 100644 --- a/tests/ansible/lib/filters/mitogen_tests.py +++ b/tests/ansible/lib/filters/mitogen_tests.py @@ -1,4 +1,3 @@ - from ansible.module_utils._text import to_text diff --git a/tests/ansible/lib/module_utils/external2.py b/tests/ansible/lib/module_utils/external2.py index c815dcdb7..9a4a9bd9c 100644 --- a/tests/ansible/lib/module_utils/external2.py +++ b/tests/ansible/lib/module_utils/external2.py @@ -1,3 +1,2 @@ - def path(): return "ansible/lib/module_utils/external2.py" diff --git a/tests/ansible/lib/module_utils/externalpkg/extmod.py b/tests/ansible/lib/module_utils/externalpkg/extmod.py index 619be5dd1..a3c0f525d 100644 --- a/tests/ansible/lib/module_utils/externalpkg/extmod.py +++ b/tests/ansible/lib/module_utils/externalpkg/extmod.py @@ -1,3 +1,2 @@ - def path(): return 'ansible/lib/module_utils/externalpkg/extmod.py' diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 9f628a034..4879ac337 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -4,7 +4,6 @@ from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.basic import get_module_path -from ansible.module_utils import six import os import pwd @@ -12,21 +11,23 @@ import sys -try: - all -except NameError: - # Python 2.4 - def all(it): - for elem in it: - if not elem: - return False - return True - - def main(): module = AnsibleModule(argument_spec={}) module.exit_json( - python_version=sys.version[:3], + fs={ + '/tmp': { + 'resolved': os.path.realpath('/tmp'), + }, + }, + python={ + 'version': { + 'full': '%i.%i.%i' % sys.version_info[:3], + 'info': list(sys.version_info), + 'major': sys.version_info[0], + 'minor': sys.version_info[1], + 'patch': sys.version_info[2], + }, + }, argv=sys.argv, __file__=__file__, argv_types_correct=all(type(s) is str for s in sys.argv), diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index a63ce8e6f..846037ecd 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,8 +1,6 @@ #!/usr/bin/python # I am an Ansible Python JSONARGS module. I should receive an encoding string. -import sys - json_arguments = """<>""" print("{") diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py index 347bedf2c..9767f8553 100644 --- a/tests/ansible/lib/modules/custom_python_modify_environ.py +++ b/tests/ansible/lib/modules/custom_python_modify_environ.py @@ -5,9 +5,6 @@ from ansible.module_utils.basic import AnsibleModule import os -import pwd -import socket -import sys def main(): diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 2e0ef0da5..728685f4a 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -1,6 +1,20 @@ # I am an Ansible new-style Python module, but I lack an interpreter. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980 # NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10 @@ -8,11 +22,46 @@ # from ansible.module_utils. # import ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index f9c176c17..c84d241ab 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -1,16 +1,65 @@ #!/usr/bin/python # I am an Ansible new-style Python module. I should receive an encoding string. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # This is the magic marker Ansible looks for: # from ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index 7fe3fd1b9..d465ac9e8 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -2,12 +2,9 @@ # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. +import json import os -try: - import json -except ImportError: - import simplejson as json print(json.dumps({ 'cwd': os.getcwd() diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py index d44f51ecb..613974886 100644 --- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py +++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py @@ -1,14 +1,12 @@ #!/usr/bin/python + +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, reload() is a Python 2.x builtin function. # issue #555: I'm a module that cutpastes an old hack. from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.basic import get_module_path -from ansible.module_utils import six - -import os -import pwd -import socket -import sys import sys reload(sys) diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py index 31e0609fe..4a6243d0a 100644 --- a/tests/ansible/lib/modules/custom_python_run_script.py +++ b/tests/ansible/lib/modules/custom_python_run_script.py @@ -3,12 +3,7 @@ # parameter. from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.basic import get_module_path -from ansible.module_utils import six -import os -import pwd -import socket import sys diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py index 1fc31b4e7..6b3a356b1 100644 --- a/tests/ansible/lib/modules/custom_python_uses_distro.py +++ b/tests/ansible/lib/modules/custom_python_uses_distro.py @@ -5,14 +5,22 @@ import ansible from ansible.module_utils.basic import AnsibleModule -if ansible.__version__ > '2.8': +def try_int(s): + try: + return int(s, 10) + except ValueError: + return s + +ansible_version = tuple(try_int(s) for s in ansible.__version__.split('.')) + +if ansible_version[:2] >= (2, 8): from ansible.module_utils import distro else: distro = None def main(): module = AnsibleModule(argument_spec={}) - if ansible.__version__ > '2.8': + if ansible_version[:2] >= (2, 8): module.exit_json(info=distro.info()) else: module.exit_json(info={'id': None}) diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index d9ea7113e..23eeeb553 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,13 +1,9 @@ #!/usr/bin/python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. +import json import sys -try: - import json -except ImportError: - import simplejson as json - WANT_JSON = 1 diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py new file mode 100644 index 000000000..41cf1c1c5 --- /dev/null +++ b/tests/ansible/lib/modules/module_finder_test.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import os +import sys + +import ansible.module_utils.external1 + +from ansible.module_utils.externalpkg.extmod import path as epem_path + +def main(): + pass diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 37ab655c4..fe8ed69a9 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -9,28 +9,169 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +import os +import stat import platform +import subprocess import sys + from ansible.module_utils.basic import AnsibleModule -def main(): - result = dict(changed=False) +# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py +# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492 +# Copyright (c) 2001 - 2023 Python Software Foundation +# Copyright (c) 2024 Alex Willmer +# License: Python Software Foundation License Version 2 + +def trace_realpath(filename, strict=False): + """ + Return the canonical path of the specified filename, and a trace of + the route taken, eliminating any symbolic links encountered in the path. + """ + path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[]) + return os.path.abspath(path), trace + + +def _join_tracepath(path, rest, strict, seen, trace): + """ + Join two paths, normalizing and eliminating any symbolic links encountered + in the second path. + """ + trace.append(rest) + if isinstance(path, bytes): + sep = b'/' + curdir = b'.' + pardir = b'..' + else: + sep = '/' + curdir = '.' + pardir = '..' + + if os.path.isabs(rest): + rest = rest[1:] + path = sep + + while rest: + name, _, rest = rest.partition(sep) + if not name or name == curdir: + # current dir + continue + if name == pardir: + # parent dir + if path: + path, name = os.path.split(path) + if name == pardir: + path = os.path.join(path, pardir, pardir) + else: + path = pardir + continue + newpath = os.path.join(path, name) + try: + st = os.lstat(newpath) + except OSError: + if strict: + raise + is_link = False + else: + is_link = stat.S_ISLNK(st.st_mode) + if not is_link: + path = newpath + continue + # Resolve the symbolic link + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + else: + # Return already resolved part + rest of the path unchanged. + return os.path.join(newpath, rest), trace, False + seen[newpath] = None # not resolved symlink + path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace) + if not ok: + return os.path.join(path, rest), False + seen[newpath] = path # resolved symlink + + return path, trace, True + +def main(): module = AnsibleModule(argument_spec=dict( - facts=dict(type=dict, default={}) + facts_copy=dict(type=dict, default={}), + facts_to_override=dict(type=dict, default={}) )) - result['ansible_facts'] = module.params['facts'] # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info - if sys.platform == 'darwin' and sys.executable != '/usr/bin/python': - if int(platform.release()[:2]) < 19: - sys.executable = sys.executable[:-3] - else: + # NB This must be synced with mitogen.parent.Connection.get_boot_command() + platform_release_major = int(platform.release().partition('.')[0]) + if sys.modules.get('mitogen') and sys.platform == 'darwin': + if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7': + sys.executable = '/usr/bin/python' + if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7): # only for tests to check version of running interpreter -- Mac 10.15+ changed python2 # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python sys.executable = "/usr/bin/python" - result['running_python_interpreter'] = sys.executable + + facts_copy = module.params['facts_copy'] + + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python) + d_i_p_proc = subprocess.Popen( + [discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + + ) + d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate() + + sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable) + + result = { + 'changed': False, + 'ansible_facts': module.params['facts_to_override'], + 'discovered_and_running_samefile': os.path.samefile( + os.path.realpath(discovered_interpreter_python), + os.path.realpath(sys.executable), + ), + 'discovered_python': { + 'as_seen': discovered_interpreter_python, + 'resolved': d_i_p_realpath, + 'trace': [os.path.abspath(p) for p in d_i_p_trace], + 'sys': { + 'executable': { + 'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'), + 'proc': { + 'stderr': d_i_p_stderr.decode('ascii'), + 'returncode': d_i_p_proc.returncode, + }, + }, + }, + }, + 'running_python': { + 'platform': { + 'release': { + 'major': platform_release_major, + }, + }, + 'sys': { + 'executable': { + 'as_seen': sys.executable, + 'resolved': sys_exec_realpath, + 'trace': [os.path.abspath(p) for p in sys_exec_trace], + }, + 'platform': sys.platform, + 'version_info': { + 'major': sys.version_info[0], + 'minor': sys.version_info[1], + }, + }, + }, + } module.exit_json(**result) diff --git a/tests/ansible/mitogen_ansible_playbook.py b/tests/ansible/mitogen_ansible_playbook.py index 54fd4283c..30ed04938 100755 --- a/tests/ansible/mitogen_ansible_playbook.py +++ b/tests/ansible/mitogen_ansible_playbook.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import os -import subprocess import sys os.environ['ANSIBLE_STRATEGY'] = 'mitogen_linear' os.execlp( diff --git a/tests/ansible/regression/_mitogen_only.yml b/tests/ansible/regression/_mitogen_only.yml new file mode 120000 index 000000000..cf6c9cf7c --- /dev/null +++ b/tests/ansible/regression/_mitogen_only.yml @@ -0,0 +1 @@ +../integration/_mitogen_only.yml \ No newline at end of file diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 0d5e43cd1..a7c8033e4 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -1,15 +1,20 @@ -- include: issue_109__target_has_old_ansible_installed.yml -- include: issue_113__duplicate_module_imports.yml -- include: issue_118__script_not_marked_exec.yml -- include: issue_122__environment_difference.yml -- include: issue_140__thread_pileup.yml -- include: issue_152__local_action_wrong_interpreter.yml -- include: issue_152__virtualenv_python_fails.yml -- include: issue_154__module_state_leaks.yml -- include: issue_177__copy_module_failing.yml -- include: issue_332_ansiblemoduleerror_first_occurrence.yml -- include: issue_558_unarchive_failed.yml -- include: issue_590__sys_modules_crap.yml -- include: issue_591__setuptools_cwd_crash.yml -- include: issue_615__streaming_transfer.yml -- include: issue_655__wait_for_connection_error.yml +- import_playbook: issue_109__target_has_old_ansible_installed.yml +- import_playbook: issue_113__duplicate_module_imports.yml +- import_playbook: issue_118__script_not_marked_exec.yml +- import_playbook: issue_122__environment_difference.yml +- import_playbook: issue_140__thread_pileup.yml +- import_playbook: issue_152__local_action_wrong_interpreter.yml +- import_playbook: issue_152__virtualenv_python_fails.yml +- import_playbook: issue_154__module_state_leaks.yml +- import_playbook: issue_177__copy_module_failing.yml +- import_playbook: issue_332_ansiblemoduleerror_first_occurrence.yml +- import_playbook: issue_558_unarchive_failed.yml +- import_playbook: issue_590__sys_modules_crap.yml +- import_playbook: issue_591__setuptools_cwd_crash.yml +- import_playbook: issue_615__streaming_transfer.yml +- import_playbook: issue_655__wait_for_connection_error.yml +- import_playbook: issue_776__load_plugins_called_twice.yml +- import_playbook: issue_952__ask_become_pass.yml +- import_playbook: issue_1066__add_host__host_key_checking.yml +- import_playbook: issue_1079__wait_for_connection_timeout.yml +- import_playbook: issue_1087__template_streamerror.yml diff --git a/tests/ansible/regression/become_test.yml b/tests/ansible/regression/become_test.yml new file mode 100644 index 000000000..5af2e123c --- /dev/null +++ b/tests/ansible/regression/become_test.yml @@ -0,0 +1,9 @@ +- name: regression/become_test.yml + hosts: test-targets:&linux_containers + become: true + become_user: mitogen__pw_required + strategy: mitogen_linear + tasks: + - command: whoami + changed_when: false + check_mode: false diff --git a/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg b/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg new file mode 100644 index 000000000..ae9cfa19d Binary files /dev/null and b/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg differ diff --git a/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg b/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg new file mode 100644 index 000000000..66f1a94b5 Binary files /dev/null and b/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg differ diff --git a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml new file mode 100644 index 000000000..dd754f84e --- /dev/null +++ b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml @@ -0,0 +1,71 @@ +- name: regression/issue_1066__add_host__host_key_checking.yml + hosts: test-targets[0] + gather_facts: false + become: false + tasks: + - name: Add hosts dynamically + add_host: + name: "{{ item.name }}" + ansible_host_key_checking: "{{ item.host_key_checking | default(omit) }}" + ansible_ssh_host_key_checking: "{{ item.host_ssh_key_checking | default(omit) }}" + ansible_host: "{{ hostvars[inventory_hostname].ansible_host | default(omit) }}" + ansible_password: "{{ hostvars[inventory_hostname].ansible_password | default(omit) }}" + ansible_port: "{{ hostvars[inventory_hostname].ansible_port | default(omit) }}" + ansible_python_interpreter: "{{ hostvars[inventory_hostname].ansible_python_interpreter | default(omit) }}" + ansible_user: "{{ hostvars[inventory_hostname].ansible_user | default(omit) }}" + loop: + - {name: issue-1066-host-hkc-false, host_key_checking: false} + - {name: issue-1066-host-hkc-true, host_key_checking: true} + - {name: issue-1066-host-hskc-false, host_ssh_key_checking: false} + - {name: issue-1066-host-hskc-true, host_ssh_key_checking: true} + delegate_to: localhost + tags: + - issue_1066 + +- name: regression/issue_1066__add_host__host_key_checking.yml + hosts: issue-1066-host-* + gather_facts: false + become: false + serial: 1 + tasks: + # FIXME https://github.com/mitogen-hq/mitogen/issues/1096 + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) + - meta: reset_connection + + # The host key might be in ~/.ssh/known_hosts. If it's removed then no + # problem - test-targets hosts have host_key_checking=false. + - name: Remove existing host keys + known_hosts: + name: "{{ ansible_host }}" + state: absent + delegate_to: localhost + + - name: Ping dynamically added hosts + ping: + ignore_errors: true + ignore_unreachable: true + register: issue_1066_ping + + - debug: + var: issue_1066_ping + + - name: Confirm dynamically added hosts are/are not reachable + vars: + expected: + issue-1066-host-hkc-false: {} + issue-1066-host-hkc-true: {unreachable: true} + issue-1066-host-hskc-false: {} + issue-1066-host-hskc-true: {unreachable: true} + assert: + that: + - issue_1066_ping.unreachable is defined == expected[inventory_hostname].unreachable is defined + - issue_1066_ping.unreachable | default(42) == expected[inventory_hostname].unreachable | default(42) + # ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for + # static inventory hosts (ansible/ansible#49254, ansible/ansible#73708). + when: + - ansible_version.full is version('2.11', '>=', strict=True) + or is_mitogen + tags: + - issue_1066 diff --git a/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml new file mode 100644 index 000000000..6ded03ea2 --- /dev/null +++ b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml @@ -0,0 +1,22 @@ +- name: regression/issue_1079__wait_for_connection_timeout.yml + hosts: issue1079 + gather_facts: false + tasks: + - name: Wait for connection at start of play + wait_for_connection: + timeout: 5 + tags: + - issue_1079 + - wait_for_connection + +- hosts: issue1079 + gather_facts: false + tasks: + - meta: reset_connection + - name: Wait for connection after reset_connection + wait_for_connection: + timeout: 5 + tags: + - issue_1079 + - reset_connection + - wait_for_connection diff --git a/tests/ansible/regression/issue_1087__template_streamerror.yml b/tests/ansible/regression/issue_1087__template_streamerror.yml new file mode 100644 index 000000000..fa950ea42 --- /dev/null +++ b/tests/ansible/regression/issue_1087__template_streamerror.yml @@ -0,0 +1,43 @@ +- name: regression/issue_1087__template_streamerror.yml + # Ansible's template module has been seen to raise mitogen.core.StreamError + # iif there is a with_items loop and the destination path has an extension. + # This printed an error message and left file permissions incorrect, + # but did not cause the task/playbook to fail. + hosts: test-targets + gather_facts: false + become: false + vars: + foos: + - dest: /tmp/foo + - dest: /tmp/foo.txt + foo: Foo + bar: Bar + tasks: + - block: + - name: Test template does not cause StreamError + delegate_to: localhost + run_once: true + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + command: + cmd: > + ansible-playbook + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + regression/template_test.yml + chdir: ../ + register: issue_1087_cmd + failed_when: + - issue_1087_cmd is failed + or issue_1087_cmd.stdout is search('ERROR|mitogen\.core\.CallError') + or issue_1087_cmd.stderr is search('ERROR|mitogen\.core\.CallError') + + always: + - name: Cleanup + file: + path: "{{ item.dest }}" + state: absent + with_items: "{{ foos }}" + tags: + - issue_1087 diff --git a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml index 75e2598a4..a7ae0908c 100644 --- a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml +++ b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml @@ -3,14 +3,14 @@ - name: regression/issue_109__target_has_old_ansible_installed.yml hosts: test-targets - any_errors_fatal: true gather_facts: true tasks: - meta: end_play - when: ansible_version.full < '2.6' + when: + - ansible_version.full is version('2.6', '<', strict=True) - # Copy the naughty 'ansible' into place. - - copy: + - name: Copy the naughty ansible into place + copy: dest: "{{ansible_user_dir}}/ansible.py" src: ansible.py @@ -20,12 +20,17 @@ - custom_python_detect_environment: register: env - # Verify interpreter config would actually trigger the bug. - - assert: + - name: Verify interpreter config would actually trigger the bug + assert: that: - env.cwd == ansible_user_dir - (not env.mitogen_loaded) or (env.python_path.count("") == 1) + fail_msg: | + ansible_user_dir={{ ansible_user_dir }} + env={{ env }} - # Run some new-style modules that 'from ansible.module_utils...' - - stat: + - name: Run some new-style from ansible.module_utils... modules + stat: path: / + tags: + - issue_109 diff --git a/tests/ansible/regression/issue_113__duplicate_module_imports.yml b/tests/ansible/regression/issue_113__duplicate_module_imports.yml index 2b9e3ea88..8ae9fdead 100644 --- a/tests/ansible/regression/issue_113__duplicate_module_imports.yml +++ b/tests/ansible/regression/issue_113__duplicate_module_imports.yml @@ -2,7 +2,6 @@ # by exercisizng the uri package. - name: regression/issue_113__duplicate_module_imports.yml - any_errors_fatal: true hosts: test-targets tasks: @@ -21,4 +20,8 @@ that: - out.status == -1 - out.url == 'http://127.0.0.1:14321/post' + fail_msg: | + out={{ out }} + tags: + - issue_113 diff --git a/tests/ansible/regression/issue_118__script_not_marked_exec.yml b/tests/ansible/regression/issue_118__script_not_marked_exec.yml index 724644f98..e02c49d35 100644 --- a/tests/ansible/regression/issue_118__script_not_marked_exec.yml +++ b/tests/ansible/regression/issue_118__script_not_marked_exec.yml @@ -6,3 +6,5 @@ tasks: - script: scripts/issue_118_saytrue + tags: + - issue_118 diff --git a/tests/ansible/regression/issue_122__environment_difference.yml b/tests/ansible/regression/issue_122__environment_difference.yml index b020cc5db..273a49ae9 100644 --- a/tests/ansible/regression/issue_122__environment_difference.yml +++ b/tests/ansible/regression/issue_122__environment_difference.yml @@ -12,3 +12,5 @@ - script: scripts/print_env.py register: env - debug: msg={{env}} + tags: + - issue_122 diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index 78d5c7b11..163440a51 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -4,11 +4,11 @@ - name: regression/issue_140__thread_pileup.yml hosts: test-targets - any_errors_fatal: true tasks: - name: Create file tree connection: local + run_once: true shell: > mkdir /tmp/filetree.in; seq -f /tmp/filetree.in/%g 1 1000 | xargs touch; @@ -18,19 +18,34 @@ - name: Delete remote file tree file: path=/tmp/filetree.out state=absent - - file: + - name: Recreate file tree + file: state: directory path: /tmp/filetree.out + mode: u=rwx,go=rx - name: Trigger nasty process pileup copy: src: "{{item.src}}" dest: "/tmp/filetree.out/{{item.path}}" - mode: 0644 + mode: u=rw,go=r with_filetree: /tmp/filetree.in when: item.state == 'file' loop_control: label: "/tmp/filetree.out/{{ item.path }}" + - name: Cleanup local file tree + connection: local + run_once: true + file: + path: /tmp/filetree.in + state: absent + + - name: Cleanup remote file tree + file: + path: /tmp/filetree.out + state: absent + tags: - resource_intensive + - issue_140 diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index 8a8f00681..aafcfad44 100644 --- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -4,12 +4,11 @@ # can test for. - name: regression/issue_152__local_action_wrong_interpreter.yml - hosts: test-targets + hosts: test-targets[0] connection: local - any_errors_fatal: true tasks: - - - copy: + - name: Create /tmp/issue_152_interpreter.sh + copy: dest: /tmp/issue_152_interpreter.sh mode: u+x content: | @@ -25,7 +24,12 @@ - assert: that: - out.env.CUSTOM_INTERPRETER == "1" + fail_msg: | + out={{ out }} - - file: + - name: Cleanup /tmp/issue_152_interpreter.sh + file: path: /tmp/issue_152_interpreter.sh state: absent + tags: + - issue_152 diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index 6df3df12d..43b00de5b 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -1,5 +1,4 @@ - name: regression/issue_152__virtualenv_python_fails.yml - any_errors_fatal: true gather_facts: true hosts: test-targets tasks: @@ -8,25 +7,44 @@ # Can't use pip module because it can't create virtualenvs, must call it # directly. - - shell: virtualenv /tmp/issue_152_virtualenv - when: lout.python_version > '2.6' + - name: Create /tmp/issue_152_virtualenv environment: https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" PATH: "{{ lookup('env', 'PATH') }}" + command: + cmd: virtualenv -p "{{ ansible_facts.python.executable }}" /tmp/issue_152_virtualenv + creates: /tmp/issue_152_virtualenv + when: + - lout.python.version.full is version('2.7', '>=', strict=True) - custom_python_detect_environment: vars: ansible_python_interpreter: /tmp/issue_152_virtualenv/bin/python register: out - when: lout.python_version > '2.6' + when: + - lout.python.version.full is version('2.7', '>=', strict=True) - - assert: + - name: Check virtualenv was used + # On macOS runners a symlink /tmp -> /private/tmp has been seen + vars: + requested_executable: /tmp/issue_152_virtualenv/bin/python + expected_executables: + - "{{ requested_executable }}" + - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}" + assert: that: - - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python" - when: lout.python_version > '2.6' + - out.sys_executable in expected_executables + fail_msg: | + out={{ out }} + when: + - lout.python.version.full is version('2.7', '>=', strict=True) - - file: + - name: Cleanup /tmp/issue_152_virtualenv + file: path: /tmp/issue_152_virtualenv state: absent - when: lout.python_version > '2.6' + when: + - lout.python.version.full is version('2.7', '>=', strict=True) + tags: + - issue_152 diff --git a/tests/ansible/regression/issue_154__module_state_leaks.yml b/tests/ansible/regression/issue_154__module_state_leaks.yml index faef6e633..6ce5376df 100644 --- a/tests/ansible/regression/issue_154__module_state_leaks.yml +++ b/tests/ansible/regression/issue_154__module_state_leaks.yml @@ -2,7 +2,6 @@ # must be reinitialized or cleared out somehow on each invocation. - name: regression/issue_154__module_state_leaks.yml - any_errors_fatal: true hosts: test-targets tasks: @@ -15,4 +14,8 @@ that: - out.results[item|int].leak1 == ["David"] - out.results[item|int].leak2 == ["David"] + fail_msg: | + out={{ out }} with_sequence: start=0 end=3 + tags: + - issue_154 diff --git a/tests/ansible/regression/issue_177__copy_module_failing.yml b/tests/ansible/regression/issue_177__copy_module_failing.yml index 593a0ae7f..c8dfc6355 100644 --- a/tests/ansible/regression/issue_177__copy_module_failing.yml +++ b/tests/ansible/regression/issue_177__copy_module_failing.yml @@ -1,9 +1,9 @@ - name: regression/issue_177__copy_module_failing.yml - any_errors_fatal: true hosts: test-targets tasks: - - copy: + - name: Copy files + copy: src: /etc/{{item}} dest: /tmp/{{item}} mode: 0644 @@ -11,9 +11,12 @@ - passwd - hosts - - file: + - name: Cleanup files + file: path: /tmp/{{item}} state: absent with_items: - passwd - hosts + tags: + - issue_177 diff --git a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml index 6f32af19a..0887d151a 100644 --- a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml +++ b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml @@ -4,10 +4,19 @@ - name: regression/issue_332_ansiblemoduleerror_first_occurrence.yml hosts: test-targets tasks: - - file: path=/usr/bin/does-not-exist mode='a-s' state=file follow=yes + - name: Attempt to modify /usr/bin/does-not-exist + file: + path: /usr/bin/does-not-exist + mode: a-s + state: file + follow: true ignore_errors: true register: out - assert: that: - out.msg == 'file (/usr/bin/does-not-exist) is absent, cannot continue' + fail_msg: | + out={{ out }} + tags: + - issue_332 diff --git a/tests/ansible/regression/issue_558_unarchive_failed.yml b/tests/ansible/regression/issue_558_unarchive_failed.yml index c6b1c9f61..829101cb2 100644 --- a/tests/ansible/regression/issue_558_unarchive_failed.yml +++ b/tests/ansible/regression/issue_558_unarchive_failed.yml @@ -4,10 +4,19 @@ - name: regression/issue_558_unarchive_failed.yml hosts: test-targets tasks: - - file: state=absent path=/tmp/foo - - file: state=directory path=/tmp/foo - - unarchive: + - name: Cleanup /tmp/foo + file: + path: /tmp/foo + state: absent + - name: Create /tmp/foo + file: + path: /tmp/foo + state: directory + - name: Unarchive unarchive_test.tar + unarchive: src: "{{git_basedir}}/tests/data/unarchive_test.tar" dest: /tmp/foo # garbage doesn't work with BSD tar when: ansible_system != 'Darwin' + tags: + - issue_558 diff --git a/tests/ansible/regression/issue_590__sys_modules_crap.yml b/tests/ansible/regression/issue_590__sys_modules_crap.yml index 41130b68b..2c2262f4a 100644 --- a/tests/ansible/regression/issue_590__sys_modules_crap.yml +++ b/tests/ansible/regression/issue_590__sys_modules_crap.yml @@ -1,8 +1,9 @@ - -- hosts: test-targets +- name: regression/issue_590__sys_modules_crap.yml + hosts: test-targets tasks: - meta: end_play - when: ansible_version.full < '2.8' + when: + - ansible_version.full is version('2.8', '<', strict=True) - custom_python_uses_distro: register: out @@ -10,3 +11,7 @@ - assert: that: - "'id' in out.info" + fail_msg: | + out={{ out }} + tags: + - issue_590 diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml index fc73825c9..ff102b135 100644 --- a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml +++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml @@ -3,10 +3,10 @@ # to call getcwd() before AnsibleModule has had a chance to clean up the # process environment. -- hosts: test-targets +- name: regression/issue_591__setuptools_cwd_crash.yml + hosts: test-targets tasks: - - meta: end_play - when: not is_mitogen + - include_tasks: _mitogen_only.yml - custom_python_run_script: script: | @@ -19,6 +19,6 @@ # Will crash if process has a nonexistent CWD. - custom_python_os_getcwd: - script: | - import os - self._connection.get_chain().call(os.getcwd) + tags: + - issue_591 + - mitogen_only diff --git a/tests/ansible/regression/issue_615__streaming_transfer.yml b/tests/ansible/regression/issue_615__streaming_transfer.yml index aa7c62c47..6fe52d55d 100644 --- a/tests/ansible/regression/issue_615__streaming_transfer.yml +++ b/tests/ansible/regression/issue_615__streaming_transfer.yml @@ -1,21 +1,39 @@ # issue #615: 'fetch' with become: was internally using slurp. -- hosts: target - any_errors_fatal: True +- name: regression/issue_615_streaming_transfer.yml + hosts: test-targets gather_facts: no + # Without Mitogen this causes Ansible to use the slurp module, which is *slow* become: true vars: mitogen_ssh_compression: false tasks: - - shell: | - dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512; - chmod go= /tmp/512mb.zero + - include_tasks: _mitogen_only.yml + - block: + - name: Create /tmp/512mb.zero + shell: | + dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512; + chmod go= /tmp/512mb.zero + args: + creates: /tmp/512mb.zero - - fetch: - src: /tmp/512mb.zero - dest: /tmp/fetch-out + - name: Fetch /tmp/512mb.zero + fetch: + src: /tmp/512mb.zero + dest: /tmp/fetch-{{ inventory_hostname }}-512mb.zero + flat: true - - file: - path: /tmp/fetch-out - state: absent - delegate_to: localhost + - name: Cleanup /tmp/512mb.zero + file: + path: /tmp/512mb.zero + state: absent + + - name: Cleanup fetched file + file: + path: /tmp/fetch-{{ inventory_hostname }}-512mb.zero + state: absent + become: false + delegate_to: localhost + tags: + - issue_615 + - mitogen_only diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml index aa9472eca..a1f39f66a 100644 --- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -4,45 +4,56 @@ # since things are ran on localhost; Azure DevOps loses connection and fails # TODO: do we want to install docker a different way to be able to do this for other tests too --- -# this should only run on our Mac hosts -- hosts: target - any_errors_fatal: True +- name: regression/issue_655__wait_for_connection_error.yml + hosts: localhost gather_facts: yes become: no tasks: + - meta: end_play + when: + # Podman versions available in Homebrew have dropped macOS 12 support. + - ansible_facts.system == 'Darwin' + - ansible_facts.distribution_version is version('13.0', '<', strict=True) + + - meta: end_play + when: + # Ansible 10 (ansible-core 2.17+) require Python 3.7+ on targets. + # On CentOS 8 /usr/libexec/platform-python is Python 3.6 + - ansible_version.full is version('2.17', '>=', strict=True) + - name: set up test container and run tests inside it block: - name: install deps - block: - - name: install docker - shell: | - # NOTE: for tracking purposes: https://github.com/docker/for-mac/issues/2359 - # using docker for mac CI workaround: https://github.com/drud/ddev/pull/1748/files#diff-19288f650af2dabdf1dcc5b354d1f245 - DOCKER_URL=https://download.docker.com/mac/stable/31259/Docker.dmg && - curl -O -sSL $DOCKER_URL && - open -W Docker.dmg && cp -r /Volumes/Docker/Docker.app /Applications - sudo /Applications/Docker.app/Contents/MacOS/Docker --quit-after-install --unattended && - ln -s /Applications/Docker.app/Contents/Resources/bin/docker /usr/local/bin/docker && - nohup /Applications/Docker.app/Contents/MacOS/Docker --unattended & - # wait 2 min for docker to come up - counter=0 && - while ! /usr/local/bin/docker ps 2>/dev/null ; do - if [ $counter -lt 24 ]; then - let counter=counter+1 - else - exit 1 - fi - sleep 5 - done + homebrew: + name: + - podman + state: present + + - name: start machine + command: + cmd: "{{ item.cmd }}" + loop: + - cmd: podman machine init + - cmd: podman machine start + - cmd: podman info + timeout: 300 + register: podman_machine + changed_when: true + + - debug: + var: podman_machine # python bindings (docker_container) aren't working on this host, so gonna shell out - - name: create docker container - shell: /usr/local/bin/docker run --name testMitogen -d --rm centos:8 bash -c "sleep infinity & wait" + - name: create container + command: + cmd: podman run --name testMitogen -d --rm centos:8 bash -c "sleep infinity & wait" + changed_when: true - name: add container to inventory add_host: name: testMitogen - ansible_connection: docker + ansible_connection: podman + ansible_python_interpreter: /usr/libexec/platform-python # Python 3.6 ansible_user: root changed_when: false environment: @@ -54,6 +65,7 @@ - name: create test file file: path: /var/run/reboot-required + mode: u=rw,go=r state: touch - name: Check if reboot is required @@ -65,13 +77,16 @@ shell: sleep 2 && shutdown -r now "Ansible updates triggered" async: 1 poll: 0 - when: reboot_required.stat.exists == True + changed_when: true + when: + - reboot_required.stat.exists - name: Wait 300 seconds for server to become available wait_for_connection: delay: 30 timeout: 300 - when: reboot_required.stat.exists == True + when: + - reboot_required.stat.exists - name: cleanup test file file: @@ -82,4 +97,13 @@ PATH: /usr/local/bin/:{{ ansible_env.PATH }} - name: remove test container - shell: /usr/local/bin/docker stop testMitogen + command: + cmd: "{{ item.cmd }}" + loop: + - cmd: podman stop testMitogen + - cmd: podman machine stop + changed_when: true + when: + - ansible_facts.pkg_mgr in ['homebrew'] + tags: + - issue_655 diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml new file mode 100755 index 000000000..ef5732762 --- /dev/null +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -0,0 +1,75 @@ +# https://github.com/mitogen-hq/mitogen/issues/776 +--- +- name: regression/issue_776__load_plugins_called_twice.yml + hosts: test-targets + become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" + # Delayed until after the end_play, due to Python version requirements + gather_facts: false + tags: + - issue_776 + vars: + ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" + package: rsync # Chosen to exist in all tested distros/package managers + tasks: + # The package management modules require using the same Python version + # as the target's package manager libraries. This is sometimes in conflict + # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not + # support Python 2.x on targets. + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) + + - name: Gather facts manually + setup: + + - name: Switch to archived package repositories + copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: u=rw,go=r + loop: "{{ pkg_repos_overrides }}" + loop_control: + label: "{{ item.dest }}" + + - name: Add signing keys + copy: + src: "{{ item.src }}" + dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}" + mode: u=rw,go=r + loop: + - src: debian-archive-bullseye-automatic.gpg # Debian 11 + - src: debian-archive-bookworm-automatic.gpg # Debian 12 + when: + # Ideally this would check for Debian 11, but distribution_major_version + # is unpopulated sometimes. + - ansible_facts.distribution == "Debian" + + - name: Update package index + apt: + update_cache: true + when: + - ansible_facts.pkg_mgr in ["apt"] + + - name: Test package module 1st call + package: + name: "{{ package }}" + state: present + + - name: Test package module 2nd call + package: + name: "{{ package }}" + state: present + + - name: Test dnf module 2nd call + dnf: + name: "{{ package }}" + state: present + when: + - ansible_facts.pkg_mgr == 'dnf' + + - name: Test dnf module 2nd call + dnf: + name: "{{ package }}" + state: present + when: + - ansible_facts.pkg_mgr == 'dnf' diff --git a/tests/ansible/regression/issue_952__ask_become_pass.yml b/tests/ansible/regression/issue_952__ask_become_pass.yml new file mode 100644 index 000000000..a0b92ff2f --- /dev/null +++ b/tests/ansible/regression/issue_952__ask_become_pass.yml @@ -0,0 +1,23 @@ +- name: regression/issue_952__ask_become_pass.yml + hosts: test-targets[0]:&linux_containers + gather_facts: false + tags: + - issue_952 + tasks: + - name: Test --ask-become-pass + delegate_to: localhost + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + expect: + command: > + ansible-playbook + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + --ask-become-pass + regression/become_test.yml + chdir: ../ + responses: + 'BECOME password:': pw_required_password + changed_when: false + check_mode: false diff --git a/tests/ansible/regression/template_test.yml b/tests/ansible/regression/template_test.yml new file mode 100644 index 000000000..0b7dd36d5 --- /dev/null +++ b/tests/ansible/regression/template_test.yml @@ -0,0 +1,28 @@ +- name: regression/template_test.yml + # Ansible's template module has been seen to raise mitogen.core.StreamError + # iif there is a with_items loop and the destination path has an extension + hosts: test-targets + gather_facts: false + become: false + vars: + foos: + - dest: /tmp/foo + - dest: /tmp/foo.txt + foo: Foo + bar: Bar + tasks: + - block: + - name: Template files + template: + src: foo.bar.j2 + dest: "{{ item.dest }}" + mode: u=rw,go=r + # This has to be with_items, loop: doesn't trigger the bug + with_items: "{{ foos }}" + + always: + - name: Cleanup + file: + path: "{{ item.dest }}" + state: absent + with_items: "{{ foos }}" diff --git a/tests/ansible/regression/templates/foo.bar.j2 b/tests/ansible/regression/templates/foo.bar.j2 new file mode 100644 index 000000000..ca51a6f4c --- /dev/null +++ b/tests/ansible/regression/templates/foo.bar.j2 @@ -0,0 +1 @@ +A {{ foo }} walks into a {{ bar }}. Ow! diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index 2c3c87c86..8cfb348ab 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,7 @@ paramiko==2.3.2 # Last 2.6-compat version. +# Incompatible with pip >= 72, due to removal of `setup.py test`: +# ModuleNotFoundError: No module named 'setuptools.command.test' +# https://github.com/pypa/setuptools/issues/4519 hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) diff --git a/tests/ansible/run_ansible_playbook.py b/tests/ansible/run_ansible_playbook.py index b2b619d28..04c0c9db1 100755 --- a/tests/ansible/run_ansible_playbook.py +++ b/tests/ansible/run_ansible_playbook.py @@ -40,15 +40,6 @@ 'git_basedir': GIT_BASEDIR, } -if '-i' in sys.argv: - extra['MITOGEN_INVENTORY_FILE'] = ( - os.path.abspath(sys.argv[1 + sys.argv.index('-i')]) - ) -else: - extra['MITOGEN_INVENTORY_FILE'] = ( - os.path.join(GIT_BASEDIR, 'tests/ansible/hosts') - ) - if 'ANSIBLE_ARGV' in os.environ: args = eval(os.environ['ANSIBLE_ARGV']) else: diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml index c51fa2950..8903494d1 100644 --- a/tests/ansible/setup/all.yml +++ b/tests/ansible/setup/all.yml @@ -1 +1,2 @@ -- include: report.yml +- import_playbook: report_controller.yml +- import_playbook: report_targets.yml diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report.yml deleted file mode 100644 index 7ccd049a1..000000000 --- a/tests/ansible/setup/report.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: Report runtime settings - hosts: localhost - gather_facts: false - tasks: - - debug: {var: ansible_forks} - - debug: {var: ansible_run_tags} - - debug: {var: ansible_skip_tags} - - debug: {var: ansible_version.full} diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml new file mode 100644 index 000000000..5e34fb176 --- /dev/null +++ b/tests/ansible/setup/report_controller.yml @@ -0,0 +1,19 @@ +- name: Report controller parameters + hosts: test-targets[0] + gather_facts: false + tasks: + - debug: + msg: + - ${ANSIBLE_STRATEGY}: "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('') }}" + - ${USER}: "{{ lookup('env', 'USER') | default('') }}" + - $(groups): "{{ lookup('pipe', 'groups') }}" + - $(pwd): "{{ lookup('pipe', 'pwd') }}" + - $(whoami): "{{ lookup('pipe', 'whoami') }}" + - ansible_inventory_sources: "{{ ansible_inventory_sources | default('') }}" + - ansible_run_tags: "{{ ansible_run_tags | default('') }}" + - ansible_playbook_python: "{{ ansible_playbook_python | default('') }}" + - ansible_skip_tags: "{{ ansible_skip_tags | default('') }}" + - ansible_version.full: "{{ ansible_version.full | default('') }}" + - is_mitogen: "{{ is_mitogen | default('') }}" + - playbook_dir: "{{ playbook_dir | default('') }}" + delegate_to: localhost diff --git a/tests/ansible/setup/report_targets.yml b/tests/ansible/setup/report_targets.yml new file mode 100644 index 000000000..5aa67124d --- /dev/null +++ b/tests/ansible/setup/report_targets.yml @@ -0,0 +1,15 @@ +- name: Report target facts + hosts: localhost:test-targets + gather_facts: true + tasks: + - debug: {var: ansible_facts.distribution} + - debug: {var: ansible_facts.distribution_major_version} + - debug: {var: ansible_facts.distribution_release} + - debug: {var: ansible_facts.distribution_version} + - debug: {var: ansible_facts.kernel} + - debug: {var: ansible_facts.kernel_version} + - debug: {var: ansible_facts.os_family} + - debug: {var: ansible_facts.osrevision} + - debug: {var: ansible_facts.osversion} + - debug: {var: ansible_facts.python} + - debug: {var: ansible_facts.system} diff --git a/tests/ansible/soak/_file_service_loop.yml b/tests/ansible/soak/_file_service_loop.yml index 96111b3c2..474abd077 100644 --- a/tests/ansible/soak/_file_service_loop.yml +++ b/tests/ansible/soak/_file_service_loop.yml @@ -1,6 +1,8 @@ - - file: + - name: Delete existing /tmp/foo- + file: path: /tmp/foo-{{inventory_hostname}} state: absent - - copy: + - name: Create /tmp/foo- + copy: dest: /tmp/foo-{{inventory_hostname}} content: "{{content}}" diff --git a/tests/ansible/soak/file_service.yml b/tests/ansible/soak/file_service.yml index 65b10b2d1..f08d6dccf 100644 --- a/tests/ansible/soak/file_service.yml +++ b/tests/ansible/soak/file_service.yml @@ -1,8 +1,9 @@ -- hosts: all +- name: soak/file_service.yml + hosts: all tasks: - set_fact: content: "{% for x in range(126977) %}x{% endfor %}" - - include: _file_service_loop.yml + - include_tasks: _file_service_loop.yml with_sequence: start=1 end=100 tags: - resource_intensive diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 new file mode 100644 index 000000000..65f2fd7d9 --- /dev/null +++ b/tests/ansible/templates/test-targets.j2 @@ -0,0 +1,89 @@ +[test-targets] +{% for c in containers %} +{{ c.name }} ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }} +{% endfor %} + +[test-targets:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password + +{% for distro, hostnames in distros | dictsort %} +[{{ distro }}] +{% for hostname in hostnames %} +{{ hostname }} +{% endfor %} +{% endfor %} + +{% for family, hostnames in families | dictsort %} +[{{ family }}] +{% for hostname in hostnames %} +{{ hostname }} +{% endfor %} +{% endfor %} + +[linux:children] +test-targets + +[linux_containers:children] +test-targets + +[issue905] +{% for c in containers[:1] %} +ssh-common-args ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }} +{% endfor %} + +[issue905:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password +ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ '{{' }} ssh_args_canary_file {{ '}}' }}" +ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ '{{' }} inventory_hostname {{ '}}' }} + +{% set tt = containers[0] %} + +[issue1079] +wait-for-connection ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter="{{ '{{' }} '{{ tt.python_path }}' | trim {{ '}}' }}" + +[issue1079:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password + +[tt_targets_bare] +tt-bare + +[tt_targets_bare:vars] +ansible_python_interpreter={{ tt.python_path }} + +[tt_become_bare] +tt-become-bare + +[tt_become_bare:vars] +ansible_host={{ tt.hostname }} +ansible_password=has_sudo_nopw_password +ansible_port={{ tt.port }} +ansible_python_interpreter={{ tt.python_path }} +ansible_user=mitogen__has_sudo_nopw + +[tt_become_by_inv] +tt-become ansible_become="{{ '{{' }} 'true' | trim {{ '}}' }}" ansible_become_user=root +tt-become-exe ansible_become=true ansible_become_exe="{{ '{{' }} 'sudo' | trim {{ '}}' }}" ansible_become_user=root +tt-become-flags ansible_become=true ansible_become_flags="{{ '{{' }} '--set-home --stdin --non-interactive' | trim {{ '}}' }}" ansible_become_user=root +tt-become-method ansible_become=true ansible_become_method="{{ '{{' }} 'sudo' | trim {{ '}}' }}" ansible_become_user=root +tt-become-pass ansible_become=true ansible_become_pass="{{ '{{' }} 'pw_required_password' | trim {{ '}}' }}" ansible_become_user=mitogen__pw_required +tt-become-user ansible_become=true ansible_become_user="{{ '{{' }} 'root' | trim {{ '}}' }}" + +[tt_become_by_inv:vars] +ansible_host={{ tt.hostname }} +ansible_password=has_sudo_nopw_password +ansible_port={{ tt.port }} +ansible_python_interpreter={{ tt.python_path }} +ansible_user=mitogen__has_sudo_nopw + +[tt_targets_inventory] +tt-host ansible_host="{{ '{{' }} '{{ tt.hostname }}' | trim {{ '}}' }}" ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw +tt-host-key-checking ansible_host={{ tt.hostname }} ansible_host_key_checking="{{ '{{' }} 'false' | trim {{ '}}' }}" ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw +tt-password ansible_host={{ tt.hostname }} ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw +tt-port ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}" ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw +tt-private-key-file ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_private_key_file="{{ '{{' }} git_basedir {{ '}}' }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_pubkey +tt-remote-user ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user="{{ '{{' }} 'mitogen__has_sudo_nopw' | trim {{ '}}' }}" +tt-ssh-executable ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_ssh_executable="{{ '{{' }} 'ssh' | trim {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw +tt-timeout ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_timeout="{{ '{{' }} 5 | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index 9572717f2..b968071be 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -1,11 +1,9 @@ - import multiprocessing import os import sys import tempfile +import unittest -import mock -import unittest2 import testlib import mitogen.parent @@ -18,7 +16,7 @@ def _set_cpu_mask(self, mask): self.mask = mask -@unittest2.skipIf( +@unittest.skipIf( reason='Linux only', condition=(not os.uname()[0] == 'Linux') ) @@ -29,139 +27,139 @@ def test_assign_controller_1core(self): # Uniprocessor . policy = self.klass(cpu_count=1) policy.assign_controller() - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) def test_assign_controller_2core(self): # Small SMP gets 1.. % cpu_count policy = self.klass(cpu_count=2) policy.assign_controller() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_controller() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_controller() def test_assign_controller_3core(self): # Small SMP gets 1.. % cpu_count policy = self.klass(cpu_count=3) policy.assign_controller() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_controller() - self.assertEquals(0x4, policy.mask) + self.assertEqual(0x4, policy.mask) policy.assign_controller() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_controller() - self.assertEquals(0x4, policy.mask) + self.assertEqual(0x4, policy.mask) policy.assign_controller() def test_assign_controller_4core(self): # Big SMP gets a dedicated core. policy = self.klass(cpu_count=4) policy.assign_controller() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_controller() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) def test_assign_muxprocess_1core(self): # Uniprocessor . policy = self.klass(cpu_count=1) policy.assign_muxprocess(0) - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) def test_assign_muxprocess_2core(self): # Small SMP gets dedicated core. policy = self.klass(cpu_count=2) policy.assign_muxprocess(0) - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) policy.assign_muxprocess(0) - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) policy.assign_muxprocess(0) def test_assign_muxprocess_3core(self): # Small SMP gets a dedicated core. policy = self.klass(cpu_count=3) policy.assign_muxprocess(0) - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) policy.assign_muxprocess(0) - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) def test_assign_muxprocess_4core(self): # Big SMP gets a dedicated core. policy = self.klass(cpu_count=4) policy.assign_muxprocess(0) - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) policy.assign_muxprocess(0) - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) def test_assign_worker_1core(self): # Balance n % 1 policy = self.klass(cpu_count=1) policy.assign_worker() - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) policy.assign_worker() - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) def test_assign_worker_2core(self): # Balance n % 1 policy = self.klass(cpu_count=2) policy.assign_worker() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_worker() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) def test_assign_worker_3core(self): # Balance n % 1 policy = self.klass(cpu_count=3) policy.assign_worker() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_worker() - self.assertEquals(0x4, policy.mask) + self.assertEqual(0x4, policy.mask) policy.assign_worker() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) def test_assign_worker_4core(self): # Balance n % 1 policy = self.klass(cpu_count=4) policy.assign_worker() - self.assertEquals(4, policy.mask) + self.assertEqual(4, policy.mask) policy.assign_worker() - self.assertEquals(8, policy.mask) + self.assertEqual(8, policy.mask) policy.assign_worker() - self.assertEquals(4, policy.mask) + self.assertEqual(4, policy.mask) def test_assign_subprocess_1core(self): # allow all except reserved. policy = self.klass(cpu_count=1) policy.assign_subprocess() - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) policy.assign_subprocess() - self.assertEquals(0x1, policy.mask) + self.assertEqual(0x1, policy.mask) def test_assign_subprocess_2core(self): # allow all except reserved. policy = self.klass(cpu_count=2) policy.assign_subprocess() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) policy.assign_subprocess() - self.assertEquals(0x2, policy.mask) + self.assertEqual(0x2, policy.mask) def test_assign_subprocess_3core(self): # allow all except reserved. policy = self.klass(cpu_count=3) policy.assign_subprocess() - self.assertEquals(0x2 + 0x4, policy.mask) + self.assertEqual(0x2 + 0x4, policy.mask) policy.assign_subprocess() - self.assertEquals(0x2 + 0x4, policy.mask) + self.assertEqual(0x2 + 0x4, policy.mask) def test_assign_subprocess_4core(self): # allow all except reserved. policy = self.klass(cpu_count=4) policy.assign_subprocess() - self.assertEquals(0x4 + 0x8, policy.mask) + self.assertEqual(0x4 + 0x8, policy.mask) policy.assign_subprocess() - self.assertEquals(0x4 + 0x8, policy.mask) + self.assertEqual(0x4 + 0x8, policy.mask) -@unittest2.skipIf( +@unittest.skipIf( reason='Linux/SMP only', condition=(not ( os.uname()[0] == 'Linux' and @@ -179,19 +177,20 @@ def _get_cpus(self, path='/proc/self/status'): try: for line in fp: if line.startswith('Cpus_allowed'): - return int(line.split()[1], 16) + mask = line.split()[1].replace(',', '') + return int(mask, 16) finally: fp.close() def test_set_cpu_mask(self): self.policy._set_cpu_mask(0x1) - self.assertEquals(0x1, self._get_cpus()) + self.assertEqual(0x1, self._get_cpus()) self.policy._set_cpu_mask(0x2) - self.assertEquals(0x2, self._get_cpus()) + self.assertEqual(0x2, self._get_cpus()) self.policy._set_cpu_mask(0x3) - self.assertEquals(0x3, self._get_cpus()) + self.assertEqual(0x3, self._get_cpus()) def test_clear_on_popen(self): tf = tempfile.NamedTemporaryFile() @@ -206,7 +205,7 @@ def test_clear_on_popen(self): proc.wait() his_cpu = self._get_cpus(tf.name) - self.assertNotEquals(my_cpu, his_cpu) + self.assertNotEqual(my_cpu, his_cpu) self.policy._clear() finally: tf.close() @@ -222,11 +221,7 @@ def test_high_cpus(self): for x in range(1, 4096, 32): policy.assign_subprocess() -MockLinuxPolicyTest = unittest2.skipIf( +MockLinuxPolicyTest = unittest.skipIf( condition=(not sys.platform.startswith('linuxPolicy')), reason='select.select() not supported' )(MockLinuxPolicyTest) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/ansible/tests/connection_test.py b/tests/ansible/tests/connection_test.py index e6578954c..649f8b659 100644 --- a/tests/ansible/tests/connection_test.py +++ b/tests/ansible/tests/connection_test.py @@ -1,19 +1,16 @@ - from __future__ import absolute_import import os -import os.path -import subprocess import tempfile -import time -import unittest2 +try: + from unittest import mock +except ImportError: + import mock -import mock import ansible.errors import ansible.playbook.play_context import mitogen.core -import mitogen.utils import ansible_mitogen.connection import ansible_mitogen.plugins.connection.mitogen_local @@ -27,7 +24,6 @@ class MuxProcessMixin(object): @classmethod def setUpClass(cls): - #mitogen.utils.log_to_file() cls.model = ansible_mitogen.process.get_classic_worker_model( _init_logging=False ) @@ -49,7 +45,7 @@ def make_connection(self): conn = self.klass(play_context, new_stdin=False) # conn functions don't fetch ActionModuleMixin objs from _get_task_vars() # through the usual walk-the-stack approach so we'll not run interpreter discovery here - conn._action = mock.MagicMock(_possible_python_interpreter='/usr/bin/python') + conn._action = mock.MagicMock(_possible_python_interpreter=testlib.base_executable()) conn.on_action_run( task_vars={}, delegate_to_hostname=None, @@ -93,20 +89,20 @@ class OptionalIntTest(testlib.TestCase): func = staticmethod(ansible_mitogen.connection.optional_int) def test_already_int(self): - self.assertEquals(0, self.func(0)) - self.assertEquals(1, self.func(1)) - self.assertEquals(-1, self.func(-1)) + self.assertEqual(0, self.func(0)) + self.assertEqual(1, self.func(1)) + self.assertEqual(-1, self.func(-1)) def test_is_string(self): - self.assertEquals(0, self.func("0")) - self.assertEquals(1, self.func("1")) - self.assertEquals(-1, self.func("-1")) + self.assertEqual(0, self.func("0")) + self.assertEqual(1, self.func("1")) + self.assertEqual(-1, self.func("-1")) def test_is_none(self): - self.assertEquals(None, self.func(None)) + self.assertEqual(None, self.func(None)) def test_is_junk(self): - self.assertEquals(None, self.func({1:2})) + self.assertEqual(None, self.func({1:2})) class FetchFileTest(ConnectionMixin, testlib.TestCase): @@ -121,7 +117,7 @@ def test_success(self): # transfer_file() uses os.rename rather than direct data # overwrite, so we must reopen. with open(ofp.name, 'rb') as fp: - self.assertEquals(ifp.read(), fp.read()) + self.assertEqual(ifp.read(), fp.read()) class PutDataTest(ConnectionMixin, testlib.TestCase): @@ -131,7 +127,8 @@ def test_out_path(self): self.conn.put_data(path, contents) self.wait_for_completion() - self.assertEquals(contents, open(path, 'rb').read()) + with open(path, 'rb') as f: + self.assertEqual(contents, f.read()) os.unlink(path) def test_mode(self): @@ -141,7 +138,7 @@ def test_mode(self): self.conn.put_data(path, contents, mode=int('0123', 8)) self.wait_for_completion() st = os.stat(path) - self.assertEquals(int('0123', 8), st.st_mode & int('0777', 8)) + self.assertEqual(int('0123', 8), st.st_mode & int('0777', 8)) os.unlink(path) @@ -165,17 +162,18 @@ def test_out_path_tiny(self): path = tempfile.mktemp(prefix='mitotest') self.conn.put_file(in_path=__file__, out_path=path) self.wait_for_completion() - self.assertEquals(open(path, 'rb').read(), - open(__file__, 'rb').read()) - + with open(path, 'rb') as path_f: + with open(__file__, 'rb') as __file__f: + self.assertEqual(path_f.read(), __file__f.read()) os.unlink(path) def test_out_path_big(self): path = tempfile.mktemp(prefix='mitotest') self.conn.put_file(in_path=self.big_path, out_path=path) self.wait_for_completion() - self.assertEquals(open(path, 'rb').read(), - open(self.big_path, 'rb').read()) + with open(path, 'rb') as path_f: + with open(self.big_path, 'rb') as big_path_f: + self.assertEqual(path_f.read(), big_path_f.read()) #self._compare_times_modes(path, __file__) os.unlink(path) @@ -183,7 +181,3 @@ def test_big_in_path_not_found(self): path = tempfile.mktemp(prefix='mitotest') self.assertRaises(ansible.errors.AnsibleFileNotFound, lambda: self.conn.put_file(in_path='/nonexistent', out_path=path)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/ansible/tests/env_file_watcher_test.py b/tests/ansible/tests/env_file_watcher_test.py index 8803a6c2e..62d437cdd 100644 --- a/tests/ansible/tests/env_file_watcher_test.py +++ b/tests/ansible/tests/env_file_watcher_test.py @@ -1,9 +1,6 @@ import os -import sys import tempfile -import mock -import unittest2 import testlib from mitogen.core import b @@ -45,7 +42,7 @@ def test_key_deleted(self): self.tf.seek(0) self.tf.truncate(0) watcher.check() - self.assertTrue(b('SOMEKEY') not in environb) + self.assertNotIn(b('SOMEKEY'), environb) def test_key_added(self): watcher = klass(self.tf.name) @@ -68,7 +65,3 @@ def test_binary_key_added(self): self.tf.flush() watcher.check() self.assertEqual(environb[b('SOMEKEY')], b('\xff\xff\xff')) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py new file mode 100644 index 000000000..79e8fdbd5 --- /dev/null +++ b/tests/ansible/tests/module_finder_test.py @@ -0,0 +1,80 @@ +import os.path +import sys +import textwrap +import unittest + +import ansible_mitogen.module_finder + +import testlib + + +class ScanFromListTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b.c; from d.e import f; from g import h, i + ''') + code = compile(source, '', 'exec') + self.assertEqual( + list(ansible_mitogen.module_finder.scan_fromlist(code)), + [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')], + ) + + +class WalkImportsTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b; import b.c; from b.d import e, f + ''') + code = compile(source, '', 'exec') + + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code)), + ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'], + ) + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')), + ['b.c', 'b.d', 'b.d.e', 'b.d.f'], + ) + + +class ScanTest(testlib.TestCase): + module_name = 'ansible_module_module_finder_test__this_should_not_matter' + module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py') + search_path = ( + 'does_not_exist/module_utils', + testlib.ANSIBLE_MODULE_UTILS_DIR, + ) + + @staticmethod + def relpath(path): + return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR) + + @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable') + def test_importlib_find_spec(self): + scan = ansible_mitogen.module_finder._scan_importlib_find_spec + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) + + @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred') + def test_imp_find_module(self): + scan = ansible_mitogen.module_finder._scan_imp_find_module + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) diff --git a/tests/ansible/tests/target_test.py b/tests/ansible/tests/target_test.py index 6bdc949bc..806175edd 100644 --- a/tests/ansible/tests/target_test.py +++ b/tests/ansible/tests/target_test.py @@ -1,11 +1,13 @@ - from __future__ import absolute_import import os.path import subprocess import tempfile -import unittest2 +import unittest -import mock +try: + from unittest import mock +except ImportError: + import mock import ansible_mitogen.target import testlib @@ -56,18 +58,18 @@ def test_no_good_candidate(self, is_good_temp_dir): -class ApplyModeSpecTest(unittest2.TestCase): +class ApplyModeSpecTest(unittest.TestCase): func = staticmethod(ansible_mitogen.target.apply_mode_spec) def test_simple(self): spec = 'u+rwx,go=x' - self.assertEquals(int('0711', 8), self.func(spec, 0)) + self.assertEqual(int('0711', 8), self.func(spec, 0)) spec = 'g-rw' - self.assertEquals(int('0717', 8), self.func(spec, int('0777', 8))) + self.assertEqual(int('0717', 8), self.func(spec, int('0777', 8))) -class IsGoodTempDirTest(unittest2.TestCase): +class IsGoodTempDirTest(unittest.TestCase): func = staticmethod(ansible_mitogen.target.is_good_temp_dir) def test_creates(self): @@ -84,8 +86,11 @@ def test_file_exists(self): fp.write('derp') self.assertTrue(os.path.isfile(bleh)) self.assertFalse(self.func(bleh)) - self.assertEquals(open(bleh).read(), 'derp') + with open(bleh) as fp: + self.assertEqual(fp.read(), 'derp') + @unittest.skipIf( + os.geteuid() == 0, 'writes by root ignore directory permissions') def test_unwriteable(self): with NamedTemporaryDirectory() as temp_path: os.chmod(temp_path, 0) @@ -103,8 +108,3 @@ def test_noexec(self, os_access): os_access.return_value = False with NamedTemporaryDirectory() as temp_path: self.assertFalse(self.func(temp_path)) - - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/ansible/tests/utils_test.py b/tests/ansible/tests/utils_test.py new file mode 100644 index 000000000..05ee8bf07 --- /dev/null +++ b/tests/ansible/tests/utils_test.py @@ -0,0 +1,11 @@ +import unittest + +import ansible_mitogen.utils + + +class AnsibleVersionTest(unittest.TestCase): + def test_ansible_version(self): + self.assertIsInstance(ansible_mitogen.utils.ansible_version, tuple) + self.assertIsInstance(ansible_mitogen.utils.ansible_version[0], int) + self.assertIsInstance(ansible_mitogen.utils.ansible_version[1], int) + self.assertEqual(2, ansible_mitogen.utils.ansible_version[0]) diff --git a/tests/ansible/tests/utils_unsafe_test.py b/tests/ansible/tests/utils_unsafe_test.py new file mode 100644 index 000000000..9aa461c58 --- /dev/null +++ b/tests/ansible/tests/utils_unsafe_test.py @@ -0,0 +1,92 @@ +import unittest + +from ansible.utils.unsafe_proxy import AnsibleUnsafeBytes +from ansible.utils.unsafe_proxy import AnsibleUnsafeText +from ansible.utils.unsafe_proxy import wrap_var + +import ansible_mitogen.utils.unsafe + +import mitogen.core + + +class Bytes(bytes): pass +class Dict(dict): pass +class List(list): pass +class Set(set): pass +class Text(mitogen.core.UnicodeType): pass +class Tuple(tuple): pass + + +class CastTest(unittest.TestCase): + def assertIsType(self, obj, cls, msg=None): + self.assertIs(type(obj), cls, msg) + + def assertUnchanged(self, obj): + self.assertIs(ansible_mitogen.utils.unsafe.cast(obj), obj) + + def assertCasts(self, obj, expected): + cast = ansible_mitogen.utils.unsafe.cast + self.assertEqual(cast(obj), expected) + self.assertIsType(cast(obj), type(expected)) + + def test_ansible_unsafe(self): + self.assertCasts(AnsibleUnsafeBytes(b'abc'), b'abc') + self.assertCasts(AnsibleUnsafeText(u'abc'), u'abc') + + def test_passthrough(self): + self.assertUnchanged(0) + self.assertUnchanged(0.0) + self.assertUnchanged(False) + self.assertUnchanged(True) + self.assertUnchanged(None) + self.assertUnchanged(b'') + self.assertUnchanged(u'') + + def test_builtins_roundtrip(self): + self.assertCasts(wrap_var(b''), b'') + self.assertCasts(wrap_var({}), {}) + self.assertCasts(wrap_var([]), []) + self.assertCasts(wrap_var(u''), u'') + self.assertCasts(wrap_var(()), []) + + def test_subtypes_roundtrip(self): + self.assertCasts(wrap_var(Bytes()), b'') + self.assertCasts(wrap_var(Dict()), {}) + self.assertCasts(wrap_var(List()), []) + self.assertCasts(wrap_var(Text()), u'') + self.assertCasts(wrap_var(Tuple()), []) + + def test_subtype_nested_dict(self): + obj = Dict(foo=Dict(bar=u'abc')) + wrapped = wrap_var(obj) + unwrapped = ansible_mitogen.utils.unsafe.cast(wrapped) + self.assertEqual(unwrapped, {'foo': {'bar': u'abc'}}) + self.assertIsType(unwrapped, dict) + self.assertIsType(unwrapped['foo'], dict) + self.assertIsType(unwrapped['foo']['bar'], mitogen.core.UnicodeType) + + def test_subtype_roundtrip_list(self): + # wrap_var() preserves sequence types, cast() does not (for now) + obj = List([List([u'abc'])]) + wrapped = wrap_var(obj) + unwrapped = ansible_mitogen.utils.unsafe.cast(wrapped) + self.assertEqual(unwrapped, [[u'abc']]) + self.assertIsType(unwrapped, list) + self.assertIsType(unwrapped[0], list) + self.assertIsType(unwrapped[0][0], mitogen.core.UnicodeType) + + def test_subtype_roundtrip_tuple(self): + # wrap_var() preserves sequence types, cast() does not (for now) + obj = Tuple([Tuple([u'abc'])]) + wrapped = wrap_var(obj) + unwrapped = ansible_mitogen.utils.unsafe.cast(wrapped) + self.assertEqual(unwrapped, [[u'abc']]) + self.assertIsType(unwrapped, list) + self.assertIsType(unwrapped[0], list) + self.assertIsType(unwrapped[0][0], mitogen.core.UnicodeType) + + def test_unknown_types_raise(self): + cast = ansible_mitogen.utils.unsafe.cast + self.assertRaises(TypeError, cast, set()) + self.assertRaises(TypeError, cast, Set()) + self.assertRaises(TypeError, cast, 4j) diff --git a/tests/bench/fork.py b/tests/bench/fork.py index af5cb3a7d..d77f4437d 100644 --- a/tests/bench/fork.py +++ b/tests/bench/fork.py @@ -5,6 +5,12 @@ import mitogen import mitogen.core +try: + xrange +except NameError: + xrange = range + + @mitogen.main() def main(router): t0 = mitogen.core.now() @@ -12,4 +18,4 @@ def main(router): t = mitogen.core.now() ctx = router.fork() ctx.shutdown(wait=True) - print '++', 1000 * ((mitogen.core.now() - t0) / (1.0+x)) + print('++ %d' % 1000 * ((mitogen.core.now() - t0) / (1.0+x))) diff --git a/tests/bench/large_messages.py b/tests/bench/large_messages.py index e977e36dc..8593a33aa 100644 --- a/tests/bench/large_messages.py +++ b/tests/bench/large_messages.py @@ -1,10 +1,5 @@ - # Verify _receive_one() quadratic behaviour fixed. -import subprocess -import time -import socket - import mitogen import mitogen.core diff --git a/tests/bench/latch_roundtrip.py b/tests/bench/latch_roundtrip.py index 1198aa48c..cbc483743 100644 --- a/tests/bench/latch_roundtrip.py +++ b/tests/bench/latch_roundtrip.py @@ -3,9 +3,7 @@ """ import threading -import time -import mitogen import mitogen.core import mitogen.utils import ansible_mitogen.affinity diff --git a/tests/bench/local.py b/tests/bench/local.py index aefeb84da..0653a0dee 100644 --- a/tests/bench/local.py +++ b/tests/bench/local.py @@ -2,8 +2,6 @@ Measure latency of .local() setup. """ -import time - import mitogen import mitogen.core import mitogen.utils diff --git a/tests/bench/megatime.py b/tests/bench/megatime.py index 40cd99860..870627698 100755 --- a/tests/bench/megatime.py +++ b/tests/bench/megatime.py @@ -2,7 +2,6 @@ import sys import os -import time import mitogen.core diff --git a/tests/bench/roundtrip.py b/tests/bench/roundtrip.py index 8f31b1a21..774dfc5b8 100644 --- a/tests/bench/roundtrip.py +++ b/tests/bench/roundtrip.py @@ -2,9 +2,6 @@ Measure latency of local RPC. """ -import time - -import mitogen import mitogen.core import mitogen.utils import ansible_mitogen.affinity diff --git a/tests/bench/service.py b/tests/bench/service.py index 267ae3f6f..d3a71b799 100644 --- a/tests/bench/service.py +++ b/tests/bench/service.py @@ -1,9 +1,6 @@ """ Measure latency of local service RPC. """ - -import time - import mitogen import mitogen.core import mitogen.service diff --git a/tests/bench/ssh-roundtrip.py b/tests/bench/ssh-roundtrip.py index 06c596c00..c5f145743 100644 --- a/tests/bench/ssh-roundtrip.py +++ b/tests/bench/ssh-roundtrip.py @@ -3,9 +3,7 @@ """ import sys -import time -import mitogen import mitogen.core import mitogen.utils import ansible_mitogen.affinity diff --git a/tests/bench/throughput.py b/tests/bench/throughput.py index acb51afa3..7d67d158c 100644 --- a/tests/bench/throughput.py +++ b/tests/bench/throughput.py @@ -1,11 +1,7 @@ # Verify throughput over sudo and SSH at various compression levels. import os -import random -import socket -import subprocess import tempfile -import time import mitogen import mitogen.core diff --git a/tests/broker_test.py b/tests/broker_test.py index 2212d8aa1..41b953878 100644 --- a/tests/broker_test.py +++ b/tests/broker_test.py @@ -1,9 +1,7 @@ - -import time -import threading - -import mock -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import testlib @@ -19,7 +17,7 @@ def test_poller_closed(self): broker.poller.close = mock.Mock() broker.shutdown() broker.join() - self.assertEquals(1, len(broker.poller.close.mock_calls)) + self.assertEqual(1, len(broker.poller.close.mock_calls)) actual_close() @@ -31,7 +29,7 @@ def test_defer(self): broker = self.klass() try: broker.defer(lambda: latch.put(123)) - self.assertEquals(123, latch.get()) + self.assertEqual(123, latch.get()) finally: broker.shutdown() broker.join() @@ -44,7 +42,7 @@ def test_defer_after_shutdown(self): e = self.assertRaises(mitogen.core.Error, lambda: broker.defer(lambda: latch.put(123))) - self.assertEquals(e.args[0], mitogen.core.Waker.broker_shutdown_msg) + self.assertEqual(e.args[0], mitogen.core.Waker.broker_shutdown_msg) class DeferSyncTest(testlib.TestCase): @@ -53,8 +51,8 @@ class DeferSyncTest(testlib.TestCase): def test_okay(self): broker = self.klass() try: - th = broker.defer_sync(lambda: threading.currentThread()) - self.assertEquals(th, broker._thread) + th = broker.defer_sync(lambda: mitogen.core.threading__current_thread()) + self.assertEqual(th, broker._thread) finally: broker.shutdown() broker.join() @@ -67,7 +65,3 @@ def test_exception(self): finally: broker.shutdown() broker.join() - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/buildah_test.py b/tests/buildah_test.py index 874205cdd..8412cff3a 100644 --- a/tests/buildah_test.py +++ b/tests/buildah_test.py @@ -1,9 +1,5 @@ import os -import mitogen - -import unittest2 - import testlib @@ -17,12 +13,8 @@ def test_okay(self): stream = self.router.stream_by_id(context.context_id) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) - self.assertEquals(argv[0], buildah_path) - self.assertEquals(argv[1], 'run') - self.assertEquals(argv[2], '--') - self.assertEquals(argv[3], 'container_name') - self.assertEquals(argv[4], stream.conn.options.python_path) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(argv[0], buildah_path) + self.assertEqual(argv[1], 'run') + self.assertEqual(argv[2], '--') + self.assertEqual(argv[3], 'container_name') + self.assertEqual(argv[4], stream.conn.options.python_path) diff --git a/tests/call_error_test.py b/tests/call_error_test.py index 00ff0ed92..9c1174b8f 100644 --- a/tests/call_error_test.py +++ b/tests/call_error_test.py @@ -1,8 +1,6 @@ import pickle import sys -import unittest2 - import mitogen.core import testlib @@ -14,28 +12,28 @@ class ConstructorTest(testlib.TestCase): def test_string_noargs(self): e = self.klass('%s%s') - self.assertEquals(e.args[0], '%s%s') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertEqual(e.args[0], '%s%s') + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_string_args(self): e = self.klass('%s%s', 1, 1) - self.assertEquals(e.args[0], '11') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertEqual(e.args[0], '11') + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_from_exc(self): ve = plain_old_module.MyError('eek') e = self.klass(ve) - self.assertEquals(e.args[0], 'plain_old_module.MyError: eek') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertEqual(e.args[0], 'plain_old_module.MyError: eek') + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_form_base_exc(self): ve = SystemExit('eek') e = self.klass(ve) cls = ve.__class__ - self.assertEquals(e.args[0], + self.assertEqual(e.args[0], # varies across 2/3. '%s.%s: eek' % (cls.__module__, cls.__name__)) - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_from_exc_tb(self): try: @@ -45,19 +43,19 @@ def test_from_exc_tb(self): e = self.klass(ve) self.assertTrue(e.args[0].startswith('plain_old_module.MyError: eek')) - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) - self.assertTrue('test_from_exc_tb' in e.args[0]) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) + self.assertIn('test_from_exc_tb', e.args[0]) def test_bytestring_conversion(self): e = self.klass(mitogen.core.b('bytes')) - self.assertEquals(u'bytes', e.args[0]) - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertEqual(u'bytes', e.args[0]) + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_reduce(self): e = self.klass('eek') func, (arg,) = e.__reduce__() - self.assertTrue(func is mitogen.core._unpickle_call_error) - self.assertEquals(arg, e.args[0]) + self.assertIs(func, mitogen.core._unpickle_call_error) + self.assertEqual(arg, e.args[0]) class UnpickleCallErrorTest(testlib.TestCase): @@ -73,10 +71,10 @@ def test_oversized(self): def test_reify(self): e = self.func(u'some error') - self.assertEquals(mitogen.core.CallError, e.__class__) - self.assertEquals(1, len(e.args)) - self.assertEquals(mitogen.core.UnicodeType, type(e.args[0])) - self.assertEquals(u'some error', e.args[0]) + self.assertEqual(mitogen.core.CallError, e.__class__) + self.assertEqual(1, len(e.args)) + self.assertEqual(mitogen.core.UnicodeType, type(e.args[0])) + self.assertEqual(u'some error', e.args[0]) class PickleTest(testlib.TestCase): @@ -85,18 +83,18 @@ class PickleTest(testlib.TestCase): def test_string_noargs(self): e = self.klass('%s%s') e2 = pickle.loads(pickle.dumps(e)) - self.assertEquals(e2.args[0], '%s%s') + self.assertEqual(e2.args[0], '%s%s') def test_string_args(self): e = self.klass('%s%s', 1, 1) e2 = pickle.loads(pickle.dumps(e)) - self.assertEquals(e2.args[0], '11') + self.assertEqual(e2.args[0], '11') def test_from_exc(self): ve = plain_old_module.MyError('eek') e = self.klass(ve) e2 = pickle.loads(pickle.dumps(e)) - self.assertEquals(e2.args[0], 'plain_old_module.MyError: eek') + self.assertEqual(e2.args[0], 'plain_old_module.MyError: eek') def test_from_exc_tb(self): try: @@ -107,8 +105,4 @@ def test_from_exc_tb(self): e2 = pickle.loads(pickle.dumps(e)) self.assertTrue(e2.args[0].startswith('plain_old_module.MyError: eek')) - self.assertTrue('test_from_exc_tb' in e2.args[0]) - - -if __name__ == '__main__': - unittest2.main() + self.assertIn('test_from_exc_tb', e2.args[0]) diff --git a/tests/call_function_test.py b/tests/call_function_test.py index 9e821b27c..1e838bdad 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -1,11 +1,7 @@ -import logging import time -import unittest2 - import mitogen.core import mitogen.parent -import mitogen.master from mitogen.core import str_partition import testlib @@ -79,7 +75,7 @@ def test_crashes(self): def test_bad_return_value(self): exc = self.assertRaises(mitogen.core.StreamError, lambda: self.local.call(func_with_bad_return_value)) - self.assertEquals( + self.assertEqual( exc.args[0], "cannot unpickle '%s'/'CrazyType'" % (__name__,), ) @@ -91,7 +87,7 @@ def test_aborted_on_local_context_disconnect(self): self.broker.defer(stream.on_disconnect, self.broker) exc = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(exc.args[0], self.router.respondent_disconnect_msg) + self.assertEqual(exc.args[0], self.router.respondent_disconnect_msg) def test_aborted_on_local_broker_shutdown(self): stream = self.router._stream_by_id[self.local.context_id] @@ -101,7 +97,7 @@ def test_aborted_on_local_broker_shutdown(self): self.broker_shutdown = True exc = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(exc.args[0], self.router.respondent_disconnect_msg) + self.assertEqual(exc.args[0], self.router.respondent_disconnect_msg) def test_accepts_returns_context(self): context = self.local.call(func_returns_arg, self.local) @@ -114,10 +110,10 @@ def test_accepts_returns_sender(self): recv = mitogen.core.Receiver(self.router) sender = recv.to_sender() sender2 = self.local.call(func_accepts_returns_sender, sender) - self.assertEquals(sender.context.context_id, + self.assertEqual(sender.context.context_id, sender2.context.context_id) - self.assertEquals(sender.dst_handle, sender2.dst_handle) - self.assertEquals(123, recv.get().unpickle()) + self.assertEqual(sender.dst_handle, sender2.dst_handle) + self.assertEqual(123, recv.get().unpickle()) self.assertRaises(mitogen.core.ChannelError, lambda: recv.get().unpickle()) @@ -132,19 +128,19 @@ def setUp(self): def test_subsequent_calls_produce_same_error(self): chain = self.klass(self.local, pipelined=True) - self.assertEquals('xx', chain.call(func_returns_arg, 'xx')) + self.assertEqual('xx', chain.call(func_returns_arg, 'xx')) chain.call_no_reply(function_that_fails, 'x1') e1 = self.assertRaises(mitogen.core.CallError, lambda: chain.call(function_that_fails, 'x2')) e2 = self.assertRaises(mitogen.core.CallError, lambda: chain.call(func_returns_arg, 'x3')) - self.assertEquals(str(e1), str(e2)) + self.assertEqual(str(e1), str(e2)) def test_unrelated_overlapping_failed_chains(self): c1 = self.klass(self.local, pipelined=True) c2 = self.klass(self.local, pipelined=True) c1.call_no_reply(function_that_fails, 'c1') - self.assertEquals('yes', c2.call(func_returns_arg, 'yes')) + self.assertEqual('yes', c2.call(func_returns_arg, 'yes')) self.assertRaises(mitogen.core.CallError, lambda: c1.call(func_returns_arg, 'yes')) @@ -154,7 +150,7 @@ def test_reset(self): e1 = self.assertRaises(mitogen.core.CallError, lambda: c1.call(function_that_fails, 'x2')) c1.reset() - self.assertEquals('x3', c1.call(func_returns_arg, 'x3')) + self.assertEqual('x3', c1.call(func_returns_arg, 'x3')) class UnsupportedCallablesTest(testlib.RouterMixin, testlib.TestCase): @@ -170,21 +166,17 @@ def test_closures_unsuppored(self): closure = lambda: a e = self.assertRaises(TypeError, lambda: self.local.call(closure)) - self.assertEquals(e.args[0], self.klass.closures_msg) + self.assertEqual(e.args[0], self.klass.closures_msg) def test_lambda_unsupported(self): lam = lambda: None e = self.assertRaises(TypeError, lambda: self.local.call(lam)) - self.assertEquals(e.args[0], self.klass.lambda_msg) + self.assertEqual(e.args[0], self.klass.lambda_msg) def test_instance_method_unsupported(self): class X: def x(): pass e = self.assertRaises(TypeError, lambda: self.local.call(X().x)) - self.assertEquals(e.args[0], self.klass.method_msg) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(e.args[0], self.klass.method_msg) diff --git a/tests/channel_test.py b/tests/channel_test.py index 31d3f35d9..096c05c3d 100644 --- a/tests/channel_test.py +++ b/tests/channel_test.py @@ -1,5 +1,3 @@ -import unittest2 - import mitogen.core import testlib @@ -14,7 +12,3 @@ def test_constructor(self): self.assertEqual(chan.dst_handle, 123) self.assertIsNotNone(chan.handle) self.assertGreater(chan.handle, 0) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/connection_test.py b/tests/connection_test.py index 619594d9c..79e27aee0 100644 --- a/tests/connection_test.py +++ b/tests/connection_test.py @@ -1,12 +1,10 @@ - +import logging import os -import signal import sys import tempfile import threading import time -import unittest2 import testlib import mitogen.core @@ -41,8 +39,8 @@ def thread(): th.join() exc, = result - self.assertTrue(isinstance(exc, mitogen.parent.CancelledError)) - self.assertEquals(mitogen.parent.BROKER_SHUTDOWN_MSG, exc.args[0]) + self.assertIsInstance(exc, mitogen.parent.CancelledError) + self.assertEqual(mitogen.parent.BROKER_SHUTDOWN_MSG, exc.args[0]) @mitogen.core.takes_econtext @@ -56,7 +54,9 @@ def do_detach(econtext): class DetachReapTest(testlib.RouterMixin, testlib.TestCase): def test_subprocess_preserved_on_shutdown(self): c1 = self.router.local() + c1_stream = self.router.stream_by_id(c1.context_id) pid = c1.call(os.getpid) + self.assertEqual(pid, c1_stream.conn.proc.pid) l = mitogen.core.Latch() mitogen.core.listen(c1, 'disconnect', l.put) @@ -66,12 +66,8 @@ def test_subprocess_preserved_on_shutdown(self): self.broker.shutdown() self.broker.join() - os.kill(pid, 0) # succeeds if process still alive + self.assertIsNone(os.kill(pid, 0)) # succeeds if process still alive # now clean up - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) - - -if __name__ == '__main__': - unittest2.main() + c1_stream.conn.proc.terminate() + c1_stream.conn.proc.proc.wait() diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..6adaa30be --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1,3 @@ +# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it. +# TODO Bump dependencies and unconstrain Pip. +setuptools<72 diff --git a/tests/context_test.py b/tests/context_test.py index 4bc4bd2e5..ee9292c46 100644 --- a/tests/context_test.py +++ b/tests/context_test.py @@ -1,6 +1,4 @@ - import pickle -import unittest2 import mitogen.core from mitogen.core import b @@ -21,17 +19,13 @@ def test_mitogen_roundtrip(self): r = mitogen.core.Receiver(self.router) r.to_sender().send(c) c2 = r.get().unpickle() - self.assertEquals(None, c2.router) - self.assertEquals(c.context_id, c2.context_id) - self.assertEquals(c.name, c2.name) + self.assertEqual(None, c2.router) + self.assertEqual(c.context_id, c2.context_id) + self.assertEqual(c.name, c2.name) def test_vanilla_roundtrip(self): c = self.router.local() c2 = pickle.loads(pickle.dumps(c)) - self.assertEquals(None, c2.router) - self.assertEquals(c.context_id, c2.context_id) - self.assertEquals(c.name, c2.name) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(None, c2.router) + self.assertEqual(c.context_id, c2.context_id) + self.assertEqual(c.name, c2.name) diff --git a/tests/create_child_test.py b/tests/create_child_test.py index 26f10d575..57b04b3ff 100644 --- a/tests/create_child_test.py +++ b/tests/create_child_test.py @@ -1,14 +1,9 @@ - import fcntl import os import stat import sys -import time import tempfile -import mock -import unittest2 - import mitogen.core import mitogen.parent from mitogen.core import b @@ -80,7 +75,8 @@ def close_proc(proc): proc.stdin.close() proc.stdout.close() if proc.stderr: - prco.stderr.close() + proc.stderr.close() + proc.proc.wait() def wait_read(fp, n): @@ -100,12 +96,13 @@ def test_stdin(self): lambda proc: proc.stdin.send(b('TEST'))) st = os.fstat(proc.stdin.fileno()) self.assertTrue(stat.S_ISSOCK(st.st_mode)) - self.assertEquals(st.st_dev, info['st_dev']) - self.assertEquals(st.st_mode, _osx_mode(info['st_mode'])) + self.assertEqual(st.st_dev, info['st_dev']) + self.assertEqual(st.st_mode, _osx_mode(info['st_mode'])) flags = fcntl.fcntl(proc.stdin.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) self.assertTrue(info['buf'], 'TEST') self.assertTrue(info['flags'] & os.O_RDWR) + close_proc(proc) class StdoutSockMixin(object): @@ -114,12 +111,13 @@ def test_stdout(self): lambda proc: wait_read(proc.stdout, 4)) st = os.fstat(proc.stdout.fileno()) self.assertTrue(stat.S_ISSOCK(st.st_mode)) - self.assertEquals(st.st_dev, info['st_dev']) - self.assertEquals(st.st_mode, _osx_mode(info['st_mode'])) + self.assertEqual(st.st_dev, info['st_dev']) + self.assertEqual(st.st_mode, _osx_mode(info['st_mode'])) flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') self.assertTrue(info['flags'] & os.O_RDWR) + close_proc(proc) class CreateChildTest(StdinSockMixin, StdoutSockMixin, testlib.TestCase): @@ -128,9 +126,10 @@ class CreateChildTest(StdinSockMixin, StdoutSockMixin, testlib.TestCase): def test_stderr(self): proc, info, _ = run_fd_check(self.func, 2, 'write') st = os.fstat(sys.stderr.fileno()) - self.assertEquals(st.st_dev, info['st_dev']) - self.assertEquals(st.st_mode, info['st_mode']) - self.assertEquals(st.st_ino, info['st_ino']) + self.assertEqual(st.st_dev, info['st_dev']) + self.assertEqual(st.st_mode, info['st_mode']) + self.assertEqual(st.st_ino, info['st_ino']) + close_proc(proc) class CreateChildMergedTest(StdinSockMixin, StdoutSockMixin, @@ -142,13 +141,14 @@ def func(self, *args, **kwargs): def test_stderr(self): proc, info, buf = run_fd_check(self.func, 2, 'write', lambda proc: wait_read(proc.stdout, 4)) - self.assertEquals(None, proc.stderr) + self.assertEqual(None, proc.stderr) st = os.fstat(proc.stdout.fileno()) self.assertTrue(stat.S_ISSOCK(st.st_mode)) flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') self.assertTrue(info['flags'] & os.O_RDWR) + close_proc(proc) class CreateChildStderrPipeTest(StdinSockMixin, StdoutSockMixin, @@ -162,13 +162,14 @@ def test_stderr(self): lambda proc: wait_read(proc.stderr, 4)) st = os.fstat(proc.stderr.fileno()) self.assertTrue(stat.S_ISFIFO(st.st_mode)) - self.assertEquals(st.st_dev, info['st_dev']) - self.assertEquals(st.st_mode, info['st_mode']) + self.assertEqual(st.st_dev, info['st_dev']) + self.assertEqual(st.st_mode, info['st_mode']) flags = fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) self.assertFalse(flags & os.O_WRONLY) self.assertFalse(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') self.assertTrue(info['flags'] & os.O_WRONLY) + close_proc(proc) class TtyCreateChildTest(testlib.TestCase): @@ -191,14 +192,14 @@ def test_dev_tty_open_succeeds(self): ]) mitogen.core.set_block(proc.stdin.fileno()) # read(3) below due to https://bugs.python.org/issue37696 - self.assertEquals(mitogen.core.b('hi\n'), proc.stdin.read(3)) + self.assertEqual(mitogen.core.b('hi\n'), proc.stdin.read(3)) waited_pid, status = os.waitpid(proc.pid, 0) - self.assertEquals(proc.pid, waited_pid) - self.assertEquals(0, status) - self.assertEquals(mitogen.core.b(''), tf.read()) - proc.stdout.close() + self.assertEqual(proc.pid, waited_pid) + self.assertEqual(0, status) + self.assertEqual(mitogen.core.b(''), tf.read()) finally: tf.close() + close_proc(proc) def test_stdin(self): proc, info, _ = run_fd_check(self.func, 0, 'read', @@ -207,14 +208,14 @@ def test_stdin(self): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stdin.fileno())) flags = fcntl.fcntl(proc.stdin.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) self.assertTrue(info['flags'] & os.O_RDWR) self.assertTrue(info['buf'], 'TEST') + close_proc(proc) def test_stdout(self): proc, info, buf = run_fd_check(self.func, 1, 'write', @@ -224,8 +225,7 @@ def test_stdout(self): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stdout.fileno())) flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) @@ -234,6 +234,7 @@ def test_stdout(self): self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') + close_proc(proc) def test_stderr(self): # proc.stderr is None in the parent since there is no separate stderr @@ -245,8 +246,7 @@ def test_stderr(self): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stdout.fileno())) flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) @@ -255,6 +255,7 @@ def test_stderr(self): self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') + close_proc(proc) def test_dev_tty_open_succeeds(self): # In the early days of UNIX, a process that lacked a controlling TTY @@ -271,14 +272,15 @@ def test_dev_tty_open_succeeds(self): proc = self.func([ 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,) ]) - self.assertEquals(mitogen.core.b('hi\n'), wait_read(proc.stdout, 3)) + self.assertEqual(mitogen.core.b('hi\n'), wait_read(proc.stdout, 3)) waited_pid, status = os.waitpid(proc.pid, 0) - self.assertEquals(proc.pid, waited_pid) - self.assertEquals(0, status) - self.assertEquals(mitogen.core.b(''), tf.read()) + self.assertEqual(proc.pid, waited_pid) + self.assertEqual(0, status) + self.assertEqual(mitogen.core.b(''), tf.read()) proc.stdout.close() finally: tf.close() + close_proc(proc) class StderrDiagTtyMixin(object): @@ -291,8 +293,7 @@ def test_stderr(self): self.assertTrue(stat.S_ISCHR(st.st_mode)) self.assertTrue(stat.S_ISCHR(info['st_mode'])) - self.assertTrue(isinstance(info['ttyname'], - mitogen.core.UnicodeType)) + self.assertIsInstance(info['ttyname'], mitogen.core.UnicodeType) self.assertTrue(os.isatty(proc.stderr.fileno())) flags = fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) @@ -301,6 +302,7 @@ def test_stderr(self): self.assertTrue(flags & os.O_RDWR) self.assertTrue(buf, 'TEST') + close_proc(proc) class HybridTtyCreateChildTest(StdinSockMixin, StdoutSockMixin, @@ -319,27 +321,25 @@ def test_stdin(self): lambda proc: proc.transmit_side.write('TEST')) st = os.fstat(proc.transmit_side.fd) self.assertTrue(stat.S_ISFIFO(st.st_mode)) - self.assertEquals(st.st_dev, info['st_dev']) - self.assertEquals(st.st_mode, info['st_mode']) + self.assertEqual(st.st_dev, info['st_dev']) + self.assertEqual(st.st_mode, info['st_mode']) flags = fcntl.fcntl(proc.transmit_side.fd, fcntl.F_GETFL) self.assertTrue(flags & os.O_WRONLY) self.assertTrue(buf, 'TEST') self.assertFalse(info['flags'] & os.O_WRONLY) self.assertFalse(info['flags'] & os.O_RDWR) + close_proc(proc) def test_stdout(self): proc, info, buf = run_fd_check(self.func, 1, 'write', lambda proc: wait_read(proc.receive_side, 4)) st = os.fstat(proc.receive_side.fd) self.assertTrue(stat.S_ISFIFO(st.st_mode)) - self.assertEquals(st.st_dev, info['st_dev']) - self.assertEquals(st.st_mode, info['st_mode']) + self.assertEqual(st.st_dev, info['st_dev']) + self.assertEqual(st.st_mode, info['st_mode']) flags = fcntl.fcntl(proc.receive_side.fd, fcntl.F_GETFL) self.assertFalse(flags & os.O_WRONLY) self.assertFalse(flags & os.O_RDWR) self.assertTrue(info['flags'] & os.O_WRONLY) self.assertTrue(buf, 'TEST') - - -if __name__ == '__main__': - unittest2.main() + close_proc(proc) diff --git a/tests/data/importer/module_finder_testmod/regular_mod.py b/tests/data/importer/module_finder_testmod/regular_mod.py index a7c0403b9..07f8759f4 100644 --- a/tests/data/importer/module_finder_testmod/regular_mod.py +++ b/tests/data/importer/module_finder_testmod/regular_mod.py @@ -1,6 +1,2 @@ - -import sys - - def say_hi(): print('hi') diff --git a/tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py index 5e64ffd63..1dedc8123 100644 --- a/tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py +++ b/tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py @@ -1,3 +1,2 @@ - from __future__ import absolute_import from module_finder_testmod.regular_mod import say_hi diff --git a/tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py index aaefff70e..f5b17796f 100644 --- a/tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py +++ b/tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py @@ -1,2 +1 @@ - from regular_mod import say_hi diff --git a/tests/data/importer/pkg_like_plumbum/colors.py b/tests/data/importer/pkg_like_plumbum/colors.py index bff195554..a36ec98b7 100644 --- a/tests/data/importer/pkg_like_plumbum/colors.py +++ b/tests/data/importer/pkg_like_plumbum/colors.py @@ -1,4 +1,3 @@ - # coding=utf-8 import sys diff --git a/tests/data/importer/simple_pkg/a.py b/tests/data/importer/simple_pkg/a.py index ce5f18872..5d369cad5 100644 --- a/tests/data/importer/simple_pkg/a.py +++ b/tests/data/importer/simple_pkg/a.py @@ -1,4 +1,3 @@ - import simple_pkg.b diff --git a/tests/data/importer/simple_pkg/b.py b/tests/data/importer/simple_pkg/b.py index 5e5d67b2e..c0883bd5a 100644 --- a/tests/data/importer/simple_pkg/b.py +++ b/tests/data/importer/simple_pkg/b.py @@ -1,3 +1,2 @@ - def subtract_one(n): return n - 1 diff --git a/tests/data/importer/simple_pkg/ping.py b/tests/data/importer/simple_pkg/ping.py index 722f7b87a..84330482a 100644 --- a/tests/data/importer/simple_pkg/ping.py +++ b/tests/data/importer/simple_pkg/ping.py @@ -1,5 +1,3 @@ - - def ping(*args): return args diff --git a/tests/data/importer/six_brokenpkg/__init__.py b/tests/data/importer/six_brokenpkg/__init__.py index e5944b83f..32356972a 100644 --- a/tests/data/importer/six_brokenpkg/__init__.py +++ b/tests/data/importer/six_brokenpkg/__init__.py @@ -53,4 +53,4 @@ else: from . import _six as six six_py_file = '{0}.py'.format(os.path.splitext(six.__file__)[0]) -exec(open(six_py_file, 'rb').read()) +with open(six_py_file, 'rb') as f: exec(f.read()) diff --git a/tests/data/importer/webproject/modules_expected_py2x.json b/tests/data/importer/webproject/modules_expected_py2x.json new file mode 100644 index 000000000..4bf0bd399 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py2x.json @@ -0,0 +1,186 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.utils", + "django.utils._os", + "django.utils.deprecation", + "django.utils.encoding", + "django.utils.functional", + "django.utils.inspect", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.six", + "django.utils.version" + ], + "django.db.models": [ + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.checks", + "django.core.checks.caches", + "django.core.checks.compatibility", + "django.core.checks.compatibility.django_1_10", + "django.core.checks.compatibility.django_1_8_0", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.urls", + "django.core.checks.utils", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.base", + "django.db.models.functions.datetime", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.decorators", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.html", + "django.utils.html_parser", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.safestring", + "django.utils.six", + "django.utils.text", + "django.utils.timezone", + "django.utils.translation", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-legacy.json b/tests/data/importer/webproject/modules_expected_py3x-legacy.json new file mode 100644 index 000000000..06e94e620 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-legacy.json @@ -0,0 +1,209 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json new file mode 100644 index 000000000..614a7f9c3 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-new.json @@ -0,0 +1,207 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" +] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/serve_django_app.py b/tests/data/importer/webproject/serve_django_app.py index afa67f0c7..a9ad1e804 100644 --- a/tests/data/importer/webproject/serve_django_app.py +++ b/tests/data/importer/webproject/serve_django_app.py @@ -1,4 +1,3 @@ - import os import sys diff --git a/tests/data/main_with_no_exec_guard.py b/tests/data/main_with_no_exec_guard.py index 153e4743a..7c23a7690 100644 --- a/tests/data/main_with_no_exec_guard.py +++ b/tests/data/main_with_no_exec_guard.py @@ -1,4 +1,3 @@ - import logging import mitogen.master diff --git a/tests/data/minimize_samples/hashbang.py b/tests/data/minimize_samples/hashbang.py index 0245c48f2..1d358671c 100644 --- a/tests/data/minimize_samples/hashbang.py +++ b/tests/data/minimize_samples/hashbang.py @@ -1,3 +1,3 @@ -#/usr/bin/python -c +#/usr/bin/env python -c # coding: utf-8 # comment diff --git a/tests/data/minimize_samples/hashbang_min.py b/tests/data/minimize_samples/hashbang_min.py index 5a74f4815..19836fa51 100644 --- a/tests/data/minimize_samples/hashbang_min.py +++ b/tests/data/minimize_samples/hashbang_min.py @@ -1,3 +1,3 @@ -#/usr/bin/python -c +#/usr/bin/env python -c # coding: utf-8 diff --git a/tests/data/plain_old_module.py b/tests/data/plain_old_module.py index 608f27a5f..7239f76a5 100755 --- a/tests/data/plain_old_module.py +++ b/tests/data/plain_old_module.py @@ -12,7 +12,8 @@ class MyError(Exception): def get_sentinel_value(): # Some proof we're even talking to the mitogen-test Docker image - return open('/etc/sentinel', 'rb').read().decode() + with open('/etc/sentinel', 'rb') as f: + return f.read().decode() def add(x, y): diff --git a/tests/data/python_never_responds.py b/tests/data/python_never_responds.py index 449d85653..9caa5a635 100755 --- a/tests/data/python_never_responds.py +++ b/tests/data/python_never_responds.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am a Python interpreter that sits idle until the connection times out. import time diff --git a/tests/data/stubs/stub-podman.py b/tests/data/stubs/stub-podman.py new file mode 100755 index 000000000..28f551891 --- /dev/null +++ b/tests/data/stubs/stub-podman.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +import sys +import os + +os.environ['ORIGINAL_ARGV'] = repr(sys.argv) +os.environ['THIS_IS_STUB_PODMAN'] = '1' +os.execv(sys.executable, sys.argv[sys.argv.index('--') + 2:]) diff --git a/tests/data/stubs/stub-python.py b/tests/data/stubs/stub-python.py index d9239c2bb..f18b0b683 100755 --- a/tests/data/stubs/stub-python.py +++ b/tests/data/stubs/stub-python.py @@ -2,7 +2,6 @@ import json import os -import subprocess import sys os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv) diff --git a/tests/data/stubs/stub-su.py b/tests/data/stubs/stub-su.py index 1f5e512db..e353a7525 100755 --- a/tests/data/stubs/stub-su.py +++ b/tests/data/stubs/stub-su.py @@ -2,7 +2,6 @@ import json import os -import subprocess import sys import time diff --git a/tests/doas_test.py b/tests/doas_test.py index d1266e2ed..b00355407 100644 --- a/tests/doas_test.py +++ b/tests/doas_test.py @@ -1,11 +1,7 @@ - import os -import mitogen +import mitogen.core import mitogen.doas -import mitogen.parent - -import unittest2 import testlib @@ -19,13 +15,13 @@ def test_okay(self): username='someuser', ) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) - self.assertEquals(argv[:4], [ + self.assertEqual(argv[:4], [ self.doas_path, '-u', 'someuser', '--', ]) - self.assertEquals('1', context.call(os.getenv, 'THIS_IS_STUB_DOAS')) + self.assertEqual('1', context.call(os.getenv, 'THIS_IS_STUB_DOAS')) # TODO: https://github.com/dw/mitogen/issues/694 they are flaky on python 2.6 MODE=mitogen DISTROS=centos7 @@ -41,7 +37,7 @@ def test_okay(self): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.doas(via=ssh) # ) -# self.assertTrue(mitogen.doas.password_required_msg in str(e)) +# self.assertIn(mitogen.doas.password_required_msg, str(e)) # def test_password_incorrect(self): # ssh = self.docker_ssh( @@ -51,7 +47,7 @@ def test_okay(self): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.doas(via=ssh, password='x') # ) -# self.assertTrue(mitogen.doas.password_incorrect_msg in str(e)) +# self.assertIn(mitogen.doas.password_incorrect_msg, str(e)) # def test_password_okay(self): # ssh = self.docker_ssh( @@ -59,8 +55,4 @@ def test_okay(self): # password='has_sudo_password', # ) # context = self.router.doas(via=ssh, password='has_sudo_password') -# self.assertEquals(0, context.call(os.getuid)) - - -if __name__ == '__main__': - unittest2.main() +# self.assertEqual(0, context.call(os.getuid)) diff --git a/tests/docker_test.py b/tests/docker_test.py index b5d157077..6f1b5cca9 100644 --- a/tests/docker_test.py +++ b/tests/docker_test.py @@ -1,9 +1,5 @@ import os -import mitogen - -import unittest2 - import testlib @@ -17,12 +13,8 @@ def test_okay(self): stream = self.router.stream_by_id(context.context_id) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) - self.assertEquals(argv[0], docker_path) - self.assertEquals(argv[1], 'exec') - self.assertEquals(argv[2], '--interactive') - self.assertEquals(argv[3], 'container_name') - self.assertEquals(argv[4], stream.conn.options.python_path) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(argv[0], docker_path) + self.assertEqual(argv[1], 'exec') + self.assertEqual(argv[2], '--interactive') + self.assertEqual(argv[3], 'container_name') + self.assertEqual(argv[4], stream.conn.options.python_path) diff --git a/tests/error_test.py b/tests/error_test.py index 2eefd567b..3138d7cde 100644 --- a/tests/error_test.py +++ b/tests/error_test.py @@ -1,6 +1,3 @@ - -import unittest2 - import testlib import mitogen.core @@ -10,24 +7,20 @@ class ConstructorTest(testlib.TestCase): def test_literal_no_format(self): e = self.klass('error') - self.assertEquals(e.args[0], 'error') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertEqual(e.args[0], 'error') + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_literal_format_chars_present(self): e = self.klass('error%s') - self.assertEquals(e.args[0], 'error%s') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertEqual(e.args[0], 'error%s') + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_format(self): e = self.klass('error%s', 123) - self.assertEquals(e.args[0], 'error123') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) + self.assertEqual(e.args[0], 'error123') + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) def test_bytes_to_unicode(self): e = self.klass(mitogen.core.b('error')) - self.assertEquals(e.args[0], 'error') - self.assertTrue(isinstance(e.args[0], mitogen.core.UnicodeType)) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(e.args[0], 'error') + self.assertIsInstance(e.args[0], mitogen.core.UnicodeType) diff --git a/tests/fakessh_test.py b/tests/fakessh_test.py index e7dde7116..2ad722df6 100644 --- a/tests/fakessh_test.py +++ b/tests/fakessh_test.py @@ -1,16 +1,14 @@ - import os import shutil - -import unittest2 +import unittest import mitogen.fakessh import testlib +@unittest.skip('broken') class RsyncTest(testlib.DockerMixin, testlib.TestCase): - @unittest2.skip('broken') def test_rsync_from_master(self): context = self.docker_ssh_any() @@ -26,7 +24,6 @@ def test_rsync_from_master(self): self.assertTrue(context.call(os.path.exists, '/tmp/data')) self.assertTrue(context.call(os.path.exists, '/tmp/data/simple_pkg/a.py')) - @unittest2.skip('broken') def test_rsync_between_direct_children(self): # master -> SSH -> mitogen__has_sudo_pubkey -> rsync(.ssh) -> master -> # mitogen__has_sudo -> rsync @@ -59,7 +56,3 @@ def test_rsync_between_direct_children(self): pubkey_acct.call(os.path.getsize, '.ssh/authorized_keys'), webapp_acct.call(os.path.getsize, dest_path + '/authorized_keys'), ) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/file_service_test.py b/tests/file_service_test.py index 45b621ac2..b400ab3ea 100644 --- a/tests/file_service_test.py +++ b/tests/file_service_test.py @@ -1,8 +1,5 @@ - import sys -import unittest2 - import mitogen.service import testlib @@ -43,7 +40,7 @@ def test_unauthorized(self): pool.stop() expect = service.unregistered_msg % ('/etc/shadow',) - self.assertTrue(expect in e.args[0]) + self.assertIn(expect, e.args[0]) if sys.platform == 'darwin': ROOT_GROUP = 'wheel' @@ -51,13 +48,13 @@ def test_unauthorized(self): ROOT_GROUP = 'root' def _validate_response(self, resp): - self.assertTrue(isinstance(resp, dict)) - self.assertEquals('root', resp['owner']) - self.assertEquals(self.ROOT_GROUP, resp['group']) - self.assertTrue(isinstance(resp['mode'], int)) - self.assertTrue(isinstance(resp['mtime'], float)) - self.assertTrue(isinstance(resp['atime'], float)) - self.assertTrue(isinstance(resp['size'], int)) + self.assertIsInstance(resp, dict) + self.assertEqual('root', resp['owner']) + self.assertEqual(self.ROOT_GROUP, resp['group']) + self.assertIsInstance(resp['mode'], int) + self.assertIsInstance(resp['mtime'], float) + self.assertIsInstance(resp['atime'], float) + self.assertIsInstance(resp['size'], int) def test_path_authorized(self): recv = mitogen.core.Receiver(self.router) @@ -120,7 +117,7 @@ def test_prefix_authorized_abspath_bad(self): pool.stop() expect = service.unregistered_msg % (path,) - self.assertTrue(expect in e.args[0]) + self.assertIn(expect, e.args[0]) def test_prefix_authorized_abspath_good(self): l1 = self.router.local() @@ -147,8 +144,4 @@ def test_prefix_authorized_abspath_good(self): pool.stop() expect = service.unregistered_msg % (path,) - self.assertTrue(expect in e.args[0]) - - -if __name__ == '__main__': - unittest2.main() + self.assertIn(expect, e.args[0]) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 53f98373a..ad7165b3f 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,8 +1,5 @@ - import subprocess -import unittest2 - import mitogen.parent from mitogen.core import b @@ -39,13 +36,9 @@ def test_valid_syntax(self): stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() - self.assertEquals(0, proc.returncode) - self.assertEquals(stdout, + self.assertEqual(0, proc.returncode) + self.assertEqual(stdout, mitogen.parent.BootstrapProtocol.EC0_MARKER+b('\n')) self.assertIn(b("Error -5 while decompressing data"), stderr) finally: fp.close() - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/fork_test.py b/tests/fork_test.py index 7ca41194f..35acfe137 100644 --- a/tests/fork_test.py +++ b/tests/fork_test.py @@ -1,8 +1,8 @@ - import os import random -import struct +import subprocess import sys +import unittest try: import _ssl @@ -21,22 +21,34 @@ ctypes = None import mitogen -import unittest2 import testlib import plain_old_module def _find_ssl_linux(): - s = testlib.subprocess__check_output(['ldd', _ssl.__file__]) - for line in s.decode().splitlines(): + proc = subprocess.Popen( + ['ldd', _ssl.__file__], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + b_stdout, b_stderr = proc.communicate() + assert proc.returncode == 0 + assert b_stderr.decode() == '' + for line in b_stdout.decode().splitlines(): bits = line.split() if bits[0].startswith('libssl'): return bits[2] + def _find_ssl_darwin(): - s = testlib.subprocess__check_output(['otool', '-l', _ssl.__file__]) - for line in s.decode().splitlines(): + proc = subprocess.Popen( + ['otool', '-l', _ssl.__file__], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + b_stdout, b_stderr = proc.communicate() + assert proc.returncode == 0 + assert b_stderr.decode() == '' + for line in b_stdout.decode().splitlines(): bits = line.split() if bits[0] == 'name' and 'libssl' in bits[1]: return bits[1] @@ -78,7 +90,7 @@ def exercise_importer(n): return simple_pkg.a.subtract_one_add_two(n) -skipIfUnsupported = unittest2.skipIf( +skipIfUnsupported = unittest.skipIf( condition=(not mitogen.fork.FORK_SUPPORTED), reason="mitogen.fork unsupported on this platform" ) @@ -94,7 +106,7 @@ def test_random_module_diverges(self): context = self.router.fork() self.assertNotEqual(context.call(random_random), random_random()) - @unittest2.skipIf( + @unittest.skipIf( condition=LIBSSL_PATH is None or ctypes is None, reason='cant test libssl on this platform', ) @@ -115,7 +127,7 @@ def on_start(econtext): sender = mitogen.core.Sender(econtext.parent, recv.handle) sender.send(123) context = self.router.fork(on_start=on_start) - self.assertEquals(123, recv.get().unpickle()) + self.assertEqual(123, recv.get().unpickle()) ForkTest = skipIfUnsupported(ForkTest) @@ -134,7 +146,7 @@ def test_okay(self): # successfully. In future, we need lots more tests. c1 = self.router.fork() c2 = self.router.fork(via=c1) - self.assertEquals(123, c2.call(ping)) + self.assertEqual(123, c2.call(ping)) def test_importer(self): c1 = self.router.fork(name='c1') @@ -142,7 +154,3 @@ def test_importer(self): self.assertEqual(2, c2.call(exercise_importer, 1)) DoubleChildTest = skipIfUnsupported(DoubleChildTest) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/id_allocation_test.py b/tests/id_allocation_test.py index 1e8d8d1e7..91ff3d4b3 100644 --- a/tests/id_allocation_test.py +++ b/tests/id_allocation_test.py @@ -1,6 +1,3 @@ - -import unittest2 - import testlib import mitogen.core @@ -17,7 +14,7 @@ class SlaveTest(testlib.RouterMixin, testlib.TestCase): def test_slave_allocates_id(self): context = self.router.local() # Master's allocator named the context 1. - self.assertEquals(1, context.context_id) + self.assertEqual(1, context.context_id) # First call from slave allocates a block (2..1001) id_ = context.call(allocate_an_id) @@ -29,8 +26,7 @@ def test_slave_allocates_id(self): # Subsequent master allocation does not collide c2 = self.router.local() - self.assertEquals(1002, c2.context_id) - + self.assertEqual(1002, c2.context_id) -if __name__ == '__main__': - unittest2.main() + context.shutdown() + c2.shutdown() diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index d61d9b3bf..5329fefa2 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -9,7 +9,7 @@ --change 'EXPOSE 22' --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} - public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + {{ container_image_name }} delegate_to: localhost - name: Stop containers diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 353a7d5b9..2972adda7 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -23,10 +23,16 @@ gather_facts: true vars: distro: "{{ansible_distribution}}" - tasks: - - when: ansible_virtualization_type != "docker" - meta: end_play + pre_tasks: + - meta: end_play + when: + - ansible_facts.virtualization_type != "docker" + + roles: + - role: sshd + + tasks: - name: Ensure requisite apt packages are installed apt: name: "{{ common_packages + packages }}" @@ -57,7 +63,7 @@ dnf: dnf clean all command: "{{ clean_command[ansible_pkg_mgr] }}" args: - warn: false + warn: "{{ False if ansible_version.full is version('2.10', '<=', strict=True) else omit }}" - name: Clean up apt package lists shell: rm -rf {{item}}/* @@ -134,10 +140,6 @@ content: | i-am-mitogen-test-docker-image - - copy: - dest: /etc/ssh/banner.txt - src: ../data/docker/ssh_login_banner.txt - - name: Ensure /etc/sudoers.d exists file: state: directory @@ -169,17 +171,6 @@ line: "%wheel ALL=(ALL) ALL" when: ansible_os_family == 'RedHat' - - name: Enable SSH banner - lineinfile: - path: /etc/ssh/sshd_config - line: Banner /etc/ssh/banner.txt - - - name: Allow remote SSH root login - lineinfile: - path: /etc/ssh/sshd_config - line: PermitRootLogin yes - regexp: '.*PermitRootLogin.*' - - name: Allow remote SSH root login lineinfile: path: /etc/pam.d/sshd diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 6224b61a3..ad5a4ef57 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -4,6 +4,8 @@ # WARNING: this creates non-privilged accounts with pre-set passwords! # +- import_playbook: ../ansible/setup/report_controller.yml + - hosts: all gather_facts: true strategy: mitogen_free @@ -37,12 +39,12 @@ normal_users: "{{ lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) - }}" + }}" all_users: "{{ special_users + normal_users - }}" + }}" tasks: - name: Disable non-localhost SSH for Mitogen users when: false @@ -71,6 +73,7 @@ - user: name: "mitogen__{{item}}" shell: /bin/bash + group: staff groups: | {{ ['com.apple.access_ssh'] + @@ -100,6 +103,7 @@ with_items: "{{all_users}}" copy: dest: /var/lib/AccountsService/users/mitogen__{{item}} + mode: u=rw,go= content: | [User] SystemAccount=true @@ -108,7 +112,7 @@ when: ansible_system == 'Linux' and out.stat.exists service: name: accounts-daemon - restarted: true + state: restarted - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" @@ -117,6 +121,9 @@ copy: dest: ~mitogen__slow_user/.{{item}} src: ../data/docker/mitogen__slow_user.profile + owner: mitogen__slow_user + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -125,6 +132,9 @@ copy: dest: ~mitogen__permdenied/.{{item}} src: ../data/docker/mitogen__permdenied.profile + owner: mitogen__permdenied + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -136,20 +146,13 @@ state: directory mode: go= owner: mitogen__has_sudo_pubkey + group: mitogen__group - copy: dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys src: ../data/docker/mitogen__has_sudo_pubkey.key.pub mode: go= owner: mitogen__has_sudo_pubkey - - - name: Install slow profile for one account - block: - - copy: - dest: ~mitogen__slow_user/.profile - src: ../data/docker/mitogen__slow_user.profile - - copy: - dest: ~mitogen__slow_user/.bashrc - src: ../data/docker/mitogen__slow_user.profile + group: mitogen__group - name: Require a TTY for two accounts lineinfile: diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg index 0745aed13..da778786a 100644 --- a/tests/image_prep/ansible.cfg +++ b/tests/image_prep/ansible.cfg @@ -6,6 +6,7 @@ retry_files_enabled = false display_args_to_stdout = True no_target_syslog = True host_key_checking = False +stdout_callback = yaml [inventory] unparsed_is_fatal = true diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 5f182f861..91ff934df 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -4,6 +4,9 @@ common_packages: - strace - sudo +container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" +container_registry: public.ecr.aws/n5z0e8q9 + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/image_prep/macos_localhost.yml b/tests/image_prep/macos_localhost.yml new file mode 100644 index 000000000..c046a2bc5 --- /dev/null +++ b/tests/image_prep/macos_localhost.yml @@ -0,0 +1,7 @@ +- name: Configure macOS + hosts: all + gather_facts: true + strategy: mitogen_free + become: true + roles: + - role: sshd diff --git a/tests/image_prep/py24-build.sh b/tests/image_prep/py24-build.sh index b30cc24b9..b99e36a04 100755 --- a/tests/image_prep/py24-build.sh +++ b/tests/image_prep/py24-build.sh @@ -15,7 +15,7 @@ tar xzvf cpython-2.4.6.tar.gz ( cd cpython-2.4.6 - ./configure --prefix=/usr/local/python2.4.6 --with-pydebug --enable-debug CFLAGS="-g -O0" # --enable-debug + ./configure --prefix=/usr/local/python2.4.6 --with-pydebug --enable-debug CFLAGS="-g -O0" # --enable-debug echo 'zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz' >> Modules/Setup.config make -j 8 sudo make install diff --git a/tests/image_prep/roles/sshd/defaults/main.yml b/tests/image_prep/roles/sshd/defaults/main.yml new file mode 100644 index 000000000..dec0cf0c8 --- /dev/null +++ b/tests/image_prep/roles/sshd/defaults/main.yml @@ -0,0 +1,3 @@ +sshd_config_file: /etc/ssh/sshd_config + +sshd_config__max_auth_tries: 50 diff --git a/tests/data/docker/ssh_login_banner.txt b/tests/image_prep/roles/sshd/files/banner.txt similarity index 100% rename from tests/data/docker/ssh_login_banner.txt rename to tests/image_prep/roles/sshd/files/banner.txt diff --git a/tests/image_prep/roles/sshd/tasks/main.yml b/tests/image_prep/roles/sshd/tasks/main.yml new file mode 100644 index 000000000..837c7d15d --- /dev/null +++ b/tests/image_prep/roles/sshd/tasks/main.yml @@ -0,0 +1,31 @@ +- name: Create login banner + copy: + src: banner.txt + dest: /etc/ssh/banner.txt + mode: u=rw,go=r + +- name: Configure sshd_config + lineinfile: + path: "{{ sshd_config_file }}" + line: "{{ item.line }}" + regexp: "{{ item.regexp }}" + loop: + - line: Banner /etc/ssh/banner.txt + regexp: '^#? *Banner.*' + - line: MaxAuthTries {{ sshd_config__max_auth_tries }} + regexp: '^#? *MaxAuthTries.*' + - line: PermitRootLogin yes + regexp: '.*PermitRootLogin.*' + loop_control: + label: "{{ item.line }}" + register: configure_sshd_result + +- name: Restart sshd + shell: | + launchctl unload /System/Library/LaunchDaemons/ssh.plist + wait 5 + launchctl load -w /System/Library/LaunchDaemons/ssh.plist + changed_when: true + when: + - ansible_facts.distribution == "MacOSX" + - configure_sshd_result is changed diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml index 9aa3285cc..b820e1bc2 100755 --- a/tests/image_prep/setup.yml +++ b/tests/image_prep/setup.yml @@ -1,6 +1,6 @@ #!/usr/bin/env ansible-playbook -- include: _container_create.yml -- include: _container_setup.yml -- include: _user_accounts.yml -- include: _container_finalize.yml +- include_playbook: _container_create.yml +- include_playbook: _container_setup.yml +- include_playbook: _user_accounts.yml +- include_playbook: _container_finalize.yml diff --git a/tests/importer_test.py b/tests/importer_test.py index c796f7d01..33aafe931 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -1,11 +1,13 @@ - import sys import threading import types import zlib +import unittest -import mock -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import mitogen.core import mitogen.utils @@ -44,6 +46,49 @@ def tearDown(self): super(ImporterMixin, self).tearDown() +class InvalidNameTest(ImporterMixin, testlib.TestCase): + modname = 'trailingdot.' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_invalid(self): + self.set_get_module_response(self.response) + self.assertEqual(self.importer.find_spec(self.modname, path=None), None) + + +class MissingModuleTest(ImporterMixin, testlib.TestCase): + modname = 'missing' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') + def test_load_module_missing(self): + self.set_get_module_response(self.response) + self.assertRaises(ImportError, self.importer.load_module, self.modname) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_missing(self): + """ + Importer should optimistically offer itself as a module loader + when there are no disqualifying criteria. + """ + import importlib.machinery + self.set_get_module_response(self.response) + spec = self.importer.find_spec(self.modname, path=None) + self.assertIsInstance(spec, importlib.machinery.ModuleSpec) + self.assertEqual(spec.name, self.modname) + self.assertEqual(spec.loader, self.importer) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_create_module_missing(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + self.assertRaises(ImportError, self.importer.create_module, spec) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModuleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -52,14 +97,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): # 0:fullname 1:pkg_present 2:path 3:compressed 4:related response = (modname, None, path, data, []) - def test_no_such_module(self): - self.set_get_module_response( - # 0:fullname 1:pkg_present 2:path 3:compressed 4:related - (self.modname, None, None, None, None) - ) - self.assertRaises(ImportError, - lambda: self.importer.load_module(self.modname)) - def test_module_added_to_sys_modules(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) @@ -69,7 +106,7 @@ def test_module_added_to_sys_modules(self): def test_module_file_set(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) - self.assertEquals(mod.__file__, 'master:' + self.path) + self.assertEqual(mod.__file__, 'master:' + self.path) def test_module_loader_set(self): self.set_get_module_response(self.response) @@ -82,6 +119,26 @@ def test_module_package_unset(self): self.assertIsNone(mod.__package__) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ModuleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'fake_module' + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -92,9 +149,28 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): def test_module_package_unset(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) - self.assertEquals(mod.__package__, 'mypkg') + self.assertEqual(mod.__package__, 'mypkg') +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class SubmoduleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'mypkg.fake_module' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'mypkg.fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModulePackageTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("func = lambda: 1\n\n")) path = 'fake_pkg/__init__.py' @@ -105,19 +181,19 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase): def test_module_file_set(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) - self.assertEquals(mod.__file__, 'master:' + self.path) + self.assertEqual(mod.__file__, 'master:' + self.path) def test_get_filename(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) filename = mod.__loader__.get_filename(self.modname) - self.assertEquals('master:fake_pkg/__init__.py', filename) + self.assertEqual('master:fake_pkg/__init__.py', filename) def test_get_source(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) source = mod.__loader__.get_source(self.modname) - self.assertEquals(source, + self.assertEqual(source, mitogen.core.to_text(zlib.decompress(self.data))) def test_module_loader_set(self): @@ -128,18 +204,53 @@ def test_module_loader_set(self): def test_module_path_present(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) - self.assertEquals(mod.__path__, []) + self.assertEqual(mod.__path__, []) def test_module_package_set(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) - self.assertEquals(mod.__package__, self.modname) + self.assertEqual(mod.__package__, self.modname) def test_module_data(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) self.assertIsInstance(mod.func, types.FunctionType) - self.assertEquals(mod.func.__module__, self.modname) + self.assertEqual(mod.func.__module__, self.modname) + + +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class PackageSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("func = lambda: 1\n\n")) + path = 'fake_pkg/__init__.py' + modname = 'fake_pkg' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, [], path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_pkg') + #self.assertFalse(hasattr(mod, '__file__')) + + def test_get_filename(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + filename = self.importer.get_filename(self.modname) + self.assertEqual('master:fake_pkg/__init__.py', filename) + + def test_get_source(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + source = self.importer.get_source(self.modname) + self.assertEqual(source, + mitogen.core.to_text(zlib.decompress(self.data))) class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase): @@ -219,9 +330,5 @@ class SelfReplacingModuleTest(testlib.RouterMixin, testlib.TestCase): # issue #590 def test_importer_handles_self_replacement(self): c = self.router.local() - self.assertEquals(0, + self.assertEqual(0, c.call(simple_pkg.imports_replaces_self.subtract_one, 1)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/io_op_test.py b/tests/io_op_test.py index 525a1b121..e0ad65f15 100644 --- a/tests/io_op_test.py +++ b/tests/io_op_test.py @@ -1,9 +1,10 @@ - import errno import select -import mock -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import testlib import mitogen.core @@ -31,10 +32,10 @@ def test_eintr_restarts(self): 'yay', ] rc, disconnected = self.func(py24_mock_fix(m), 'input') - self.assertEquals(rc, 'yay') + self.assertEqual(rc, 'yay') self.assertFalse(disconnected) - self.assertEquals(4, m.call_count) - self.assertEquals(m.mock_calls, [ + self.assertEqual(4, m.call_count) + self.assertEqual(m.mock_calls, [ mock.call('input'), mock.call('input'), mock.call('input'), @@ -59,10 +60,10 @@ def test_disconnection(self): m = mock.Mock() m.side_effect = self.exception_class(self.errno) rc, disconnected = self.func(m, 'input') - self.assertEquals(rc, None) + self.assertEqual(rc, None) self.assertTrue(disconnected) - self.assertEquals(1, m.call_count) - self.assertEquals(m.mock_calls, [ + self.assertEqual(1, m.call_count) + self.assertEqual(m.mock_calls, [ mock.call('input'), ]) @@ -107,9 +108,9 @@ def test_exception(self): m.side_effect = self.exception_class(self.errno) e = self.assertRaises(self.exception_class, lambda: self.func(m, 'input')) - self.assertEquals(e, m.side_effect) - self.assertEquals(1, m.call_count) - self.assertEquals(m.mock_calls, [ + self.assertEqual(e, m.side_effect) + self.assertEqual(1, m.call_count) + self.assertEqual(m.mock_calls, [ mock.call('input'), ]) @@ -122,7 +123,3 @@ class SelectExceptionTest(ExceptionTest, testlib.TestCase): class OsErrorExceptionTest(ExceptionTest, testlib.TestCase): errno = errno.EBADF exception_class = OSError - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/iter_split_test.py b/tests/iter_split_test.py index ee5e97d97..39c11a470 100644 --- a/tests/iter_split_test.py +++ b/tests/iter_split_test.py @@ -1,57 +1,49 @@ - -import mock -import unittest2 +import unittest import mitogen.core -import testlib - -try: - next -except NameError: - def next(it): - return it.next() +from mitogen.core import next -class IterSplitTest(unittest2.TestCase): +class IterSplitTest(unittest.TestCase): func = staticmethod(mitogen.core.iter_split) def test_empty_buffer(self): lst = [] trailer, cont = self.func(buf='', delim='\n', func=lst.append) self.assertTrue(cont) - self.assertEquals('', trailer) - self.assertEquals([], lst) + self.assertEqual('', trailer) + self.assertEqual([], lst) def test_empty_line(self): lst = [] trailer, cont = self.func(buf='\n', delim='\n', func=lst.append) self.assertTrue(cont) - self.assertEquals('', trailer) - self.assertEquals([''], lst) + self.assertEqual('', trailer) + self.assertEqual([''], lst) def test_one_line(self): buf = 'xxxx\n' lst = [] trailer, cont = self.func(buf=buf, delim='\n', func=lst.append) self.assertTrue(cont) - self.assertEquals('', trailer) - self.assertEquals(lst, ['xxxx']) + self.assertEqual('', trailer) + self.assertEqual(lst, ['xxxx']) def test_one_incomplete(self): buf = 'xxxx\nyy' lst = [] trailer, cont = self.func(buf=buf, delim='\n', func=lst.append) self.assertTrue(cont) - self.assertEquals('yy', trailer) - self.assertEquals(lst, ['xxxx']) + self.assertEqual('yy', trailer) + self.assertEqual(lst, ['xxxx']) def test_returns_false_immediately(self): buf = 'xxxx\nyy' func = lambda buf: False trailer, cont = self.func(buf=buf, delim='\n', func=func) self.assertFalse(cont) - self.assertEquals('yy', trailer) + self.assertEqual('yy', trailer) def test_returns_false_second_call(self): buf = 'xxxx\nyy\nzz' @@ -59,8 +51,4 @@ def test_returns_false_second_call(self): func = lambda buf: next(it) trailer, cont = self.func(buf=buf, delim='\n', func=func) self.assertFalse(cont) - self.assertEquals('zz', trailer) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual('zz', trailer) diff --git a/tests/jail_test.py b/tests/jail_test.py index 7239d32fa..5c0ad9469 100644 --- a/tests/jail_test.py +++ b/tests/jail_test.py @@ -1,11 +1,5 @@ - import os -import mitogen -import mitogen.parent - -import unittest2 - import testlib @@ -20,14 +14,10 @@ def test_okay(self): stream = self.router.stream_by_id(context.context_id) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) - self.assertEquals(argv[:4], [ + self.assertEqual(argv[:4], [ self.jexec_path, 'somejail', stream.conn.options.python_path, '-c', ]) - self.assertEquals('1', context.call(os.getenv, 'THIS_IS_STUB_JEXEC')) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual('1', context.call(os.getenv, 'THIS_IS_STUB_JEXEC')) diff --git a/tests/kubectl_test.py b/tests/kubectl_test.py index 0bac30485..2b21cfd68 100644 --- a/tests/kubectl_test.py +++ b/tests/kubectl_test.py @@ -1,11 +1,5 @@ - import os -import mitogen -import mitogen.parent - -import unittest2 - import testlib @@ -19,11 +13,7 @@ def test_okay(self): ) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) - self.assertEquals(argv[0], self.kubectl_path) - self.assertEquals(argv[1], 'exec') - self.assertEquals(argv[2], '-it') - self.assertEquals(argv[3], 'pod_name') - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(argv[0], self.kubectl_path) + self.assertEqual(argv[1], 'exec') + self.assertEqual(argv[2], '-it') + self.assertEqual(argv[3], 'pod_name') diff --git a/tests/latch_test.py b/tests/latch_test.py index 6ae43221d..1fac8219d 100644 --- a/tests/latch_test.py +++ b/tests/latch_test.py @@ -1,9 +1,6 @@ - import sys import threading -import unittest2 - import mitogen.core import testlib @@ -47,19 +44,19 @@ def test_nonempty(self): obj = object() latch = self.klass() latch.put(obj) - self.assertEquals(obj, latch.get()) + self.assertEqual(obj, latch.get()) def test_nonempty_noblock(self): obj = object() latch = self.klass() latch.put(obj) - self.assertEquals(obj, latch.get(block=False)) + self.assertEqual(obj, latch.get(block=False)) def test_nonempty_zero_timeout(self): obj = object() latch = self.klass() latch.put(obj) - self.assertEquals(obj, latch.get(timeout=0)) + self.assertEqual(obj, latch.get(timeout=0)) class ThreadedGetTest(testlib.TestCase): @@ -93,8 +90,8 @@ def test_one_thread(self): self.start_one(lambda: latch.get(timeout=3.0)) latch.put('test') self.join() - self.assertEquals(self.results, ['test']) - self.assertEquals(self.excs, []) + self.assertEqual(self.results, ['test']) + self.assertEqual(self.excs, []) def test_five_threads(self): latch = self.klass() @@ -103,8 +100,8 @@ def test_five_threads(self): for x in range(5): latch.put(x) self.join() - self.assertEquals(sorted(self.results), list(range(5))) - self.assertEquals(self.excs, []) + self.assertEqual(sorted(self.results), list(range(5))) + self.assertEqual(self.excs, []) @@ -114,7 +111,7 @@ class PutTest(testlib.TestCase): def test_put(self): latch = self.klass() latch.put(None) - self.assertEquals(None, latch.get()) + self.assertEqual(None, latch.get()) class CloseTest(testlib.TestCase): @@ -199,9 +196,9 @@ def test_one_thread(self): self.start_one(lambda: latch.get(timeout=3.0)) latch.close() self.join() - self.assertEquals(self.results, [None]) + self.assertEqual(self.results, [None]) for exc in self.excs: - self.assertTrue(isinstance(exc, mitogen.core.LatchError)) + self.assertIsInstance(exc, mitogen.core.LatchError) def test_five_threads(self): latch = self.klass() @@ -209,11 +206,6 @@ def test_five_threads(self): self.start_one(lambda: latch.get(timeout=3.0)) latch.close() self.join() - self.assertEquals(self.results, [None]*5) + self.assertEqual(self.results, [None]*5) for exc in self.excs: - self.assertTrue(isinstance(exc, mitogen.core.LatchError)) - - - -if __name__ == '__main__': - unittest2.main() + self.assertIsInstance(exc, mitogen.core.LatchError) diff --git a/tests/local_test.py b/tests/local_test.py index fe2bd1497..a361777d7 100644 --- a/tests/local_test.py +++ b/tests/local_test.py @@ -1,11 +1,6 @@ - import os import sys -import unittest2 - -import mitogen - import testlib @@ -23,18 +18,18 @@ class ConstructionTest(testlib.RouterMixin, testlib.TestCase): def test_stream_name(self): context = self.router.local() pid = context.call(os.getpid) - self.assertEquals('local.%d' % (pid,), context.name) + self.assertEqual('local.%d' % (pid,), context.name) def test_python_path_inherited(self): context = self.router.local() - self.assertEquals(sys.executable, context.call(get_sys_executable)) + self.assertEqual(sys.executable, context.call(get_sys_executable)) def test_python_path_string(self): context = self.router.local( python_path=self.stub_python_path, ) env = context.call(get_os_environ) - self.assertEquals('1', env['THIS_IS_STUB_PYTHON']) + self.assertEqual('1', env['THIS_IS_STUB_PYTHON']) def test_python_path_list(self): context = self.router.local( @@ -44,11 +39,7 @@ def test_python_path_list(self): sys.executable ] ) - self.assertEquals(sys.executable, context.call(get_sys_executable)) + self.assertEqual(sys.executable, context.call(get_sys_executable)) env = context.call(get_os_environ) - self.assertEquals('magic_first_arg', env['STUB_PYTHON_FIRST_ARG']) - self.assertEquals('1', env['THIS_IS_STUB_PYTHON']) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual('magic_first_arg', env['STUB_PYTHON_FIRST_ARG']) + self.assertEqual('1', env['THIS_IS_STUB_PYTHON']) diff --git a/tests/log_handler_test.py b/tests/log_handler_test.py index 8f4d9dd52..da929426d 100644 --- a/tests/log_handler_test.py +++ b/tests/log_handler_test.py @@ -1,9 +1,12 @@ - import logging -import mock import sys +import unittest + +try: + from unittest import mock +except ImportError: + import mock -import unittest2 import testlib import mitogen.core import mitogen.master @@ -38,8 +41,8 @@ def test_initially_buffered(self): context, handler = self.build() rec = self.record() handler.emit(rec) - self.assertEquals(0, context.send.call_count) - self.assertEquals(1, len(handler._buffer)) + self.assertEqual(0, context.send.call_count) + self.assertEqual(1, len(handler._buffer)) def test_uncork(self): context, handler = self.build() @@ -47,14 +50,14 @@ def test_uncork(self): handler.emit(rec) handler.uncork() - self.assertEquals(1, context.send.call_count) - self.assertEquals(None, handler._buffer) + self.assertEqual(1, context.send.call_count) + self.assertEqual(None, handler._buffer) _, args, _ = context.send.mock_calls[0] msg, = args - self.assertEquals(mitogen.core.FORWARD_LOG, msg.handle) - self.assertEquals(b('name\x0099\x00msg'), msg.data) + self.assertEqual(mitogen.core.FORWARD_LOG, msg.handle) + self.assertEqual(b('name\x0099\x00msg'), msg.data) class StartupTest(testlib.RouterMixin, testlib.TestCase): @@ -66,8 +69,8 @@ def test_earliest_messages_logged(self): c1.shutdown(wait=True) logs = log.stop() - self.assertTrue('Python version is' in logs) - self.assertTrue('Parent is context 0 (master)' in logs) + self.assertIn('Python version is', logs) + self.assertIn('Parent is context 0 (master)', logs) def test_earliest_messages_logged_via(self): c1 = self.router.local(name='c1') @@ -81,16 +84,12 @@ def test_earliest_messages_logged_via(self): c2.shutdown(wait=True) logs = log.stop() - self.assertTrue('Python version is' in logs) + self.assertIn('Python version is', logs) expect = 'Parent is context %s (%s)' % (c1.context_id, 'parent') - self.assertTrue(expect in logs) + self.assertIn(expect, logs) -StartupTest = unittest2.skipIf( +StartupTest = unittest.skipIf( condition=sys.version_info < (2, 7) or sys.version_info >= (3, 6), reason="Message log flaky on Python < 2.7 or >= 3.6" )(StartupTest) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/lxc_test.py b/tests/lxc_test.py index f78846ff7..a613cefae 100644 --- a/tests/lxc_test.py +++ b/tests/lxc_test.py @@ -1,14 +1,9 @@ import os -import mitogen import mitogen.lxc +import mitogen.parent -try: - any -except NameError: - from mitogen.core import any - -import unittest2 +from mitogen.core import any import testlib @@ -27,8 +22,8 @@ def test_okay(self): ) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) - self.assertEquals(argv[0], self.lxc_attach_path) - self.assertTrue('--clear-env' in argv) + self.assertEqual(argv[0], self.lxc_attach_path) + self.assertIn('--clear-env', argv) self.assertTrue(has_subseq(argv, ['--name', 'container_name'])) def test_eof(self): @@ -39,7 +34,3 @@ def test_eof(self): ) ) self.assertTrue(str(e).endswith(mitogen.lxc.Connection.eof_error_hint)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/lxd_test.py b/tests/lxd_test.py index c80f82515..6573e1ba9 100644 --- a/tests/lxd_test.py +++ b/tests/lxd_test.py @@ -1,11 +1,8 @@ import os -import mitogen import mitogen.lxd import mitogen.parent -import unittest2 - import testlib @@ -18,10 +15,10 @@ def test_okay(self): ) argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) - self.assertEquals(argv[0], lxc_path) - self.assertEquals(argv[1], 'exec') - self.assertEquals(argv[2], '--mode=noninteractive') - self.assertEquals(argv[3], 'container_name') + self.assertEqual(argv[0], lxc_path) + self.assertEqual(argv[1], 'exec') + self.assertEqual(argv[2], '--mode=noninteractive') + self.assertEqual(argv[3], 'container_name') def test_eof(self): e = self.assertRaises(mitogen.parent.EofError, @@ -31,7 +28,3 @@ def test_eof(self): ) ) self.assertTrue(str(e).endswith(mitogen.lxd.Connection.eof_error_hint)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/master_test.py b/tests/master_test.py index 31d110139..2af00718f 100644 --- a/tests/master_test.py +++ b/tests/master_test.py @@ -1,7 +1,5 @@ import inspect -import unittest2 - import testlib import mitogen.master @@ -16,16 +14,12 @@ class ScanCodeImportsTest(testlib.TestCase): SIMPLE_EXPECT = [ (level, 'inspect', ()), - (level, 'unittest2', ()), (level, 'testlib', ()), (level, 'mitogen.master', ()), ] def test_simple(self): source_path = inspect.getsourcefile(ScanCodeImportsTest) - co = compile(open(source_path).read(), source_path, 'exec') - self.assertEquals(list(self.func(co)), self.SIMPLE_EXPECT) - - -if __name__ == '__main__': - unittest2.main() + with open(source_path) as f: + co = compile(f.read(), source_path, 'exec') + self.assertEqual(list(self.func(co)), self.SIMPLE_EXPECT) diff --git a/tests/message_test.py b/tests/message_test.py index 79deb2c67..fc50a3270 100644 --- a/tests/message_test.py +++ b/tests/message_test.py @@ -1,9 +1,11 @@ - import sys import struct +import unittest -import mock -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import mitogen.core import mitogen.master @@ -16,46 +18,46 @@ class ConstructorTest(testlib.TestCase): klass = mitogen.core.Message def test_dst_id_default(self): - self.assertEquals(self.klass().dst_id, None) + self.assertEqual(self.klass().dst_id, None) def test_dst_id_explicit(self): - self.assertEquals(self.klass(dst_id=1111).dst_id, 1111) + self.assertEqual(self.klass(dst_id=1111).dst_id, 1111) @mock.patch('mitogen.context_id', 1234) def test_src_id_default(self): - self.assertEquals(self.klass().src_id, 1234) + self.assertEqual(self.klass().src_id, 1234) def test_src_id_explicit(self): - self.assertEquals(self.klass(src_id=4321).src_id, 4321) + self.assertEqual(self.klass(src_id=4321).src_id, 4321) @mock.patch('mitogen.context_id', 5555) def test_auth_id_default(self): - self.assertEquals(self.klass().auth_id, 5555) + self.assertEqual(self.klass().auth_id, 5555) def test_auth_id_explicit(self): - self.assertEquals(self.klass(auth_id=2222).auth_id, 2222) + self.assertEqual(self.klass(auth_id=2222).auth_id, 2222) def test_handle_default(self): - self.assertEquals(self.klass().handle, None) + self.assertEqual(self.klass().handle, None) def test_handle_explicit(self): - self.assertEquals(self.klass(handle=1234).handle, 1234) + self.assertEqual(self.klass(handle=1234).handle, 1234) def test_reply_to_default(self): - self.assertEquals(self.klass().reply_to, None) + self.assertEqual(self.klass().reply_to, None) def test_reply_to_explicit(self): - self.assertEquals(self.klass(reply_to=8888).reply_to, 8888) + self.assertEqual(self.klass(reply_to=8888).reply_to, 8888) def test_data_default(self): m = self.klass() - self.assertEquals(m.data, b('')) - self.assertTrue(isinstance(m.data, mitogen.core.BytesType)) + self.assertEqual(m.data, b('')) + self.assertIsInstance(m.data, mitogen.core.BytesType) def test_data_explicit(self): m = self.klass(data=b('asdf')) - self.assertEquals(m.data, b('asdf')) - self.assertTrue(isinstance(m.data, mitogen.core.BytesType)) + self.assertEqual(m.data, b('asdf')) + self.assertIsInstance(m.data, mitogen.core.BytesType) def test_data_hates_unicode(self): self.assertRaises(Exception, @@ -66,62 +68,62 @@ class PackTest(testlib.TestCase): klass = mitogen.core.Message def test_header_format_sanity(self): - self.assertEquals(self.klass.HEADER_LEN, + self.assertEqual(self.klass.HEADER_LEN, struct.calcsize(self.klass.HEADER_FMT)) def test_header_length_correct(self): s = self.klass(dst_id=123, handle=123).pack() - self.assertEquals(len(s), self.klass.HEADER_LEN) + self.assertEqual(len(s), self.klass.HEADER_LEN) def test_magic(self): s = self.klass(dst_id=123, handle=123).pack() magic, = struct.unpack('>h', s[:2]) - self.assertEquals(self.klass.HEADER_MAGIC, magic) + self.assertEqual(self.klass.HEADER_MAGIC, magic) def test_dst_id(self): s = self.klass(dst_id=123, handle=123).pack() dst_id, = struct.unpack('>L', s[2:6]) - self.assertEquals(123, dst_id) + self.assertEqual(123, dst_id) def test_src_id(self): s = self.klass(src_id=5432, dst_id=123, handle=123).pack() src_id, = struct.unpack('>L', s[6:10]) - self.assertEquals(5432, src_id) + self.assertEqual(5432, src_id) def test_auth_id(self): s = self.klass(auth_id=1919, src_id=5432, dst_id=123, handle=123).pack() auth_id, = struct.unpack('>L', s[10:14]) - self.assertEquals(1919, auth_id) + self.assertEqual(1919, auth_id) def test_handle(self): s = self.klass(dst_id=123, handle=9999).pack() handle, = struct.unpack('>L', s[14:18]) - self.assertEquals(9999, handle) + self.assertEqual(9999, handle) def test_reply_to(self): s = self.klass(dst_id=1231, handle=7777, reply_to=9132).pack() reply_to, = struct.unpack('>L', s[18:22]) - self.assertEquals(9132, reply_to) + self.assertEqual(9132, reply_to) def test_data_length_empty(self): s = self.klass(dst_id=1231, handle=7777).pack() data_length, = struct.unpack('>L', s[22:26]) - self.assertEquals(0, data_length) + self.assertEqual(0, data_length) def test_data_length_present(self): s = self.klass(dst_id=1231, handle=7777, data=b('hello')).pack() data_length, = struct.unpack('>L', s[22:26]) - self.assertEquals(5, data_length) + self.assertEqual(5, data_length) def test_data_empty(self): s = self.klass(dst_id=1231, handle=7777).pack() data = s[26:] - self.assertEquals(b(''), data) + self.assertEqual(b(''), data) def test_data_present(self): s = self.klass(dst_id=11, handle=77, data=b('hello')).pack() data = s[26:] - self.assertEquals(b('hello'), data) + self.assertEqual(b('hello'), data) class IsDeadTest(testlib.TestCase): @@ -141,15 +143,15 @@ class DeadTest(testlib.TestCase): def test_no_reason(self): msg = self.klass.dead() - self.assertEquals(msg.reply_to, mitogen.core.IS_DEAD) + self.assertEqual(msg.reply_to, mitogen.core.IS_DEAD) self.assertTrue(msg.is_dead) - self.assertEquals(msg.data, b('')) + self.assertEqual(msg.data, b('')) def test_with_reason(self): msg = self.klass.dead(reason=u'oh no') - self.assertEquals(msg.reply_to, mitogen.core.IS_DEAD) + self.assertEqual(msg.reply_to, mitogen.core.IS_DEAD) self.assertTrue(msg.is_dead) - self.assertEquals(msg.data, b('oh no')) + self.assertEqual(msg.data, b('oh no')) class EvilObject(object): @@ -168,9 +170,9 @@ def roundtrip(self, v, router=None): def test_bool(self): for b in True, False: - self.assertEquals(b, self.roundtrip(b)) + self.assertEqual(b, self.roundtrip(b)) - @unittest2.skipIf(condition=sys.version_info < (2, 6), + @unittest.skipIf(condition=sys.version_info < (2, 6), reason='bytearray missing on <2.6') def test_bytearray(self): ba = bytearray(b('123')) @@ -180,48 +182,48 @@ def test_bytearray(self): def test_bytes(self): by = b('123') - self.assertEquals(by, self.roundtrip(by)) + self.assertEqual(by, self.roundtrip(by)) def test_dict(self): d = {1: 2, u'a': 3, b('b'): 4, 'c': {}} roundtrip = self.roundtrip(d) - self.assertEquals(d, roundtrip) - self.assertTrue(isinstance(roundtrip, dict)) + self.assertEqual(d, roundtrip) + self.assertIsInstance(roundtrip, dict) for k in d: - self.assertTrue(isinstance(roundtrip[k], type(d[k]))) + self.assertIsInstance(roundtrip[k], type(d[k])) def test_int(self): - self.assertEquals(123, self.klass.pickled(123).unpickle()) + self.assertEqual(123, self.klass.pickled(123).unpickle()) def test_list(self): l = [1, u'b', b('c')] roundtrip = self.roundtrip(l) - self.assertTrue(isinstance(roundtrip, list)) - self.assertEquals(l, roundtrip) + self.assertIsInstance(roundtrip, list) + self.assertEqual(l, roundtrip) for k in range(len(l)): - self.assertTrue(isinstance(roundtrip[k], type(l[k]))) + self.assertIsInstance(roundtrip[k], type(l[k])) - @unittest2.skipIf(condition=sys.version_info > (3, 0), + @unittest.skipIf(condition=sys.version_info > (3, 0), reason='long missing in >3.x') def test_long(self): l = long(0xffffffffffff) roundtrip = self.roundtrip(l) - self.assertEquals(l, roundtrip) - self.assertTrue(isinstance(roundtrip, long)) + self.assertEqual(l, roundtrip) + self.assertIsInstance(roundtrip, long) def test_tuple(self): l = (1, u'b', b('c')) roundtrip = self.roundtrip(l) - self.assertEquals(l, roundtrip) - self.assertTrue(isinstance(roundtrip, tuple)) + self.assertEqual(l, roundtrip) + self.assertIsInstance(roundtrip, tuple) for k in range(len(l)): - self.assertTrue(isinstance(roundtrip[k], type(l[k]))) + self.assertIsInstance(roundtrip[k], type(l[k])) def test_unicode(self): u = u'abcd' roundtrip = self.roundtrip(u) - self.assertEquals(u, roundtrip) - self.assertTrue(isinstance(roundtrip, mitogen.core.UnicodeType)) + self.assertEqual(u, roundtrip) + self.assertIsInstance(roundtrip, mitogen.core.UnicodeType) #### custom types. see also: types_test.py, call_error_test.py @@ -232,40 +234,40 @@ def test_unicode(self): def test_blob_nonempty(self): v = mitogen.core.Blob(b('dave')) roundtrip = self.roundtrip(v) - self.assertTrue(isinstance(roundtrip, mitogen.core.Blob)) - self.assertEquals(b('dave'), roundtrip) + self.assertIsInstance(roundtrip, mitogen.core.Blob) + self.assertEqual(b('dave'), roundtrip) def test_blob_empty(self): v = mitogen.core.Blob(b('')) roundtrip = self.roundtrip(v) - self.assertTrue(isinstance(roundtrip, mitogen.core.Blob)) - self.assertEquals(b(''), v) + self.assertIsInstance(roundtrip, mitogen.core.Blob) + self.assertEqual(b(''), v) def test_secret_nonempty(self): s = mitogen.core.Secret(u'dave') roundtrip = self.roundtrip(s) - self.assertTrue(isinstance(roundtrip, mitogen.core.Secret)) - self.assertEquals(u'dave', roundtrip) + self.assertIsInstance(roundtrip, mitogen.core.Secret) + self.assertEqual(u'dave', roundtrip) def test_secret_empty(self): s = mitogen.core.Secret(u'') roundtrip = self.roundtrip(s) - self.assertTrue(isinstance(roundtrip, mitogen.core.Secret)) - self.assertEquals(u'', roundtrip) + self.assertIsInstance(roundtrip, mitogen.core.Secret) + self.assertEqual(u'', roundtrip) def test_call_error(self): ce = mitogen.core.CallError('nope') ce2 = self.assertRaises(mitogen.core.CallError, lambda: self.roundtrip(ce)) - self.assertEquals(ce.args[0], ce2.args[0]) + self.assertEqual(ce.args[0], ce2.args[0]) def test_context(self): router = mitogen.master.Router() try: c = router.context_by_id(1234) roundtrip = self.roundtrip(c) - self.assertTrue(isinstance(roundtrip, mitogen.core.Context)) - self.assertEquals(c.context_id, 1234) + self.assertIsInstance(roundtrip, mitogen.core.Context) + self.assertEqual(c.context_id, 1234) finally: router.broker.shutdown() router.broker.join() @@ -276,9 +278,9 @@ def test_sender(self): recv = mitogen.core.Receiver(router) sender = recv.to_sender() roundtrip = self.roundtrip(sender, router=router) - self.assertTrue(isinstance(roundtrip, mitogen.core.Sender)) - self.assertEquals(roundtrip.context.context_id, mitogen.context_id) - self.assertEquals(roundtrip.dst_handle, sender.dst_handle) + self.assertIsInstance(roundtrip, mitogen.core.Sender) + self.assertEqual(roundtrip.context.context_id, mitogen.context_id) + self.assertEqual(roundtrip.dst_handle, sender.dst_handle) finally: router.broker.shutdown() router.broker.join() @@ -299,15 +301,15 @@ def test_reply_calls_router_route(self): msg = self.klass(src_id=1234, reply_to=9191) router = mock.Mock() msg.reply(123, router=router) - self.assertEquals(1, router.route.call_count) + self.assertEqual(1, router.route.call_count) def test_reply_pickles_object(self): msg = self.klass(src_id=1234, reply_to=9191) router = mock.Mock() msg.reply(123, router=router) _, (reply,), _ = router.route.mock_calls[0] - self.assertEquals(reply.dst_id, 1234) - self.assertEquals(reply.unpickle(), 123) + self.assertEqual(reply.dst_id, 1234) + self.assertEqual(reply.unpickle(), 123) def test_reply_uses_preformatted_message(self): msg = self.klass(src_id=1234, reply_to=9191) @@ -315,23 +317,23 @@ def test_reply_uses_preformatted_message(self): my_reply = mitogen.core.Message.pickled(4444) msg.reply(my_reply, router=router) _, (reply,), _ = router.route.mock_calls[0] - self.assertTrue(my_reply is reply) - self.assertEquals(reply.dst_id, 1234) - self.assertEquals(reply.unpickle(), 4444) + self.assertIs(my_reply, reply) + self.assertEqual(reply.dst_id, 1234) + self.assertEqual(reply.unpickle(), 4444) def test_reply_sets_dst_id(self): msg = self.klass(src_id=1234, reply_to=9191) router = mock.Mock() msg.reply(123, router=router) _, (reply,), _ = router.route.mock_calls[0] - self.assertEquals(reply.dst_id, 1234) + self.assertEqual(reply.dst_id, 1234) def test_reply_sets_handle(self): msg = self.klass(src_id=1234, reply_to=9191) router = mock.Mock() msg.reply(123, router=router) _, (reply,), _ = router.route.mock_calls[0] - self.assertEquals(reply.handle, 9191) + self.assertEqual(reply.handle, 9191) class UnpickleTest(testlib.TestCase): @@ -343,13 +345,13 @@ def test_throw(self): m = self.klass.pickled(ce) ce2 = self.assertRaises(mitogen.core.CallError, lambda: m.unpickle()) - self.assertEquals(ce.args[0], ce2.args[0]) + self.assertEqual(ce.args[0], ce2.args[0]) def test_no_throw(self): ce = mitogen.core.CallError('nope') m = self.klass.pickled(ce) ce2 = m.unpickle(throw=False) - self.assertEquals(ce.args[0], ce2.args[0]) + self.assertEqual(ce.args[0], ce2.args[0]) def test_throw_dead(self): m = self.klass.pickled('derp', reply_to=mitogen.core.IS_DEAD) @@ -358,7 +360,7 @@ def test_throw_dead(self): def test_no_throw_dead(self): m = self.klass.pickled('derp', reply_to=mitogen.core.IS_DEAD) - self.assertEquals('derp', m.unpickle(throw_dead=False)) + self.assertEqual('derp', m.unpickle(throw_dead=False)) class UnpickleCompatTest(testlib.TestCase): @@ -539,7 +541,3 @@ class ReprTest(testlib.TestCase): def test_repr(self): # doesn't crash repr(self.klass.pickled('test')) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/minify_test.py b/tests/minify_test.py index 156098966..97a51ed19 100644 --- a/tests/minify_test.py +++ b/tests/minify_test.py @@ -3,8 +3,6 @@ import pprint import sys -import unittest2 - import mitogen.minify import testlib @@ -71,7 +69,7 @@ def _test_syntax_valid(self, minified, name): compile(minified, name, 'exec') def _test_line_counts_match(self, original, minified): - self.assertEquals(original.count('\n'), + self.assertEqual(original.count('\n'), minified.count('\n')) def _test_non_blank_lines_match(self, name, original, minified): @@ -112,7 +110,3 @@ def test_minify_all(self): self._test_syntax_valid(minified, name) self._test_line_counts_match(original, minified) self._test_non_blank_lines_match(name, original, minified) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/mitogen_protocol_test.py b/tests/mitogen_protocol_test.py index 834fb4374..46a0cd65b 100644 --- a/tests/mitogen_protocol_test.py +++ b/tests/mitogen_protocol_test.py @@ -1,6 +1,7 @@ - -import unittest2 -import mock +try: + from unittest import mock +except ImportError: + import mock import mitogen.core @@ -25,10 +26,6 @@ def test_corruption(self): protocol.on_receive(broker, junk) capture.stop() - self.assertEquals(1, stream.on_disconnect.call_count) + self.assertEqual(1, stream.on_disconnect.call_count) expect = self.klass.corrupt_msg % (stream.name, junk) - self.assertTrue(expect in capture.raw()) - - -if __name__ == '__main__': - unittest2.main() + self.assertIn(expect, capture.raw()) diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index ac3bfe6c1..67e937ed0 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -1,14 +1,13 @@ import inspect +import json import os import sys - -import unittest2 +import unittest import mitogen.master from mitogen.core import b import testlib -from testlib import MODS_DIR class ConstructorTest(testlib.TestCase): @@ -22,7 +21,7 @@ class ReprTest(testlib.TestCase): klass = mitogen.master.ModuleFinder def test_simple(self): - self.assertEquals('ModuleFinder()', repr(self.klass())) + self.assertEqual('ModuleFinder()', repr(self.klass())) class IsStdlibNameTest(testlib.TestCase): @@ -59,21 +58,21 @@ def call(self, fullname): return self.klass().find(fullname) def test_builtin(self): - self.assertEquals(None, self.call('sys')) + self.assertEqual(None, self.call('sys')) def test_not_main(self): - self.assertEquals(None, self.call('mitogen')) + self.assertEqual(None, self.call('mitogen')) def test_main(self): import __main__ path, source, is_pkg = self.call('__main__') - self.assertTrue(path is not None) + self.assertIsNotNone(path) self.assertTrue(os.path.exists(path)) - self.assertEquals(path, __main__.__file__) + self.assertEqual(path, __main__.__file__) fp = open(path, 'rb') try: - self.assertEquals(source, fp.read()) + self.assertEqual(source, fp.read()) finally: fp.close() self.assertFalse(is_pkg) @@ -87,24 +86,24 @@ def call(self, fullname): def test_empty_source_pkg(self): path, src, is_pkg = self.call('module_finder_testmod') - self.assertEquals(path, - os.path.join(MODS_DIR, 'module_finder_testmod/__init__.py')) - self.assertEquals(mitogen.core.b(''), src) + self.assertEqual(path, + os.path.join(testlib.MODS_DIR, 'module_finder_testmod/__init__.py')) + self.assertEqual(mitogen.core.b(''), src) self.assertTrue(is_pkg) def test_empty_source_module(self): path, src, is_pkg = self.call('module_finder_testmod.empty_mod') - self.assertEquals(path, - os.path.join(MODS_DIR, 'module_finder_testmod/empty_mod.py')) - self.assertEquals(mitogen.core.b(''), src) + self.assertEqual(path, + os.path.join(testlib.MODS_DIR, 'module_finder_testmod/empty_mod.py')) + self.assertEqual(mitogen.core.b(''), src) self.assertFalse(is_pkg) def test_regular_mod(self): from module_finder_testmod import regular_mod path, src, is_pkg = self.call('module_finder_testmod.regular_mod') - self.assertEquals(path, - os.path.join(MODS_DIR, 'module_finder_testmod/regular_mod.py')) - self.assertEquals(mitogen.core.to_text(src), + self.assertEqual(path, + os.path.join(testlib.MODS_DIR, 'module_finder_testmod/regular_mod.py')) + self.assertEqual(mitogen.core.to_text(src), inspect.getsource(regular_mod)) self.assertFalse(is_pkg) @@ -118,14 +117,15 @@ def call(self, fullname): def test_main(self): import __main__ path, src, is_pkg = self.call('__main__') - self.assertEquals(path, __main__.__file__) + self.assertEqual(path, __main__.__file__) # linecache adds a line ending to the final line if one is missing. - actual_src = open(path, 'rb').read() + with open(path, 'rb') as f: + actual_src = f.read() if actual_src[-1:] != b('\n'): actual_src += b('\n') - self.assertEquals(src, actual_src) + self.assertEqual(src, actual_src) self.assertFalse(is_pkg) def test_dylib_fails(self): @@ -140,9 +140,7 @@ def test_builtin_fails(self): self.assertIsNone(tup) -class GetModuleViaParentEnumerationTest(testlib.TestCase): - klass = mitogen.master.ParentEnumerationMethod - +class ParentEnumerationMixin(object): def call(self, fullname): return self.klass().find(fullname) @@ -165,66 +163,81 @@ def test_plumbum_colors_like_pkg_succeeds(self): # plumbum has been eating too many rainbow-colored pills import pkg_like_plumbum.colors path, src, is_pkg = self.call('pkg_like_plumbum.colors') - modpath = os.path.join(MODS_DIR, 'pkg_like_plumbum/colors.py') - self.assertEquals(path, modpath) + modpath = os.path.join(testlib.MODS_DIR, 'pkg_like_plumbum/colors.py') + self.assertEqual(path, modpath) - self.assertEquals(src, open(modpath, 'rb').read()) + with open(modpath, 'rb') as f: + self.assertEqual(src, f.read()) self.assertFalse(is_pkg) def test_ansible_module_utils_distro_succeeds(self): # #590: a package that turns itself into a module. import pkg_like_ansible.module_utils.distro as d - self.assertEquals(d.I_AM, "the module that replaced the package") - self.assertEquals( + self.assertEqual(d.I_AM, "the module that replaced the package") + self.assertEqual( sys.modules['pkg_like_ansible.module_utils.distro'].__name__, 'pkg_like_ansible.module_utils.distro._distro' ) # ensure we can resolve the subpackage. path, src, is_pkg = self.call('pkg_like_ansible.module_utils.distro') - modpath = os.path.join(MODS_DIR, + modpath = os.path.join(testlib.MODS_DIR, 'pkg_like_ansible/module_utils/distro/__init__.py') - self.assertEquals(path, modpath) - self.assertEquals(src, open(modpath, 'rb').read()) - self.assertEquals(is_pkg, True) + self.assertEqual(path, modpath) + with open(modpath, 'rb') as f: + self.assertEqual(src, f.read()) + self.assertEqual(is_pkg, True) # ensure we can resolve a child of the subpackage. path, src, is_pkg = self.call( 'pkg_like_ansible.module_utils.distro._distro' ) - modpath = os.path.join(MODS_DIR, + modpath = os.path.join(testlib.MODS_DIR, 'pkg_like_ansible/module_utils/distro/_distro.py') - self.assertEquals(path, modpath) - self.assertEquals(src, open(modpath, 'rb').read()) - self.assertEquals(is_pkg, False) + self.assertEqual(path, modpath) + with open(modpath, 'rb') as f: + self.assertEqual(src, f.read()) + self.assertEqual(is_pkg, False) def test_ansible_module_utils_system_distro_succeeds(self): # #590: a package that turns itself into a module. # #590: a package that turns itself into a module. import pkg_like_ansible.module_utils.sys_distro as d - self.assertEquals(d.I_AM, "the system module that replaced the subpackage") - self.assertEquals( + self.assertEqual(d.I_AM, "the system module that replaced the subpackage") + self.assertEqual( sys.modules['pkg_like_ansible.module_utils.sys_distro'].__name__, 'system_distro' ) # ensure we can resolve the subpackage. path, src, is_pkg = self.call('pkg_like_ansible.module_utils.sys_distro') - modpath = os.path.join(MODS_DIR, + modpath = os.path.join(testlib.MODS_DIR, 'pkg_like_ansible/module_utils/sys_distro/__init__.py') - self.assertEquals(path, modpath) - self.assertEquals(src, open(modpath, 'rb').read()) - self.assertEquals(is_pkg, True) + self.assertEqual(path, modpath) + with open(modpath, 'rb') as f: + self.assertEqual(src, f.read()) + self.assertEqual(is_pkg, True) # ensure we can resolve a child of the subpackage. path, src, is_pkg = self.call( 'pkg_like_ansible.module_utils.sys_distro._distro' ) - modpath = os.path.join(MODS_DIR, + modpath = os.path.join(testlib.MODS_DIR, 'pkg_like_ansible/module_utils/sys_distro/_distro.py') - self.assertEquals(path, modpath) - self.assertEquals(src, open(modpath, 'rb').read()) - self.assertEquals(is_pkg, False) + self.assertEqual(path, modpath) + with open(modpath, 'rb') as f: + self.assertEqual(src, f.read()) + self.assertEqual(is_pkg, False) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4') +class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentImpEnumerationMethod + + +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentSpecEnumerationMethod class ResolveRelPathTest(testlib.TestCase): @@ -234,21 +247,21 @@ def call(self, fullname, level): return self.klass().resolve_relpath(fullname, level) def test_empty(self): - self.assertEquals('', self.call('', 0)) - self.assertEquals('', self.call('', 1)) - self.assertEquals('', self.call('', 2)) + self.assertEqual('', self.call('', 0)) + self.assertEqual('', self.call('', 1)) + self.assertEqual('', self.call('', 2)) def test_absolute(self): - self.assertEquals('', self.call('email.utils', 0)) + self.assertEqual('', self.call('email.utils', 0)) def test_rel1(self): - self.assertEquals('email.', self.call('email.utils', 1)) + self.assertEqual('email.', self.call('email.utils', 1)) def test_rel2(self): - self.assertEquals('', self.call('email.utils', 2)) + self.assertEqual('', self.call('email.utils', 2)) def test_rel_overflow(self): - self.assertEquals('', self.call('email.utils', 3)) + self.assertEqual('', self.call('email.utils', 3)) class FakeSshTest(testlib.TestCase): @@ -260,10 +273,9 @@ def call(self, fullname): def test_simple(self): import mitogen.fakessh related = self.call('mitogen.fakessh') - self.assertEquals(related, [ + self.assertEqual(related, [ 'mitogen', 'mitogen.core', - 'mitogen.master', 'mitogen.parent', ]) @@ -277,8 +289,6 @@ def call(self, fullname): SIMPLE_EXPECT = set([ u'mitogen', u'mitogen.core', - u'mitogen.master', - u'mitogen.minify', u'mitogen.parent', ]) @@ -292,205 +302,68 @@ def call(self, fullname): def test_simple(self): import mitogen.fakessh related = self.call('mitogen.fakessh') - self.assertEquals(set(related), self.SIMPLE_EXPECT) - - -if sys.version_info > (2, 6): - class DjangoMixin(object): - WEBPROJECT_PATH = os.path.join(MODS_DIR, 'webproject') - - # TODO: rip out Django and replace with a static tree of weird imports - # that don't depend on .. Django! The hack below is because the version - # of Django we need to test against 2.6 doesn't actually run on 3.6. - # But we don't care, we just need to be able to import it. - # - # File "django/utils/html_parser.py", line 12, in - # AttributeError: module 'html.parser' has no attribute - # 'HTMLParseError' - # - from django.utils.six.moves import html_parser as _html_parser - _html_parser.HTMLParseError = Exception - - @classmethod - def setUpClass(cls): - super(DjangoMixin, cls).setUpClass() - sys.path.append(cls.WEBPROJECT_PATH) - os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.WEBPROJECT_PATH) - del os.environ['DJANGO_SETTINGS_MODULE'] - super(DjangoMixin, cls).tearDownClass() - - - class FindRelatedImportsTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - - def call(self, fullname): - return self.klass().find_related_imports(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEquals(related, [ - 'django', - 'django.core', - 'django.core.signals', - 'django.db.utils', - 'django.utils.functional', - ]) - - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.maxDiff=None - self.assertEquals(related, [ - u'django', - u'django.core.exceptions', - u'django.db', - u'django.db.models', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.query', - u'django.db.models.signals', - ]) - - - class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - maxDiff = None - - def call(self, fullname): - return self.klass().find_related(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEquals(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.signals', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.utils', - u'django.utils._os', - u'django.utils.encoding', - u'django.utils.functional', - u'django.utils.importlib', - u'django.utils.module_loading', - u'django.utils.six', - ]) - - @unittest2.skipIf( - condition=(sys.version_info >= (3, 0)), - reason='broken due to ancient vendored six.py' - ) - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.assertEquals(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.files', - u'django.core.files.base', - u'django.core.files.images', - u'django.core.files.locks', - u'django.core.files.move', - u'django.core.files.storage', - u'django.core.files.utils', - u'django.core.signals', - u'django.core.validators', - u'django.db', - u'django.db.backends', - u'django.db.backends.signals', - u'django.db.backends.util', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.constants', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.proxy', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.options', - u'django.db.models.query', - u'django.db.models.query_utils', - u'django.db.models.related', - u'django.db.models.signals', - u'django.db.models.sql', - u'django.db.models.sql.aggregates', - u'django.db.models.sql.constants', - u'django.db.models.sql.datastructures', - u'django.db.models.sql.expressions', - u'django.db.models.sql.query', - u'django.db.models.sql.subqueries', - u'django.db.models.sql.where', - u'django.db.transaction', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.forms', - u'django.forms.fields', - u'django.forms.forms', - u'django.forms.formsets', - u'django.forms.models', - u'django.forms.util', - u'django.forms.widgets', - u'django.utils', - u'django.utils._os', - u'django.utils.crypto', - u'django.utils.datastructures', - u'django.utils.dateformat', - u'django.utils.dateparse', - u'django.utils.dates', - u'django.utils.datetime_safe', - u'django.utils.decorators', - u'django.utils.deprecation', - u'django.utils.encoding', - u'django.utils.formats', - u'django.utils.functional', - u'django.utils.html', - u'django.utils.html_parser', - u'django.utils.importlib', - u'django.utils.ipv6', - u'django.utils.itercompat', - u'django.utils.module_loading', - u'django.utils.numberformat', - u'django.utils.safestring', - u'django.utils.six', - u'django.utils.text', - u'django.utils.timezone', - u'django.utils.translation', - u'django.utils.tree', - u'django.utils.tzinfo', - u'pytz', - u'pytz.exceptions', - u'pytz.lazy', - u'pytz.tzfile', - u'pytz.tzinfo', - ]) - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(set(related), self.SIMPLE_EXPECT) + + +class DjangoMixin(object): + WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') + + @classmethod + def modules_expected_path(cls): + if sys.version_info[0:2] < (3, 0): + modules_expected_filename = 'modules_expected_py2x.json' + elif sys.version_info[0:2] <= (3, 6): + modules_expected_filename = 'modules_expected_py3x-legacy.json' + elif sys.version_info[0:2] >= (3, 10): + modules_expected_filename = 'modules_expected_py3x-new.json' + return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename) + + @classmethod + def setUpClass(cls): + super(DjangoMixin, cls).setUpClass() + sys.path.append(cls.WEBPROJECT_PATH) + os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' + with open(cls.modules_expected_path(), 'rb') as f: + cls.MODULES_EXPECTED = json.load(f) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.WEBPROJECT_PATH) + del os.environ['DJANGO_SETTINGS_MODULE'] + super(DjangoMixin, cls).tearDownClass() + + +class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db') + expected = self.MODULES_EXPECTED['find_related']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db.models') + expected = self.MODULES_EXPECTED['find_related']['django.db.models'] + self.assertEqual(related, expected) + + +class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db.models') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models'] + self.assertEqual(related, expected) diff --git a/tests/nested_test.py b/tests/nested_test.py index 668c21e3d..718c2eeaa 100644 --- a/tests/nested_test.py +++ b/tests/nested_test.py @@ -1,7 +1,5 @@ import os -import unittest2 - import testlib @@ -13,7 +11,3 @@ def test_nested(self): pid = context.call(os.getpid) self.assertIsInstance(pid, int) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/os_fork_test.py b/tests/os_fork_test.py index 14ea8465c..e910a26a3 100644 --- a/tests/os_fork_test.py +++ b/tests/os_fork_test.py @@ -1,6 +1,4 @@ - import testlib -import unittest2 import mitogen.os_fork import mitogen.service @@ -15,7 +13,7 @@ def ping(self, latch): def test_cork_broker(self): latch = mitogen.core.Latch() self.broker.defer(self.ping, latch) - self.assertEquals('pong', latch.get()) + self.assertEqual('pong', latch.get()) corker = self.klass(brokers=(self.broker,)) corker.cork() @@ -25,14 +23,14 @@ def test_cork_broker(self): self.assertRaises(mitogen.core.TimeoutError, lambda: latch.get(timeout=0.5)) corker.uncork() - self.assertEquals('pong', latch.get()) + self.assertEqual('pong', latch.get()) def test_cork_pool(self): pool = mitogen.service.Pool(self.router, services=(), size=4) try: latch = mitogen.core.Latch() pool.defer(self.ping, latch) - self.assertEquals('pong', latch.get()) + self.assertEqual('pong', latch.get()) corker = self.klass(pools=(pool,)) corker.cork() @@ -42,11 +40,6 @@ def test_cork_pool(self): self.assertRaises(mitogen.core.TimeoutError, lambda: latch.get(timeout=0.5)) corker.uncork() - self.assertEquals('pong', latch.get()) + self.assertEqual('pong', latch.get()) finally: pool.stop(join=True) - - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/parent_test.py b/tests/parent_test.py index d6efe998c..558a89b6c 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -2,15 +2,15 @@ import fcntl import os import signal -import subprocess import sys -import tempfile import time +import unittest -import mock -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import testlib -from testlib import Popen__terminate import mitogen.core import mitogen.parent @@ -72,20 +72,20 @@ def test_slashes(self, mock_gethostname, mock_getuser, mock_getpid): mock_gethostname.return_value = 'box' mock_getuser.return_value = 'ECORP\\Administrator' mock_getpid.return_value = 123 - self.assertEquals("ECORP_Administrator@box:123", self.func()) + self.assertEqual("ECORP_Administrator@box:123", self.func()) class ReturncodeToStrTest(testlib.TestCase): func = staticmethod(mitogen.parent.returncode_to_str) def test_return_zero(self): - self.assertEquals(self.func(0), 'exited with return code 0') + self.assertEqual(self.func(0), 'exited with return code 0') def test_return_one(self): - self.assertEquals(self.func(1), 'exited with return code 1') + self.assertEqual(self.func(1), 'exited with return code 1') def test_sigkill(self): - self.assertEquals(self.func(-signal.SIGKILL), + self.assertEqual(self.func(-signal.SIGKILL), 'exited due to signal %s (SIGKILL)' % (int(signal.SIGKILL),) ) @@ -110,7 +110,7 @@ def test_connect_timeout(self): e = self.assertRaises(OSError, lambda: os.kill(conn.proc.pid, 0) ) - self.assertEquals(e.args[0], errno.ESRCH) + self.assertEqual(e.args[0], errno.ESRCH) class StreamErrorTest(testlib.RouterMixin, testlib.TestCase): @@ -135,7 +135,7 @@ def test_via_eof(self): ) ) expect = mitogen.parent.Connection.eof_error_msg - self.assertTrue(expect in e.args[0]) + self.assertIn(expect, e.args[0]) def test_direct_enoent(self): e = self.assertRaises(mitogen.core.StreamError, @@ -157,7 +157,7 @@ def test_via_enoent(self): ) ) s = 'Child start failed: [Errno 2] No such file or directory' - self.assertTrue(s in e.args[0]) + self.assertIn(s, e.args[0]) class ContextTest(testlib.RouterMixin, testlib.TestCase): @@ -176,9 +176,9 @@ def test_pty_returned(self): master_fp, slave_fp = self.func() try: self.assertTrue(master_fp.isatty()) - self.assertTrue(isinstance(master_fp, file)) + self.assertIsInstance(master_fp, file) self.assertTrue(slave_fp.isatty()) - self.assertTrue(isinstance(slave_fp, file)) + self.assertIsInstance(slave_fp, file) finally: master_fp.close() slave_fp.close() @@ -189,9 +189,9 @@ def test_max_reached(self, openpty): e = self.assertRaises(mitogen.core.StreamError, lambda: self.func()) msg = mitogen.parent.OPENPTY_MSG % (openpty.side_effect,) - self.assertEquals(e.args[0], msg) + self.assertEqual(e.args[0], msg) - @unittest2.skipIf(condition=(os.uname()[0] != 'Linux'), + @unittest.skipIf(condition=(os.uname()[0] != 'Linux'), reason='Fallback only supported on Linux') @mock.patch('os.openpty') def test_broken_linux_fallback(self, openpty): @@ -199,12 +199,12 @@ def test_broken_linux_fallback(self, openpty): master_fp, slave_fp = self.func() try: st = os.fstat(master_fp.fileno()) - self.assertEquals(5, os.major(st.st_rdev)) + self.assertEqual(5, os.major(st.st_rdev)) flags = fcntl.fcntl(master_fp.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) st = os.fstat(slave_fp.fileno()) - self.assertEquals(136, os.major(st.st_rdev)) + self.assertEqual(136, os.major(st.st_rdev)) flags = fcntl.fcntl(slave_fp.fileno(), fcntl.F_GETFL) self.assertTrue(flags & os.O_RDWR) finally: @@ -221,7 +221,7 @@ def test_child_disconnected(self): c1.shutdown(wait=True) e = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(e.args[0], self.router.respondent_disconnect_msg) + self.assertEqual(e.args[0], self.router.respondent_disconnect_msg) def test_indirect_child_disconnected(self): # Achievement unlocked: process notices an indirectly connected child @@ -232,7 +232,7 @@ def test_indirect_child_disconnected(self): c2.shutdown(wait=True) e = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(e.args[0], self.router.respondent_disconnect_msg) + self.assertEqual(e.args[0], self.router.respondent_disconnect_msg) def test_indirect_child_intermediary_disconnected(self): # Battlefield promotion: process notices indirect child disconnected @@ -243,7 +243,7 @@ def test_indirect_child_intermediary_disconnected(self): c1.shutdown(wait=True) e = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(e.args[0], self.router.respondent_disconnect_msg) + self.assertEqual(e.args[0], self.router.respondent_disconnect_msg) def test_near_sibling_disconnected(self): # Hard mode: child notices sibling connected to same parent has @@ -291,7 +291,3 @@ def test_far_sibling_disconnected(self): lambda: recv.get().unpickle()) s = 'mitogen.core.ChannelError: ' + self.router.respondent_disconnect_msg self.assertTrue(e.args[0].startswith(s)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/podman_test.py b/tests/podman_test.py new file mode 100644 index 000000000..8e9816782 --- /dev/null +++ b/tests/podman_test.py @@ -0,0 +1,44 @@ +import os + +import testlib + + +class ConstructorTest(testlib.RouterMixin, testlib.TestCase): + def test_okay(self): + stub_path = testlib.data_path('stubs/stub-podman.py') + + context = self.router.podman( + container='container_name', + podman_path=stub_path, + ) + stream = self.router.stream_by_id(context.context_id) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + expected_call = [ + stub_path, + 'exec', + '--interactive', + '--', + 'container_name', + stream.conn.options.python_path + ] + self.assertEqual(argv[:len(expected_call)], expected_call) + + context = self.router.podman( + container='container_name', + podman_path=stub_path, + username='some_user', + ) + stream = self.router.stream_by_id(context.context_id) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + expected_call = [ + stub_path, + 'exec', + '--user=some_user', + '--interactive', + '--', + 'container_name', + stream.conn.options.python_path + ] + self.assertEqual(argv[:len(expected_call)], expected_call) diff --git a/tests/policy_function_test.py b/tests/policy_function_test.py index 56e33b890..77bd17d19 100644 --- a/tests/policy_function_test.py +++ b/tests/policy_function_test.py @@ -1,6 +1,7 @@ - -import mock -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import mitogen.core import mitogen.parent @@ -34,7 +35,3 @@ def call(self, auth_id, remote_id): def test_okay(self): self.assertFalse(0, 1) self.assertTrue(1, 1) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/poller_test.py b/tests/poller_test.py index 3ed59ae35..0abc836d5 100644 --- a/tests/poller_test.py +++ b/tests/poller_test.py @@ -1,23 +1,16 @@ - import errno import os import select import socket import sys -import time - -import unittest2 +import unittest import mitogen.core import mitogen.parent -import testlib +from mitogen.core import next -try: - next -except NameError: - # Python 2.4 - from mitogen.core import next +import testlib class SockMixin(object): @@ -88,69 +81,69 @@ def tearDown(self): class ReceiveStateMixin(PollerMixin, SockMixin): def test_start_receive_adds_reader(self): self.p.start_receive(self.l1) - self.assertEquals([(self.l1, self.l1)], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([(self.l1, self.l1)], self.p.readers) + self.assertEqual([], self.p.writers) def test_start_receive_adds_reader_data(self): data = object() self.p.start_receive(self.l1, data=data) - self.assertEquals([(self.l1, data)], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([(self.l1, data)], self.p.readers) + self.assertEqual([], self.p.writers) def test_stop_receive(self): self.p.start_receive(self.l1) self.p.stop_receive(self.l1) - self.assertEquals([], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([], self.p.writers) def test_stop_receive_dup(self): self.p.start_receive(self.l1) self.p.stop_receive(self.l1) - self.assertEquals([], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([], self.p.writers) self.p.stop_receive(self.l1) - self.assertEquals([], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([], self.p.writers) def test_stop_receive_noexist(self): p = self.klass() p.stop_receive(123) # should not fail - self.assertEquals([], p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], p.readers) + self.assertEqual([], self.p.writers) class TransmitStateMixin(PollerMixin, SockMixin): def test_start_transmit_adds_writer(self): self.p.start_transmit(self.r1) - self.assertEquals([], self.p.readers) - self.assertEquals([(self.r1, self.r1)], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([(self.r1, self.r1)], self.p.writers) def test_start_transmit_adds_writer_data(self): data = object() self.p.start_transmit(self.r1, data=data) - self.assertEquals([], self.p.readers) - self.assertEquals([(self.r1, data)], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([(self.r1, data)], self.p.writers) def test_stop_transmit(self): self.p.start_transmit(self.r1) self.p.stop_transmit(self.r1) - self.assertEquals([], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([], self.p.writers) def test_stop_transmit_dup(self): self.p.start_transmit(self.r1) self.p.stop_transmit(self.r1) - self.assertEquals([], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([], self.p.writers) self.p.stop_transmit(self.r1) - self.assertEquals([], self.p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], self.p.readers) + self.assertEqual([], self.p.writers) def test_stop_transmit_noexist(self): p = self.klass() p.stop_receive(123) # should not fail - self.assertEquals([], p.readers) - self.assertEquals([], self.p.writers) + self.assertEqual([], p.readers) + self.assertEqual([], self.p.writers) class CloseMixin(PollerMixin): @@ -165,42 +158,42 @@ def test_double_close(self): class PollMixin(PollerMixin): def test_empty_zero_timeout(self): t0 = mitogen.core.now() - self.assertEquals([], list(self.p.poll(0))) - self.assertTrue((mitogen.core.now() - t0) < .1) # vaguely reasonable + self.assertEqual([], list(self.p.poll(0))) + self.assertLess((mitogen.core.now() - t0), .1) # vaguely reasonable def test_empty_small_timeout(self): t0 = mitogen.core.now() - self.assertEquals([], list(self.p.poll(.2))) - self.assertTrue((mitogen.core.now() - t0) >= .2) + self.assertEqual([], list(self.p.poll(.2))) + self.assertGreaterEqual((mitogen.core.now() - t0), .2) class ReadableMixin(PollerMixin, SockMixin): def test_unreadable(self): self.p.start_receive(self.l1) - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) def test_readable_before_add(self): self.fill(self.r1) self.p.start_receive(self.l1) - self.assertEquals([self.l1], list(self.p.poll(0))) + self.assertEqual([self.l1], list(self.p.poll(0))) def test_readable_after_add(self): self.p.start_receive(self.l1) self.fill(self.r1) - self.assertEquals([self.l1], list(self.p.poll(0))) + self.assertEqual([self.l1], list(self.p.poll(0))) def test_readable_then_unreadable(self): self.fill(self.r1) self.p.start_receive(self.l1) - self.assertEquals([self.l1], list(self.p.poll(0))) + self.assertEqual([self.l1], list(self.p.poll(0))) self.drain(self.l1) - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) def test_readable_data(self): data = object() self.fill(self.r1) self.p.start_receive(self.l1, data=data) - self.assertEquals([data], list(self.p.poll(0))) + self.assertEqual([data], list(self.p.poll(0))) def test_double_readable_data(self): data1 = object() @@ -209,35 +202,35 @@ def test_double_readable_data(self): self.p.start_receive(self.l1, data=data1) self.fill(self.r2) self.p.start_receive(self.l2, data=data2) - self.assertEquals(set([data1, data2]), set(self.p.poll(0))) + self.assertEqual(set([data1, data2]), set(self.p.poll(0))) class WriteableMixin(PollerMixin, SockMixin): def test_writeable(self): self.p.start_transmit(self.r1) - self.assertEquals([self.r1], list(self.p.poll(0))) + self.assertEqual([self.r1], list(self.p.poll(0))) def test_writeable_data(self): data = object() self.p.start_transmit(self.r1, data=data) - self.assertEquals([data], list(self.p.poll(0))) + self.assertEqual([data], list(self.p.poll(0))) def test_unwriteable_before_add(self): self.fill(self.r1) self.p.start_transmit(self.r1) - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) def test_unwriteable_after_add(self): self.p.start_transmit(self.r1) self.fill(self.r1) - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) def test_unwriteable_then_writeable(self): self.fill(self.r1) self.p.start_transmit(self.r1) - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) self.drain(self.l1) - self.assertEquals([self.r1], list(self.p.poll(0))) + self.assertEqual([self.r1], list(self.p.poll(0))) def test_double_unwriteable_then_Writeable(self): self.fill(self.r1) @@ -246,13 +239,13 @@ def test_double_unwriteable_then_Writeable(self): self.fill(self.r2) self.p.start_transmit(self.r2) - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) self.drain(self.l1) - self.assertEquals([self.r1], list(self.p.poll(0))) + self.assertEqual([self.r1], list(self.p.poll(0))) self.drain(self.l2) - self.assertEquals(set([self.r1, self.r2]), set(self.p.poll(0))) + self.assertEqual(set([self.r1, self.r2]), set(self.p.poll(0))) class MutateDuringYieldMixin(PollerMixin, SockMixin): @@ -264,13 +257,13 @@ def test_one_readable_removed_before_yield(self): self.p.start_receive(self.r1) p = self.p.poll(0) self.p.stop_receive(self.r1) - self.assertEquals([], list(p)) + self.assertEqual([], list(p)) def test_one_writeable_removed_before_yield(self): self.p.start_transmit(self.r1) p = self.p.poll(0) self.p.stop_transmit(self.r1) - self.assertEquals([], list(p)) + self.assertEqual([], list(p)) def test_one_readable_readded_before_yield(self): # fd removed, closed, another fd opened, gets same fd number, re-added. @@ -280,7 +273,7 @@ def test_one_readable_readded_before_yield(self): p = self.p.poll(0) self.p.stop_receive(self.r1) self.p.start_receive(self.r1) - self.assertEquals([], list(p)) + self.assertEqual([], list(p)) def test_one_readable_readded_during_yield(self): self.fill(self.l1) @@ -301,7 +294,7 @@ def test_one_readable_readded_during_yield(self): # the start_receive() may be for a totally new underlying file object, # the live loop iteration must not yield any buffered readiness event. - self.assertEquals([], list(p)) + self.assertEqual([], list(p)) class FileClosedMixin(PollerMixin, SockMixin): @@ -310,10 +303,10 @@ class FileClosedMixin(PollerMixin, SockMixin): def test_writeable_then_closed(self): self.p.start_transmit(self.r1) - self.assertEquals([self.r1], list(self.p.poll(0))) + self.assertEqual([self.r1], list(self.p.poll(0))) self.close_socks() try: - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) except select.error: # a crash is also reasonable here. pass @@ -323,7 +316,7 @@ def test_writeable_closed_before_yield(self): p = self.p.poll(0) self.close_socks() try: - self.assertEquals([], list(p)) + self.assertEqual([], list(p)) except select.error: # a crash is also reasonable here. pass @@ -331,10 +324,10 @@ def test_writeable_closed_before_yield(self): def test_readable_then_closed(self): self.fill(self.l1) self.p.start_receive(self.r1) - self.assertEquals([self.r1], list(self.p.poll(0))) + self.assertEqual([self.r1], list(self.p.poll(0))) self.close_socks() try: - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) except select.error: # a crash is also reasonable here. pass @@ -345,7 +338,7 @@ def test_readable_closed_before_yield(self): p = self.p.poll(0) self.close_socks() try: - self.assertEquals([], list(p)) + self.assertEqual([], list(p)) except select.error: # a crash is also reasonable here. pass @@ -357,10 +350,10 @@ def test_tty_hangup_detected(self): master_fp, slave_fp = mitogen.parent.openpty() try: self.p.start_receive(master_fp.fileno()) - self.assertEquals([], list(self.p.poll(0))) + self.assertEqual([], list(self.p.poll(0))) slave_fp.close() slave_fp = None - self.assertEquals([master_fp.fileno()], list(self.p.poll(0))) + self.assertEqual([master_fp.fileno()], list(self.p.poll(0))) finally: if slave_fp is not None: slave_fp.close() @@ -377,9 +370,9 @@ def test_one_distinct(self): self.p.start_receive(self.r1, data=rdata) self.p.start_transmit(self.r1, data=wdata) - self.assertEquals([wdata], list(self.p.poll(0))) + self.assertEqual([wdata], list(self.p.poll(0))) self.fill(self.l1) # r1 is now readable and writeable. - self.assertEquals(set([rdata, wdata]), set(self.p.poll(0))) + self.assertEqual(set([rdata, wdata]), set(self.p.poll(0))) class AllMixin(ReceiveStateMixin, @@ -397,41 +390,32 @@ class AllMixin(ReceiveStateMixin, """ -class SelectTest(AllMixin, testlib.TestCase): +class CorePollerTest(AllMixin, testlib.TestCase): klass = mitogen.core.Poller -SelectTest = unittest2.skipIf( - condition=(not SelectTest.klass.SUPPORTED), - reason='select.select() not supported' -)(SelectTest) - class PollTest(AllMixin, testlib.TestCase): klass = mitogen.parent.PollPoller -PollTest = unittest2.skipIf( +PollTest = unittest.skipIf( condition=(not PollTest.klass.SUPPORTED), - reason='select.poll() not supported' + reason='select.poll() not available', )(PollTest) class KqueueTest(AllMixin, testlib.TestCase): klass = mitogen.parent.KqueuePoller -KqueueTest = unittest2.skipIf( +KqueueTest = unittest.skipIf( condition=(not KqueueTest.klass.SUPPORTED), - reason='select.kqueue() not supported' + reason='select.kqueue() not available', )(KqueueTest) class EpollTest(AllMixin, testlib.TestCase): klass = mitogen.parent.EpollPoller -EpollTest = unittest2.skipIf( +EpollTest = unittest.skipIf( condition=(not EpollTest.klass.SUPPORTED), - reason='select.epoll() not supported' + reason='select.epoll() not available', )(EpollTest) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/polyfill_functions_test.py b/tests/polyfill_functions_test.py index ae65eb2f4..c3441a63e 100644 --- a/tests/polyfill_functions_test.py +++ b/tests/polyfill_functions_test.py @@ -1,6 +1,4 @@ - import testlib -import unittest2 import mitogen.core from mitogen.core import b @@ -11,30 +9,30 @@ class BytesPartitionTest(testlib.TestCase): def test_no_sep(self): left, sep, right = self.func(b('dave'), b('x')) - self.assertTrue(isinstance(left, mitogen.core.BytesType)) - self.assertTrue(isinstance(sep, mitogen.core.BytesType)) - self.assertTrue(isinstance(right, mitogen.core.BytesType)) - self.assertEquals(left, b('dave')) - self.assertEquals(sep, b('')) - self.assertEquals(right, b('')) + self.assertIsInstance(left, mitogen.core.BytesType) + self.assertIsInstance(sep, mitogen.core.BytesType) + self.assertIsInstance(right, mitogen.core.BytesType) + self.assertEqual(left, b('dave')) + self.assertEqual(sep, b('')) + self.assertEqual(right, b('')) def test_one_sep(self): left, sep, right = self.func(b('davexdave'), b('x')) - self.assertTrue(isinstance(left, mitogen.core.BytesType)) - self.assertTrue(isinstance(sep, mitogen.core.BytesType)) - self.assertTrue(isinstance(right, mitogen.core.BytesType)) - self.assertEquals(left, b('dave')) - self.assertEquals(sep, b('x')) - self.assertEquals(right, b('dave')) + self.assertIsInstance(left, mitogen.core.BytesType) + self.assertIsInstance(sep, mitogen.core.BytesType) + self.assertIsInstance(right, mitogen.core.BytesType) + self.assertEqual(left, b('dave')) + self.assertEqual(sep, b('x')) + self.assertEqual(right, b('dave')) def test_two_seps(self): left, sep, right = self.func(b('davexdavexdave'), b('x')) - self.assertTrue(isinstance(left, mitogen.core.BytesType)) - self.assertTrue(isinstance(sep, mitogen.core.BytesType)) - self.assertTrue(isinstance(right, mitogen.core.BytesType)) - self.assertEquals(left, b('dave')) - self.assertEquals(sep, b('x')) - self.assertEquals(right, b('davexdave')) + self.assertIsInstance(left, mitogen.core.BytesType) + self.assertIsInstance(sep, mitogen.core.BytesType) + self.assertIsInstance(right, mitogen.core.BytesType) + self.assertEqual(left, b('dave')) + self.assertEqual(sep, b('x')) + self.assertEqual(right, b('davexdave')) class StrPartitionTest(testlib.TestCase): @@ -42,30 +40,30 @@ class StrPartitionTest(testlib.TestCase): def test_no_sep(self): left, sep, right = self.func(u'dave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) - self.assertEquals(left, u'dave') - self.assertEquals(sep, u'') - self.assertEquals(right, u'') + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) + self.assertEqual(left, u'dave') + self.assertEqual(sep, u'') + self.assertEqual(right, u'') def test_one_sep(self): left, sep, right = self.func(u'davexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) - self.assertEquals(left, u'dave') - self.assertEquals(sep, u'x') - self.assertEquals(right, u'dave') + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) + self.assertEqual(left, u'dave') + self.assertEqual(sep, u'x') + self.assertEqual(right, u'dave') def test_two_seps(self): left, sep, right = self.func(u'davexdavexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) - self.assertEquals(left, u'dave') - self.assertEquals(sep, u'x') - self.assertEquals(right, u'davexdave') + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) + self.assertEqual(left, u'dave') + self.assertEqual(sep, u'x') + self.assertEqual(right, u'davexdave') class StrRpartitionTest(testlib.TestCase): @@ -73,31 +71,27 @@ class StrRpartitionTest(testlib.TestCase): def test_no_sep(self): left, sep, right = self.func(u'dave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) - self.assertEquals(left, u'') - self.assertEquals(sep, u'') - self.assertEquals(right, u'dave') + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) + self.assertEqual(left, u'') + self.assertEqual(sep, u'') + self.assertEqual(right, u'dave') def test_one_sep(self): left, sep, right = self.func(u'davexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) - self.assertEquals(left, u'dave') - self.assertEquals(sep, u'x') - self.assertEquals(right, u'dave') + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) + self.assertEqual(left, u'dave') + self.assertEqual(sep, u'x') + self.assertEqual(right, u'dave') def test_two_seps(self): left, sep, right = self.func(u'davexdavexdave', u'x') - self.assertTrue(isinstance(left, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(sep, mitogen.core.UnicodeType)) - self.assertTrue(isinstance(right, mitogen.core.UnicodeType)) - self.assertEquals(left, u'davexdave') - self.assertEquals(sep, u'x') - self.assertEquals(right, u'dave') - - -if __name__ == '__main__': - unittest2.main() + self.assertIsInstance(left, mitogen.core.UnicodeType) + self.assertIsInstance(sep, mitogen.core.UnicodeType) + self.assertIsInstance(right, mitogen.core.UnicodeType) + self.assertEqual(left, u'davexdave') + self.assertEqual(sep, u'x') + self.assertEqual(right, u'dave') diff --git a/tests/push_file_service_test.py b/tests/push_file_service_test.py index 1dfff2419..f494a5af9 100644 --- a/tests/push_file_service_test.py +++ b/tests/push_file_service_test.py @@ -1,7 +1,4 @@ - -import os import tempfile -import unittest2 import mitogen.core import mitogen.service @@ -44,13 +41,9 @@ def test_two_grandchild_one_intermediary(self): service.propagate_to(context=c2, path=path) s = c1.call(wait_for_file, path=path) - self.assertEquals(b('test'), s) + self.assertEqual(b('test'), s) s = c2.call(wait_for_file, path=path) - self.assertEquals(b('test'), s) + self.assertEqual(b('test'), s) finally: tf.close() - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/reaper_test.py b/tests/reaper_test.py index e78fdbf28..560d48ff4 100644 --- a/tests/reaper_test.py +++ b/tests/reaper_test.py @@ -1,29 +1,29 @@ - import signal -import unittest2 + import testlib -import mock +try: + from unittest import mock +except ImportError: + import mock import mitogen.parent class ReaperTest(testlib.TestCase): - @mock.patch('os.kill') - def test_calc_delay(self, kill): + def test_calc_delay(self): broker = mock.Mock() proc = mock.Mock() proc.poll.return_value = None reaper = mitogen.parent.Reaper(broker, proc, True, True) - self.assertEquals(50, int(1000 * reaper._calc_delay(0))) - self.assertEquals(86, int(1000 * reaper._calc_delay(1))) - self.assertEquals(147, int(1000 * reaper._calc_delay(2))) - self.assertEquals(254, int(1000 * reaper._calc_delay(3))) - self.assertEquals(437, int(1000 * reaper._calc_delay(4))) - self.assertEquals(752, int(1000 * reaper._calc_delay(5))) - self.assertEquals(1294, int(1000 * reaper._calc_delay(6))) - - @mock.patch('os.kill') - def test_reap_calls(self, kill): + self.assertEqual(50, int(1000 * reaper._calc_delay(0))) + self.assertEqual(86, int(1000 * reaper._calc_delay(1))) + self.assertEqual(147, int(1000 * reaper._calc_delay(2))) + self.assertEqual(254, int(1000 * reaper._calc_delay(3))) + self.assertEqual(437, int(1000 * reaper._calc_delay(4))) + self.assertEqual(752, int(1000 * reaper._calc_delay(5))) + self.assertEqual(1294, int(1000 * reaper._calc_delay(6))) + + def test_reap_calls(self): broker = mock.Mock() proc = mock.Mock() proc.poll.return_value = None @@ -31,24 +31,20 @@ def test_reap_calls(self, kill): reaper = mitogen.parent.Reaper(broker, proc, True, True) reaper.reap() - self.assertEquals(0, kill.call_count) + self.assertEqual(0, proc.send_signal.call_count) reaper.reap() - self.assertEquals(1, kill.call_count) + self.assertEqual(1, proc.send_signal.call_count) reaper.reap() reaper.reap() reaper.reap() - self.assertEquals(1, kill.call_count) + self.assertEqual(1, proc.send_signal.call_count) reaper.reap() - self.assertEquals(2, kill.call_count) + self.assertEqual(2, proc.send_signal.call_count) - self.assertEquals(kill.mock_calls, [ - mock.call(proc.pid, signal.SIGTERM), - mock.call(proc.pid, signal.SIGKILL), + self.assertEqual(proc.send_signal.mock_calls, [ + mock.call(signal.SIGTERM), + mock.call(signal.SIGKILL), ]) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/receiver_test.py b/tests/receiver_test.py index 65c5f7ff8..0cb9b15de 100644 --- a/tests/receiver_test.py +++ b/tests/receiver_test.py @@ -1,7 +1,6 @@ - import sys import threading -import unittest2 +import unittest import mitogen.core import testlib @@ -17,8 +16,8 @@ def yield_stuff_then_die(sender): class ConstructorTest(testlib.RouterMixin, testlib.TestCase): def test_handle(self): recv = mitogen.core.Receiver(self.router) - self.assertTrue(isinstance(recv.handle, int)) - self.assertTrue(recv.handle > 100) + self.assertIsInstance(recv.handle, int) + self.assertGreater(recv.handle, 100) self.router.route( mitogen.core.Message.pickled( 'hi', @@ -26,7 +25,7 @@ def test_handle(self): handle=recv.handle, ) ) - self.assertEquals('hi', recv.get().unpickle()) + self.assertEqual('hi', recv.get().unpickle()) class IterationTest(testlib.RouterMixin, testlib.TestCase): @@ -34,8 +33,8 @@ def test_dead_stops_iteration(self): recv = mitogen.core.Receiver(self.router) fork = self.router.local() ret = fork.call_async(yield_stuff_then_die, recv.to_sender()) - self.assertEquals(list(range(5)), list(m.unpickle() for m in recv)) - self.assertEquals(10, ret.get().unpickle()) + self.assertEqual(list(range(5)), list(m.unpickle() for m in recv)) + self.assertEqual(10, ret.get().unpickle()) def iter_and_put(self, recv, latch): try: @@ -76,7 +75,7 @@ def throw(): raise latch.get() t.join() e = self.assertRaises(mitogen.core.ChannelError, throw) - self.assertEquals(e.args[0], mitogen.core.Receiver.closed_msg) + self.assertEqual(e.args[0], mitogen.core.Receiver.closed_msg) def test_closes_all(self): latch = mitogen.core.Latch() @@ -92,7 +91,7 @@ def throw(): raise latch.get() for x in range(5): e = self.assertRaises(mitogen.core.ChannelError, throw) - self.assertEquals(e.args[0], mitogen.core.Receiver.closed_msg) + self.assertEqual(e.args[0], mitogen.core.Receiver.closed_msg) for t in ts: t.join() @@ -118,9 +117,9 @@ def throw(): raise latch.get() t.join() e = self.assertRaises(mitogen.core.ChannelError, throw) - self.assertEquals(e.args[0], sender.explicit_close_msg) + self.assertEqual(e.args[0], sender.explicit_close_msg) - @unittest2.skip(reason=( + @unittest.skip(reason=( 'Unclear if a asingle dead message received from remote should ' 'cause all threads to wake up.' )) @@ -139,7 +138,7 @@ def throw(): raise latch.get() for x in range(5): e = self.assertRaises(mitogen.core.ChannelError, throw) - self.assertEquals(e.args[0], mitogen.core.Receiver.closed_msg) + self.assertEqual(e.args[0], mitogen.core.Receiver.closed_msg) for t in ts: t.join() @@ -152,8 +151,4 @@ class ToSenderTest(testlib.RouterMixin, testlib.TestCase): def test_returned_context(self): myself = self.router.myself() recv = self.klass(self.router) - self.assertEquals(myself, recv.to_sender().context) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(myself, recv.to_sender().context) diff --git a/tests/requirements-tox.txt b/tests/requirements-tox.txt new file mode 100644 index 000000000..bc7f7c2af --- /dev/null +++ b/tests/requirements-tox.txt @@ -0,0 +1,4 @@ +tox==3.28; python_version == '2.7' +tox==3.28; python_version == '3.6' +tox==4.8.0; python_version == '3.7' +tox>=4.13.0,~=4.0; python_version >= '3.8' diff --git a/tests/requirements.txt b/tests/requirements.txt index 76e6545d1..c5671b37e 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,17 +1,35 @@ -psutil==5.4.8 -coverage==4.5.1 -Django==1.6.11 # Last version supporting 2.6. -mock==2.0.0 -pytz==2018.5 -cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect -pycparser==2.18 # Last version supporting 2.6. -faulthandler==3.1; python_version < '3.3' # used by testlib -pytest-catchlog==1.2.2 -pytest==3.1.2 +cffi==1.15.1; python_version < '3.8' +cffi==1.17.1; python_version >= '3.8' + +coverage==5.5; python_version == '2.7' +coverage==6.2; python_version == '3.6' +coverage==7.2.7; python_version == '3.7' +coverage==7.4.3; python_version >= '3.8' + +Django==1.11.29; python_version < '3.0' +Django==3.2.20; python_version >= '3.6' + +mock==3.0.5; python_version == '2.7' +mock==5.1.0; python_version >= '3.6' + +pexpect==4.8 + +psutil==5.9.8 + +pytest==4.6.11; python_version == '2.7' +pytest==7.0.1; python_version == '3.6' +pytest==7.4.4; python_version == '3.7' +pytest==8.0.2; python_version >= '3.8' + +subprocess32==3.5.4; python_version < '3.0' timeoutcontext==1.2.0 -unittest2==1.1.0 # Fix InsecurePlatformWarning while creating py26 tox environment # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings -urllib3[secure]; python_version < '2.7.9' +urllib3[secure]==1.23; python_version < '2.7' +urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9' # Last idna compatible with Python 2.6 was idna 2.7. idna==2.7; python_version < '2.7' + +virtualenv==20.15.1; python_version == '2.7' +virtualenv==20.17.1; python_version == '3.6' +virtualenv==20.25.1; python_version >= '3.7' diff --git a/tests/responder_test.py b/tests/responder_test.py index 2653589c2..d1e658167 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -1,10 +1,12 @@ - -import mock import textwrap import subprocess import sys +import unittest -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import mitogen.master import testlib @@ -25,9 +27,9 @@ def test_missing_exec_guard(self): args = [sys.executable, path] proc = subprocess.Popen(args, stderr=subprocess.PIPE) _, stderr = proc.communicate() - self.assertEquals(1, proc.returncode) + self.assertEqual(1, proc.returncode) expect = self.klass.main_guard_msg % (path,) - self.assertTrue(expect in stderr.decode()) + self.assertIn(expect, stderr.decode()) HAS_MITOGEN_MAIN = mitogen.core.b( textwrap.dedent(""" @@ -44,7 +46,7 @@ def main(router): def test_mitogen_main(self): untouched = self.call("derp.py", self.HAS_MITOGEN_MAIN) - self.assertEquals(untouched, self.HAS_MITOGEN_MAIN) + self.assertEqual(untouched, self.HAS_MITOGEN_MAIN) HAS_EXEC_GUARD = mitogen.core.b( textwrap.dedent(""" @@ -64,7 +66,7 @@ def main(): def test_exec_guard(self): touched = self.call("derp.py", self.HAS_EXEC_GUARD) bits = touched.decode().split() - self.assertEquals(bits[-3:], ['def', 'main():', 'pass']) + self.assertEqual(bits[-3:], ['def', 'main():', 'pass']) class GoodModulesTest(testlib.RouterMixin, testlib.TestCase): @@ -73,30 +75,32 @@ def test_plain_old_module(self): # package machinery damage. context = self.router.local() - self.assertEquals(256, context.call(plain_old_module.pow, 2, 8)) + self.assertEqual(256, context.call(plain_old_module.pow, 2, 8)) os_fork = int(sys.version_info < (2, 6)) # mitogen.os_fork - self.assertEquals(1+os_fork, self.router.responder.get_module_count) - self.assertEquals(1+os_fork, self.router.responder.good_load_module_count) + self.assertEqual(1+os_fork, self.router.responder.get_module_count) + self.assertEqual(1+os_fork, self.router.responder.good_load_module_count) self.assertLess(300, self.router.responder.good_load_module_size) def test_simple_pkg(self): # Ensure success of a simple package containing two submodules, one of # which imports the other. context = self.router.local() - self.assertEquals(3, + self.assertEqual(3, context.call(simple_pkg.a.subtract_one_add_two, 2)) os_fork = int(sys.version_info < (2, 6)) # mitogen.os_fork - self.assertEquals(2+os_fork, self.router.responder.get_module_count) - self.assertEquals(3+os_fork, self.router.responder.good_load_module_count) - self.assertEquals(0, self.router.responder.bad_load_module_count) + self.assertEqual(2+os_fork, self.router.responder.get_module_count) + self.assertEqual(3+os_fork, self.router.responder.good_load_module_count) + self.assertEqual(0, self.router.responder.bad_load_module_count) self.assertLess(450, self.router.responder.good_load_module_size) def test_self_contained_program(self): # Ensure a program composed of a single script can be imported # successfully. args = [sys.executable, testlib.data_path('self_contained_program.py')] - output = testlib.subprocess__check_output(args).decode() - self.assertEquals(output, "['__main__', 50]\n") + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + b_stdout, _ = proc.communicate() + self.assertEqual(proc.returncode, 0) + self.assertEqual(b_stdout.decode(), "['__main__', 50]\n") class BrokenModulesTest(testlib.TestCase): @@ -117,20 +121,20 @@ def test_obviously_missing(self): responder = mitogen.master.ModuleResponder(router) responder._on_get_module(msg) - self.assertEquals(1, len(router._async_route.mock_calls)) + self.assertEqual(1, len(router._async_route.mock_calls)) - self.assertEquals(1, responder.get_module_count) - self.assertEquals(0, responder.good_load_module_count) - self.assertEquals(0, responder.good_load_module_size) - self.assertEquals(1, responder.bad_load_module_count) + self.assertEqual(1, responder.get_module_count) + self.assertEqual(0, responder.good_load_module_count) + self.assertEqual(0, responder.good_load_module_size) + self.assertEqual(1, responder.bad_load_module_count) call = router._async_route.mock_calls[0] msg, = call[1] - self.assertEquals(mitogen.core.LOAD_MODULE, msg.handle) - self.assertEquals(('non_existent_module', None, None, None, ()), + self.assertEqual(mitogen.core.LOAD_MODULE, msg.handle) + self.assertEqual(('non_existent_module', None, None, None, ()), msg.unpickle()) - @unittest2.skipIf( + @unittest.skipIf( condition=sys.version_info < (2, 6), reason='Ancient Python lacked "from . import foo"', ) @@ -155,15 +159,15 @@ def test_ansible_six_messed_up_path(self): responder = mitogen.master.ModuleResponder(router) responder._on_get_module(msg) - self.assertEquals(1, len(router._async_route.mock_calls)) + self.assertEqual(1, len(router._async_route.mock_calls)) - self.assertEquals(1, responder.get_module_count) - self.assertEquals(1, responder.good_load_module_count) - self.assertEquals(0, responder.bad_load_module_count) + self.assertEqual(1, responder.get_module_count) + self.assertEqual(1, responder.good_load_module_count) + self.assertEqual(0, responder.bad_load_module_count) call = router._async_route.mock_calls[0] msg, = call[1] - self.assertEquals(mitogen.core.LOAD_MODULE, msg.handle) + self.assertEqual(mitogen.core.LOAD_MODULE, msg.handle) tup = msg.unpickle() self.assertIsInstance(tup, tuple) @@ -181,7 +185,7 @@ def test_forward_to_nonexistent_context(self): ) ) s = capture.stop() - self.assertTrue('dropping forward of' in s) + self.assertIn('dropping forward of', s) def test_stats(self): # Forwarding stats broken because forwarding is broken. See #469. @@ -189,30 +193,26 @@ def test_stats(self): c2 = self.router.local(via=c1) os_fork = int(sys.version_info < (2, 6)) - self.assertEquals(256, c2.call(plain_old_module.pow, 2, 8)) - self.assertEquals(2+os_fork, self.router.responder.get_module_count) - self.assertEquals(2+os_fork, self.router.responder.good_load_module_count) + self.assertEqual(256, c2.call(plain_old_module.pow, 2, 8)) + self.assertEqual(2+os_fork, self.router.responder.get_module_count) + self.assertEqual(2+os_fork, self.router.responder.good_load_module_count) self.assertLess(10000, self.router.responder.good_load_module_size) self.assertGreater(40000, self.router.responder.good_load_module_size) class BlacklistTest(testlib.TestCase): - @unittest2.skip('implement me') + @unittest.skip('implement me') def test_whitelist_no_blacklist(self): assert 0 - @unittest2.skip('implement me') + @unittest.skip('implement me') def test_whitelist_has_blacklist(self): assert 0 - @unittest2.skip('implement me') + @unittest.skip('implement me') def test_blacklist_no_whitelist(self): assert 0 - @unittest2.skip('implement me') + @unittest.skip('implement me') def test_blacklist_has_whitelist(self): assert 0 - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/router_test.py b/tests/router_test.py index 61e7a2c55..276681540 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -1,11 +1,8 @@ import errno import os import sys -import time import zlib -import unittest2 - import testlib import mitogen.core import mitogen.master @@ -74,7 +71,7 @@ def test_bad_auth_id(self): # Ensure error was logged. expect = 'bad auth_id: got %r via' % (self.child2_msg.auth_id,) - self.assertTrue(expect in log.stop()) + self.assertIn(expect, log.stop()) def test_parent_unaware_of_disconnect(self): # Parent -> Child A -> Child B. B disconnects concurrent to Parent @@ -99,7 +96,7 @@ def test_parent_unaware_of_disconnect(self): e = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get().unpickle() ) - self.assertEquals(e.args[0], self.router.no_route_msg % ( + self.assertEqual(e.args[0], self.router.no_route_msg % ( 1234, c1.context_id, )) @@ -128,7 +125,7 @@ def test_bad_src_id(self): # Ensure error was lgoged. expect = 'bad src_id: got %d via' % (self.child1_msg.src_id,) - self.assertTrue(expect in log.stop()) + self.assertIn(expect, log.stop()) class PolicyTest(testlib.RouterMixin, testlib.TestCase): @@ -138,7 +135,7 @@ def test_allow_any(self): recv.to_sender().send(123) self.sync_with_broker() self.assertFalse(recv.empty()) - self.assertEquals(123, recv.get().unpickle()) + self.assertEqual(123, recv.get().unpickle()) def test_refuse_all(self): # Deliver a message locally from child2 with the correct auth_id, but @@ -168,7 +165,7 @@ def test_refuse_all(self): self.sync_with_broker() # Verify log. - self.assertTrue(self.router.refused_msg in log.stop()) + self.assertIn(self.router.refused_msg, log.stop()) # Verify message was not delivered. self.assertTrue(recv.empty()) @@ -176,7 +173,7 @@ def test_refuse_all(self): # Verify CallError received by reply_to target. e = self.assertRaises(mitogen.core.ChannelError, lambda: reply_target.get().unpickle()) - self.assertEquals(e.args[0], self.router.refused_msg) + self.assertEqual(e.args[0], self.router.refused_msg) class CrashTest(testlib.BrokerMixin, testlib.TestCase): @@ -205,7 +202,7 @@ def test_shutdown(self): # Ensure it was logged. expect = 'broker crashed' - self.assertTrue(expect in log.stop()) + self.assertIn(expect, log.stop()) self.broker.join() @@ -227,7 +224,7 @@ def test_cannot_double_register(self): router.add_handler((lambda: None), handle=1234) e = self.assertRaises(mitogen.core.Error, lambda: router.add_handler((lambda: None), handle=1234)) - self.assertEquals(router.duplicate_handle_msg, e.args[0]) + self.assertEqual(router.duplicate_handle_msg, e.args[0]) router.del_handler(1234) finally: router.broker.shutdown() @@ -248,9 +245,9 @@ def test_can_reregister(self): class MyselfTest(testlib.RouterMixin, testlib.TestCase): def test_myself(self): myself = self.router.myself() - self.assertEquals(myself.context_id, mitogen.context_id) + self.assertEqual(myself.context_id, mitogen.context_id) # TODO: context should know its own name too. - self.assertEquals(myself.name, 'self') + self.assertEqual(myself.name, 'self') class MessageSizeTest(testlib.BrokerMixin, testlib.TestCase): @@ -267,7 +264,7 @@ def test_local_exceeded(self): router.broker.defer_sync(lambda: None) expect = 'message too large (max 4096 bytes)' - self.assertTrue(expect in logs.stop()) + self.assertIn(expect, logs.stop()) def test_local_dead_message(self): # Local router should generate dead message when reply_to is set. @@ -283,9 +280,9 @@ def test_local_dead_message(self): child = router.local() e = self.assertRaises(mitogen.core.ChannelError, lambda: child.call(zlib.crc32, ' '*8192)) - self.assertEquals(e.args[0], expect) + self.assertEqual(e.args[0], expect) - self.assertTrue(expect in logs.stop()) + self.assertIn(expect, logs.stop()) def test_remote_dead_message(self): # Router should send dead message to original recipient when reply_to @@ -302,20 +299,20 @@ def test_remote_dead_message(self): lambda: recv.get().unpickle() ) expect = router.too_large_msg % (4096,) - self.assertEquals(e.args[0], expect) + self.assertEqual(e.args[0], expect) def test_remote_configured(self): router = self.klass(broker=self.broker, max_message_size=64*1024) remote = router.local() size = remote.call(return_router_max_message_size) - self.assertEquals(size, 64*1024) + self.assertEqual(size, 64*1024) def test_remote_of_remote_configured(self): router = self.klass(broker=self.broker, max_message_size=64*1024) remote = router.local() remote2 = router.local(via=remote) size = remote2.call(return_router_max_message_size) - self.assertEquals(size, 64*1024) + self.assertEqual(size, 64*1024) def test_remote_exceeded(self): # Ensure new contexts receive a router with the same value. @@ -328,7 +325,7 @@ def test_remote_exceeded(self): remote.call(send_n_sized_reply, recv.to_sender(), 128*1024) expect = 'message too large (max %d bytes)' % (64*1024,) - self.assertTrue(expect in logs.stop()) + self.assertIn(expect, logs.stop()) class NoRouteTest(testlib.RouterMixin, testlib.TestCase): @@ -338,14 +335,14 @@ def test_invalid_handle_returns_dead(self): l1 = self.router.local() recv = l1.send_async(mitogen.core.Message(handle=999)) msg = recv.get(throw_dead=False) - self.assertEquals(msg.is_dead, True) - self.assertEquals(msg.src_id, l1.context_id) - self.assertEquals(msg.data, self.router.invalid_handle_msg.encode()) + self.assertEqual(msg.is_dead, True) + self.assertEqual(msg.src_id, l1.context_id) + self.assertEqual(msg.data, self.router.invalid_handle_msg.encode()) recv = l1.send_async(mitogen.core.Message(handle=999)) e = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(e.args[0], self.router.invalid_handle_msg) + self.assertEqual(e.args[0], self.router.invalid_handle_msg) def test_totally_invalid_context_returns_dead(self): recv = mitogen.core.Receiver(self.router) @@ -356,9 +353,9 @@ def test_totally_invalid_context_returns_dead(self): ) self.router.route(msg) rmsg = recv.get(throw_dead=False) - self.assertEquals(rmsg.is_dead, True) - self.assertEquals(rmsg.src_id, mitogen.context_id) - self.assertEquals(rmsg.data, (self.router.no_route_msg % ( + self.assertEqual(rmsg.is_dead, True) + self.assertEqual(rmsg.src_id, mitogen.context_id) + self.assertEqual(rmsg.data, (self.router.no_route_msg % ( 1234, mitogen.context_id, )).encode()) @@ -366,7 +363,7 @@ def test_totally_invalid_context_returns_dead(self): self.router.route(msg) e = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(e.args[0], (self.router.no_route_msg % ( + self.assertEqual(e.args[0], (self.router.no_route_msg % ( 1234, mitogen.context_id, ))) @@ -382,9 +379,9 @@ def test_previously_alive_context_returns_dead(self): ) self.router.route(msg) rmsg = recv.get(throw_dead=False) - self.assertEquals(rmsg.is_dead, True) - self.assertEquals(rmsg.src_id, mitogen.context_id) - self.assertEquals(rmsg.data, (self.router.no_route_msg % ( + self.assertEqual(rmsg.is_dead, True) + self.assertEqual(rmsg.src_id, mitogen.context_id) + self.assertEqual(rmsg.data, (self.router.no_route_msg % ( l1.context_id, mitogen.context_id, )).encode()) @@ -392,7 +389,7 @@ def test_previously_alive_context_returns_dead(self): self.router.route(msg) e = self.assertRaises(mitogen.core.ChannelError, lambda: recv.get()) - self.assertEquals(e.args[0], self.router.no_route_msg % ( + self.assertEqual(e.args[0], self.router.no_route_msg % ( l1.context_id, mitogen.context_id, )) @@ -467,7 +464,7 @@ def test_egress_ids_populated(self): # causes messages to be sent. pass - self.assertEquals(c1s.protocol.egress_ids, set([ + self.assertEqual(c1s.protocol.egress_ids, set([ mitogen.context_id, c2.context_id, ])) @@ -498,11 +495,11 @@ def test_shutdown_wait_false(self): e = self.assertRaises(OSError, lambda: os.waitpid(pid, 0)) - self.assertEquals(e.args[0], errno.ECHILD) + self.assertEqual(e.args[0], errno.ECHILD) e = self.assertRaises(mitogen.core.ChannelError, lambda: l1.call(os.getpid)) - self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % ( l1.context_id, mitogen.context_id, )) @@ -520,11 +517,11 @@ def test_shutdown_wait_true(self): e = self.assertRaises(OSError, lambda: os.waitpid(pid, 0)) - self.assertEquals(e.args[0], errno.ECHILD) + self.assertEqual(e.args[0], errno.ECHILD) e = self.assertRaises(mitogen.core.ChannelError, lambda: l1.call(os.getpid)) - self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % ( l1.context_id, mitogen.context_id, )) @@ -547,11 +544,11 @@ def test_disconnect_valid_context(self): e = self.assertRaises(OSError, lambda: os.waitpid(pid, 0)) - self.assertEquals(e.args[0], errno.ECHILD) + self.assertEqual(e.args[0], errno.ECHILD) e = self.assertRaises(mitogen.core.ChannelError, lambda: l1.call(os.getpid)) - self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % ( l1.context_id, mitogen.context_id, )) @@ -574,16 +571,12 @@ def test_disconnect_all(self): for pid in pids: e = self.assertRaises(OSError, lambda: os.waitpid(pid, 0)) - self.assertEquals(e.args[0], errno.ECHILD) + self.assertEqual(e.args[0], errno.ECHILD) for ctx in l1, l2: e = self.assertRaises(mitogen.core.ChannelError, lambda: ctx.call(os.getpid)) - self.assertEquals(e.args[0], mitogen.core.Router.no_route_msg % ( + self.assertEqual(e.args[0], mitogen.core.Router.no_route_msg % ( ctx.context_id, mitogen.context_id, )) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/select_test.py b/tests/select_test.py index 56e7e6cd3..deaa37a6c 100644 --- a/tests/select_test.py +++ b/tests/select_test.py @@ -1,6 +1,4 @@ - -import unittest2 - +import mitogen.core import mitogen.select import testlib @@ -18,7 +16,7 @@ def test_latch(self): latch.put(123) self.assertTrue(select) - self.assertEquals(123, select.get()) + self.assertEqual(123, select.get()) self.assertFalse(select) def test_receiver(self): @@ -30,7 +28,7 @@ def test_receiver(self): recv._on_receive(mitogen.core.Message.pickled('123')) self.assertTrue(select) - self.assertEquals('123', select.get().unpickle()) + self.assertEqual('123', select.get().unpickle()) self.assertFalse(select) @@ -41,34 +39,34 @@ def test_latch(self): latch = mitogen.core.Latch() select = self.klass() select.add(latch) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(latch, select._receivers[0]) - self.assertEquals(select._put, latch.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(latch, select._receivers[0]) + self.assertEqual(select._put, latch.notify) def test_receiver(self): recv = mitogen.core.Receiver(self.router) select = self.klass() select.add(recv) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(recv, select._receivers[0]) - self.assertEquals(select._put, recv.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(recv, select._receivers[0]) + self.assertEqual(select._put, recv.notify) def test_channel(self): context = self.router.local() chan = mitogen.core.Channel(self.router, context, 1234) select = self.klass() select.add(chan) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(chan, select._receivers[0]) - self.assertEquals(select._put, chan.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(chan, select._receivers[0]) + self.assertEqual(select._put, chan.notify) def test_subselect_empty(self): select = self.klass() subselect = self.klass() select.add(subselect) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(subselect, select._receivers[0]) - self.assertEquals(select._put, subselect.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(subselect, select._receivers[0]) + self.assertEqual(select._put, subselect.notify) def test_subselect_nonempty(self): recv = mitogen.core.Receiver(self.router) @@ -77,15 +75,15 @@ def test_subselect_nonempty(self): subselect.add(recv) select.add(subselect) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(subselect, select._receivers[0]) - self.assertEquals(select._put, subselect.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(subselect, select._receivers[0]) + self.assertEqual(select._put, subselect.notify) def test_subselect_loop_direct(self): select = self.klass() exc = self.assertRaises(mitogen.select.Error, lambda: select.add(select)) - self.assertEquals(str(exc), self.klass.loop_msg) + self.assertEqual(str(exc), self.klass.loop_msg) def test_subselect_loop_indirect(self): s0 = self.klass() @@ -96,7 +94,7 @@ def test_subselect_loop_indirect(self): s1.add(s2) exc = self.assertRaises(mitogen.select.Error, lambda: s2.add(s0)) - self.assertEquals(str(exc), self.klass.loop_msg) + self.assertEqual(str(exc), self.klass.loop_msg) def test_double_add_receiver(self): select = self.klass() @@ -104,7 +102,7 @@ def test_double_add_receiver(self): select.add(recv) exc = self.assertRaises(mitogen.select.Error, lambda: select.add(recv)) - self.assertEquals(str(exc), self.klass.owned_msg) + self.assertEqual(str(exc), self.klass.owned_msg) def test_double_add_subselect(self): select = self.klass() @@ -112,7 +110,7 @@ def test_double_add_subselect(self): select.add(select2) exc = self.assertRaises(mitogen.select.Error, lambda: select.add(select2)) - self.assertEquals(str(exc), self.klass.owned_msg) + self.assertEqual(str(exc), self.klass.owned_msg) class RemoveTest(testlib.RouterMixin, testlib.TestCase): @@ -123,7 +121,7 @@ def test_receiver_empty(self): recv = mitogen.core.Receiver(self.router) exc = self.assertRaises(mitogen.select.Error, lambda: select.remove(recv)) - self.assertEquals(str(exc), self.klass.not_present_msg) + self.assertEqual(str(exc), self.klass.not_present_msg) def test_receiver_absent(self): select = self.klass() @@ -132,22 +130,22 @@ def test_receiver_absent(self): select.add(recv2) exc = self.assertRaises(mitogen.select.Error, lambda: select.remove(recv)) - self.assertEquals(str(exc), self.klass.not_present_msg) + self.assertEqual(str(exc), self.klass.not_present_msg) def test_receiver_present(self): select = self.klass() recv = mitogen.core.Receiver(self.router) select.add(recv) select.remove(recv) - self.assertEquals(0, len(select._receivers)) - self.assertEquals(None, recv.notify) + self.assertEqual(0, len(select._receivers)) + self.assertEqual(None, recv.notify) def test_latch_empty(self): select = self.klass() latch = mitogen.core.Latch() exc = self.assertRaises(mitogen.select.Error, lambda: select.remove(latch)) - self.assertEquals(str(exc), self.klass.not_present_msg) + self.assertEqual(str(exc), self.klass.not_present_msg) def test_latch_absent(self): select = self.klass() @@ -156,15 +154,15 @@ def test_latch_absent(self): select.add(latch2) exc = self.assertRaises(mitogen.select.Error, lambda: select.remove(latch)) - self.assertEquals(str(exc), self.klass.not_present_msg) + self.assertEqual(str(exc), self.klass.not_present_msg) def test_latch_present(self): select = self.klass() latch = mitogen.core.Latch() select.add(latch) select.remove(latch) - self.assertEquals(0, len(select._receivers)) - self.assertEquals(None, latch.notify) + self.assertEqual(0, len(select._receivers)) + self.assertEqual(None, latch.notify) class CloseTest(testlib.RouterMixin, testlib.TestCase): @@ -179,24 +177,24 @@ def test_one_latch(self): latch = mitogen.core.Latch() select.add(latch) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(select._put, latch.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(select._put, latch.notify) select.close() - self.assertEquals(0, len(select._receivers)) - self.assertEquals(None, latch.notify) + self.assertEqual(0, len(select._receivers)) + self.assertEqual(None, latch.notify) def test_one_receiver(self): select = self.klass() recv = mitogen.core.Receiver(self.router) select.add(recv) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(select._put, recv.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(select._put, recv.notify) select.close() - self.assertEquals(0, len(select._receivers)) - self.assertEquals(None, recv.notify) + self.assertEqual(0, len(select._receivers)) + self.assertEqual(None, recv.notify) def test_one_subselect(self): select = self.klass() @@ -206,16 +204,16 @@ def test_one_subselect(self): recv = mitogen.core.Receiver(self.router) subselect.add(recv) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(subselect._put, recv.notify) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(subselect._put, recv.notify) select.close() - self.assertEquals(0, len(select._receivers)) - self.assertEquals(subselect._put, recv.notify) + self.assertEqual(0, len(select._receivers)) + self.assertEqual(subselect._put, recv.notify) subselect.close() - self.assertEquals(None, recv.notify) + self.assertEqual(None, recv.notify) class EmptyTest(testlib.RouterMixin, testlib.TestCase): @@ -265,20 +263,20 @@ class IterTest(testlib.RouterMixin, testlib.TestCase): def test_empty(self): select = self.klass() - self.assertEquals([], list(select)) + self.assertEqual([], list(select)) def test_nonempty_receiver(self): recv = mitogen.core.Receiver(self.router) select = self.klass([recv]) msg = mitogen.core.Message.pickled('123') recv._on_receive(msg) - self.assertEquals([msg], list(select)) + self.assertEqual([msg], list(select)) def test_nonempty_latch(self): latch = mitogen.core.Latch() select = self.klass([latch]) latch.put(123) - self.assertEquals([123], list(select)) + self.assertEqual([123], list(select)) class OneShotTest(testlib.RouterMixin, testlib.TestCase): @@ -290,9 +288,9 @@ def test_true_receiver_removed_after_get(self): msg = mitogen.core.Message.pickled('123') recv._on_receive(msg) msg_ = select.get() - self.assertEquals(msg, msg_) - self.assertEquals(0, len(select._receivers)) - self.assertEquals(None, recv.notify) + self.assertEqual(msg, msg_) + self.assertEqual(0, len(select._receivers)) + self.assertEqual(None, recv.notify) def test_false_receiver_persists_after_get(self): recv = mitogen.core.Receiver(self.router) @@ -300,28 +298,28 @@ def test_false_receiver_persists_after_get(self): msg = mitogen.core.Message.pickled('123') recv._on_receive(msg) - self.assertEquals(msg, select.get()) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(recv, select._receivers[0]) - self.assertEquals(select._put, recv.notify) + self.assertEqual(msg, select.get()) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(recv, select._receivers[0]) + self.assertEqual(select._put, recv.notify) def test_true_latch_removed_after_get(self): latch = mitogen.core.Latch() select = self.klass([latch]) latch.put(123) - self.assertEquals(123, select.get()) - self.assertEquals(0, len(select._receivers)) - self.assertEquals(None, latch.notify) + self.assertEqual(123, select.get()) + self.assertEqual(0, len(select._receivers)) + self.assertEqual(None, latch.notify) def test_false_latch_persists_after_get(self): latch = mitogen.core.Latch() select = self.klass([latch], oneshot=False) latch.put(123) - self.assertEquals(123, select.get()) - self.assertEquals(1, len(select._receivers)) - self.assertEquals(latch, select._receivers[0]) - self.assertEquals(select._put, latch.notify) + self.assertEqual(123, select.get()) + self.assertEqual(1, len(select._receivers)) + self.assertEqual(latch, select._receivers[0]) + self.assertEqual(select._put, latch.notify) class GetReceiverTest(testlib.RouterMixin, testlib.TestCase): @@ -331,13 +329,13 @@ def test_no_receivers(self): select = self.klass() exc = self.assertRaises(mitogen.select.Error, lambda: select.get()) - self.assertEquals(str(exc), self.klass.empty_msg) + self.assertEqual(str(exc), self.klass.empty_msg) def test_timeout_no_receivers(self): select = self.klass() exc = self.assertRaises(mitogen.select.Error, lambda: select.get(timeout=1.0)) - self.assertEquals(str(exc), self.klass.empty_msg) + self.assertEqual(str(exc), self.klass.empty_msg) def test_zero_timeout(self): recv = mitogen.core.Receiver(self.router) @@ -356,7 +354,7 @@ def test_nonempty_before_add(self): recv._on_receive(mitogen.core.Message.pickled('123')) select = self.klass([recv]) msg = select.get() - self.assertEquals('123', msg.unpickle()) + self.assertEqual('123', msg.unpickle()) def test_nonempty_multiple_items_before_add(self): recv = mitogen.core.Receiver(self.router) @@ -364,9 +362,9 @@ def test_nonempty_multiple_items_before_add(self): recv._on_receive(mitogen.core.Message.pickled('234')) select = self.klass([recv], oneshot=False) msg = select.get() - self.assertEquals('123', msg.unpickle()) + self.assertEqual('123', msg.unpickle()) msg = select.get() - self.assertEquals('234', msg.unpickle()) + self.assertEqual('234', msg.unpickle()) self.assertRaises(mitogen.core.TimeoutError, lambda: select.get(block=False)) @@ -375,21 +373,21 @@ def test_nonempty_after_add(self): select = self.klass([recv]) recv._on_receive(mitogen.core.Message.pickled('123')) msg = select.get() - self.assertEquals('123', msg.unpickle()) + self.assertEqual('123', msg.unpickle()) def test_nonempty_receiver_attr_set(self): recv = mitogen.core.Receiver(self.router) select = self.klass([recv]) recv._on_receive(mitogen.core.Message.pickled('123')) msg = select.get() - self.assertEquals(msg.receiver, recv) + self.assertEqual(msg.receiver, recv) def test_drained_by_other_thread(self): recv = mitogen.core.Receiver(self.router) recv._on_receive(mitogen.core.Message.pickled('123')) select = self.klass([recv]) msg = recv.get() - self.assertEquals('123', msg.unpickle()) + self.assertEqual('123', msg.unpickle()) self.assertRaises(mitogen.core.TimeoutError, lambda: select.get(timeout=0.0)) @@ -401,13 +399,13 @@ def test_no_latches(self): select = self.klass() exc = self.assertRaises(mitogen.select.Error, lambda: select.get()) - self.assertEquals(str(exc), self.klass.empty_msg) + self.assertEqual(str(exc), self.klass.empty_msg) def test_timeout_no_receivers(self): select = self.klass() exc = self.assertRaises(mitogen.select.Error, lambda: select.get(timeout=1.0)) - self.assertEquals(str(exc), self.klass.empty_msg) + self.assertEqual(str(exc), self.klass.empty_msg) def test_zero_timeout(self): latch = mitogen.core.Latch() @@ -425,15 +423,15 @@ def test_nonempty_before_add(self): latch = mitogen.core.Latch() latch.put(123) select = self.klass([latch]) - self.assertEquals(123, select.get()) + self.assertEqual(123, select.get()) def test_nonempty_multiple_items_before_add(self): latch = mitogen.core.Latch() latch.put(123) latch.put(234) select = self.klass([latch], oneshot=False) - self.assertEquals(123, select.get()) - self.assertEquals(234, select.get()) + self.assertEqual(123, select.get()) + self.assertEqual(234, select.get()) self.assertRaises(mitogen.core.TimeoutError, lambda: select.get(block=False)) @@ -441,13 +439,13 @@ def test_nonempty_after_add(self): latch = mitogen.core.Latch() select = self.klass([latch]) latch.put(123) - self.assertEquals(123, latch.get()) + self.assertEqual(123, latch.get()) def test_drained_by_other_thread(self): latch = mitogen.core.Latch() latch.put(123) select = self.klass([latch]) - self.assertEquals(123, latch.get()) + self.assertEqual(123, latch.get()) self.assertRaises(mitogen.core.TimeoutError, lambda: select.get(timeout=0.0)) @@ -459,24 +457,20 @@ def test_empty(self): select = self.klass() exc = self.assertRaises(mitogen.select.Error, lambda: select.get()) - self.assertEquals(str(exc), self.klass.empty_msg) + self.assertEqual(str(exc), self.klass.empty_msg) def test_latch(self): latch = mitogen.core.Latch() latch.put(123) select = self.klass([latch]) event = select.get_event() - self.assertEquals(latch, event.source) - self.assertEquals(123, event.data) + self.assertEqual(latch, event.source) + self.assertEqual(123, event.data) def test_receiver(self): recv = mitogen.core.Receiver(self.router) recv._on_receive(mitogen.core.Message.pickled('123')) select = self.klass([recv]) event = select.get_event() - self.assertEquals(recv, event.source) - self.assertEquals('123', event.data.unpickle()) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(recv, event.source) + self.assertEqual('123', event.data.unpickle()) diff --git a/tests/service_test.py b/tests/service_test.py index a3e75e14a..d22558f10 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -1,5 +1,3 @@ -import unittest2 - import mitogen.core import mitogen.service import testlib @@ -45,7 +43,7 @@ def call_service_in(context, service_name, method_name): class CallTest(testlib.RouterMixin, testlib.TestCase): def test_local(self): pool = mitogen.service.get_or_create_pool(router=self.router) - self.assertEquals( + self.assertEqual( 'privileged!', mitogen.service.call(MyService, 'privileged_op') ) @@ -65,7 +63,7 @@ def test_remote_bad_arg(self): def test_local_unicode(self): pool = mitogen.service.get_or_create_pool(router=self.router) - self.assertEquals( + self.assertEqual( 'privileged!', mitogen.service.call(MyService.name(), 'privileged_op') ) @@ -73,7 +71,7 @@ def test_local_unicode(self): def test_remote(self): c1 = self.router.local() - self.assertEquals( + self.assertEqual( 'privileged!', mitogen.service.call(MyService, 'privileged_op', call_context=c1) @@ -84,15 +82,15 @@ class ActivationTest(testlib.RouterMixin, testlib.TestCase): def test_parent_can_activate(self): l1 = self.router.local() counter, id_ = l1.call_service(MyService, 'get_id') - self.assertEquals(1, counter) - self.assertTrue(isinstance(id_, int)) + self.assertEqual(1, counter) + self.assertIsInstance(id_, int) def test_sibling_cannot_activate_framework(self): l1 = self.router.local(name='l1') l2 = self.router.local(name='l2') exc = self.assertRaises(mitogen.core.CallError, lambda: l2.call(call_service_in, l1, MyService2.name(), 'get_id')) - self.assertTrue(mitogen.core.Router.refused_msg in exc.args[0]) + self.assertIn(mitogen.core.Router.refused_msg, exc.args[0]) def test_sibling_cannot_activate_service(self): l1 = self.router.local() @@ -106,15 +104,15 @@ def test_sibling_cannot_activate_service(self): finally: capture.stop() msg = mitogen.service.Activator.not_active_msg % (MyService2.name(),) - self.assertTrue(msg in exc.args[0]) + self.assertIn(msg, exc.args[0]) def test_activates_only_once(self): l1 = self.router.local() counter, id_ = l1.call_service(MyService, 'get_id') counter2, id_2 = l1.call_service(MyService, 'get_id') - self.assertEquals(1, counter) - self.assertEquals(2, counter2) - self.assertEquals(id_, id_2) + self.assertEqual(1, counter) + self.assertEqual(2, counter2) + self.assertEqual(id_, id_2) class PermissionTest(testlib.RouterMixin, testlib.TestCase): @@ -122,7 +120,7 @@ def test_sibling_unprivileged_ok(self): l1 = self.router.local() l1.call_service(MyService, 'get_id') l2 = self.router.local() - self.assertEquals('unprivileged!', + self.assertEqual('unprivileged!', l2.call(call_service_in, l1, MyService.name(), 'unprivileged_op')) def test_sibling_privileged_bad(self): @@ -140,7 +138,7 @@ def test_sibling_privileged_bad(self): u'privileged_op', MyService.name(), ) - self.assertTrue(msg in exc.args[0]) + self.assertIn(msg, exc.args[0]) class CloseTest(testlib.RouterMixin, testlib.TestCase): @@ -149,12 +147,8 @@ class CloseTest(testlib.RouterMixin, testlib.TestCase): def test_receiver_closed(self): pool = self.klass(router=self.router, services=[]) pool.stop() - self.assertEquals(None, pool._receiver.handle) + self.assertEqual(None, pool._receiver.handle) e = self.assertRaises(mitogen.core.ChannelError, lambda: self.router.myself().call_service(MyService, 'foobar')) - self.assertEquals(e.args[0], self.router.invalid_handle_msg) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(e.args[0], self.router.invalid_handle_msg) diff --git a/tests/setns_test.py b/tests/setns_test.py index 6b432d40b..b0161ef8b 100644 --- a/tests/setns_test.py +++ b/tests/setns_test.py @@ -1,12 +1,8 @@ - -import os import socket import sys +import unittest -import mitogen -import mitogen.parent - -import unittest2 +import mitogen.core import testlib @@ -18,7 +14,7 @@ # try: # root = self.router.sudo() # except mitogen.core.StreamError: -# raise unittest2.SkipTest("requires sudo to localhost root") +# raise unittest.SkipTest("requires sudo to localhost root") # via_ssh = self.docker_ssh( # username='mitogen__has_sudo', @@ -31,17 +27,13 @@ # via=root, # ) -# self.assertEquals( +# self.assertEqual( # via_ssh.call(socket.gethostname), # via_setns.call(socket.gethostname), # ) -# DockerTest = unittest2.skipIf( +# DockerTest = unittest.skipIf( # condition=sys.version_info < (2, 5), # reason="mitogen.setns unsupported on Python <2.4" # )(DockerTest) - - -# if __name__ == '__main__': -# unittest2.main() diff --git a/tests/signals_test.py b/tests/signals_test.py index 79b59e8a5..bf842e200 100644 --- a/tests/signals_test.py +++ b/tests/signals_test.py @@ -1,6 +1,3 @@ - -import unittest2 - import testlib import mitogen.core @@ -17,7 +14,7 @@ def test_no_args(self): lambda: latch.put('event fired')) mitogen.core.fire(thing, 'event') - self.assertEquals('event fired', latch.get()) + self.assertEqual('event fired', latch.get()) self.assertTrue(latch.empty()) def test_with_args(self): @@ -25,7 +22,7 @@ def test_with_args(self): latch = mitogen.core.Latch() mitogen.core.listen(thing, 'event', latch.put) mitogen.core.fire(thing, 'event', 'event fired') - self.assertEquals('event fired', latch.get()) + self.assertEqual('event fired', latch.get()) self.assertTrue(latch.empty()) def test_two_listeners(self): @@ -35,11 +32,7 @@ def test_two_listeners(self): mitogen.core.listen(thing, 'event', latch.put) mitogen.core.listen(thing, 'event', latch2.put) mitogen.core.fire(thing, 'event', 'event fired') - self.assertEquals('event fired', latch.get()) - self.assertEquals('event fired', latch2.get()) + self.assertEqual('event fired', latch.get()) + self.assertEqual('event fired', latch2.get()) self.assertTrue(latch.empty()) self.assertTrue(latch2.empty()) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/soak/cpu_load.py b/tests/soak/cpu_load.py index 8890272cc..cfc65896c 100644 --- a/tests/soak/cpu_load.py +++ b/tests/soak/cpu_load.py @@ -7,7 +7,6 @@ import ctypes import multiprocessing import os -import time LIBC = ctypes.CDLL('libc.so.6') sched_yield = LIBC.sched_yield @@ -25,7 +24,7 @@ def burn(): mul = 1.5 count = int(mul * multiprocessing.cpu_count()) -print count +print(count) procs = [multiprocessing.Process(target=burn) for _ in range(count)] diff --git a/tests/ssh_test.py b/tests/ssh_test.py index 273412e88..e8e0e1ebd 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -1,13 +1,9 @@ import os -import sys import tempfile -import mitogen import mitogen.ssh import mitogen.utils -import unittest2 - import testlib import plain_old_module @@ -38,10 +34,10 @@ def test_okay(self): ) #context.call(mitogen.utils.log_to_file, '/tmp/log') #context.call(mitogen.utils.disable_site_packages) - self.assertEquals(3, context.call(plain_old_module.add, 1, 2)) + self.assertEqual(3, context.call(plain_old_module.add, 1, 2)) -class SshTest(testlib.DockerMixin, testlib.TestCase): +class SshMixin(testlib.DockerMixin): def test_debug_decoding(self): # ensure filter_debug_logs() decodes the logged string. capture = testlib.LogCapturer() @@ -56,7 +52,7 @@ def test_debug_decoding(self): s = capture.stop() expect = "%s: debug1: Reading configuration data" % (context.name,) - self.assertTrue(expect in s) + self.assertIn(expect, s) def test_bash_permission_denied(self): # issue #271: only match Permission Denied at start of line. @@ -72,10 +68,10 @@ def test_stream_name(self): password='has_sudo_password', ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) - self.assertEquals(name, context.name) + self.assertEqual(name, context.name) def test_via_stream_name(self): context = self.docker_ssh( @@ -88,7 +84,7 @@ def test_via_stream_name(self): self.dockerized_ssh.host, self.dockerized_ssh.port, ) - self.assertEquals(name, sudo.name) + self.assertEqual(name, sudo.name) def test_password_required(self): e = self.assertRaises(mitogen.ssh.PasswordError, @@ -138,26 +134,28 @@ def test_pubkey_specified(self): def test_enforce_unknown_host_key(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: e = self.assertRaises(mitogen.ssh.HostKeyError, lambda: self.docker_ssh( username='mitogen__has_sudo_pubkey', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) ) - self.assertEquals(e.args[0], mitogen.ssh.hostkey_failed_msg) + self.assertEqual(e.args[0], mitogen.ssh.hostkey_failed_msg) finally: fp.close() def test_accept_enforce_host_keys(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='accept', ) context.shutdown(wait=True) @@ -165,12 +163,12 @@ def test_accept_enforce_host_keys(self): fp.seek(0) # Lame test, but we're about to use enforce mode anyway, which # verifies the file contents. - self.assertTrue(len(fp.read()) > 0) + self.assertGreater(len(fp.read()), 0) context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) context.shutdown(wait=True) @@ -178,7 +176,18 @@ def test_accept_enforce_host_keys(self): fp.close() -class BannerTest(testlib.DockerMixin, testlib.TestCase): +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SshTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SshMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass + + +class BannerMixin(testlib.DockerMixin): # Verify the ability to disambiguate random spam appearing in the SSHd's # login banner from a legitimate password prompt. def test_verbose_enabled(self): @@ -188,10 +197,22 @@ def test_verbose_enabled(self): ssh_debug_level=3, ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) - self.assertEquals(name, context.name) + self.assertEqual(name, context.name) + context.shutdown(wait=True) + + +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'BannerTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (BannerMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase): @@ -208,23 +229,19 @@ class StubCheckHostKeysTest(StubSshMixin, testlib.TestCase): def test_check_host_keys_accept(self): # required=true, host_key_checking=accept context = self.stub_ssh(STUBSSH_MODE='ask', check_host_keys='accept') - self.assertEquals('1', context.call(os.getenv, 'STDERR_WAS_TTY')) + self.assertEqual('1', context.call(os.getenv, 'STDERR_WAS_TTY')) def test_check_host_keys_enforce(self): # required=false, host_key_checking=enforce context = self.stub_ssh(check_host_keys='enforce') - self.assertEquals(None, context.call(os.getenv, 'STDERR_WAS_TTY')) + self.assertEqual(None, context.call(os.getenv, 'STDERR_WAS_TTY')) def test_check_host_keys_ignore(self): # required=false, host_key_checking=ignore context = self.stub_ssh(check_host_keys='ignore') - self.assertEquals(None, context.call(os.getenv, 'STDERR_WAS_TTY')) + self.assertEqual(None, context.call(os.getenv, 'STDERR_WAS_TTY')) def test_password_present(self): # required=true, password is not None context = self.stub_ssh(check_host_keys='ignore', password='willick') - self.assertEquals('1', context.call(os.getenv, 'STDERR_WAS_TTY')) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual('1', context.call(os.getenv, 'STDERR_WAS_TTY')) diff --git a/tests/su_test.py b/tests/su_test.py index 320f9cefb..3750454ce 100644 --- a/tests/su_test.py +++ b/tests/su_test.py @@ -1,11 +1,8 @@ - import os -import mitogen +import mitogen.core import mitogen.su -import unittest2 - import testlib @@ -22,11 +19,11 @@ def run_su(self, **kwargs): def test_basic(self): context, argv = self.run_su() - self.assertEquals(argv[1], 'root') - self.assertEquals(argv[2], '-c') + self.assertEqual(argv[1], 'root') + self.assertEqual(argv[2], '-c') -class SuTest(testlib.DockerMixin, testlib.TestCase): +class SuMixin(testlib.DockerMixin): stub_su_path = testlib.data_path('stubs/stub-su.py') def test_slow_auth_failure(self): @@ -48,7 +45,7 @@ def test_password_required(self): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.su(via=ssh) ) - self.assertTrue(mitogen.su.password_required_msg in str(e)) + self.assertIn(mitogen.su.password_required_msg, str(e)) def test_password_incorrect(self): ssh = self.docker_ssh( @@ -58,7 +55,7 @@ def test_password_incorrect(self): e = self.assertRaises(mitogen.core.StreamError, lambda: self.router.su(via=ssh, password='x') ) - self.assertTrue(mitogen.su.password_incorrect_msg in str(e)) + self.assertIn(mitogen.su.password_incorrect_msg, str(e)) def test_password_okay(self): ssh = self.docker_ssh( @@ -66,8 +63,15 @@ def test_password_okay(self): password='has_sudo_password', ) context = self.router.su(via=ssh, password='rootpassword') - self.assertEquals(0, context.call(os.getuid)) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(0, context.call(os.getuid)) + + +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SuTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SuMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass diff --git a/tests/sudo_test.py b/tests/sudo_test.py index 7a6523e5e..a8ec1d056 100644 --- a/tests/sudo_test.py +++ b/tests/sudo_test.py @@ -1,11 +1,5 @@ - import os -import mitogen -import mitogen.sudo - -import unittest2 - import testlib @@ -23,7 +17,7 @@ def run_sudo(self, **kwargs): def test_basic(self): context, argv = self.run_sudo() - self.assertEquals(argv[:4], [ + self.assertEqual(argv[:4], [ self.sudo_path, '-u', 'root', '--' @@ -34,7 +28,7 @@ def test_selinux_type_role(self): selinux_type='setype', selinux_role='serole', ) - self.assertEquals(argv[:8], [ + self.assertEqual(argv[:8], [ self.sudo_path, '-u', 'root', '-r', 'serole', @@ -46,7 +40,7 @@ def test_reparse_args(self): context, argv = self.run_sudo( sudo_args=['--type', 'setype', '--role', 'serole', '--user', 'user'] ) - self.assertEquals(argv[:8], [ + self.assertEqual(argv[:8], [ self.sudo_path, '-u', 'user', '-r', 'serole', @@ -59,7 +53,7 @@ def test_tty_preserved(self): os.environ['PREHISTORIC_SUDO'] = '1' try: context, argv = self.run_sudo() - self.assertEquals('1', context.call(os.getenv, 'PREHISTORIC_SUDO')) + self.assertEqual('1', context.call(os.getenv, 'PREHISTORIC_SUDO')) finally: del os.environ['PREHISTORIC_SUDO'] @@ -79,7 +73,7 @@ def test_tty_preserved(self): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.sudo(via=ssh) # ) -# self.assertTrue(mitogen.sudo.password_required_msg in str(e)) +# self.assertIn(mitogen.sudo.password_required_msg, str(e)) # def test_password_incorrect(self): # ssh = self.docker_ssh( @@ -91,7 +85,7 @@ def test_tty_preserved(self): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.sudo(via=ssh, password='x') # ) -# self.assertTrue(mitogen.sudo.password_incorrect_msg in str(e)) +# self.assertIn(mitogen.sudo.password_incorrect_msg, str(e)) # def test_password_okay(self): # ssh = self.docker_ssh( @@ -103,8 +97,4 @@ def test_tty_preserved(self): # e = self.assertRaises(mitogen.core.StreamError, # lambda: self.router.sudo(via=ssh, password='rootpassword') # ) -# self.assertTrue(mitogen.sudo.password_incorrect_msg in str(e)) - - -if __name__ == '__main__': - unittest2.main() +# self.assertIn(mitogen.sudo.password_incorrect_msg, str(e)) diff --git a/tests/testlib.py b/tests/testlib.py index e16309ee1..05779dc0f 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,17 +1,27 @@ - +import errno +import io import logging import os import random import re -import signal import socket -import subprocess +import stat import sys import threading import time import traceback +import unittest + +try: + import configparser +except ImportError: + import ConfigParser as configparser -import unittest2 +import psutil +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess import mitogen.core import mitogen.fork @@ -40,8 +50,22 @@ LOG = logging.getLogger(__name__) -DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') -MODS_DIR = os.path.join(DATA_DIR, 'importer') + +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + +TESTS_DIR = os.path.join(os.path.dirname(__file__)) +ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') +ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') +ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules') +DATA_DIR = os.path.join(TESTS_DIR, 'data') +MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer') sys.path.append(DATA_DIR) sys.path.append(MODS_DIR) @@ -62,13 +86,65 @@ mitogen.core.LOG.propagate = True +def base_executable(executable=None): + '''Return the path of the Python executable used to create the virtualenv. + ''' + # https://docs.python.org/3/library/venv.html + # https://github.com/pypa/virtualenv/blob/main/src/virtualenv/discovery/py_info.py + # https://virtualenv.pypa.io/en/16.7.9/reference.html#compatibility-with-the-stdlib-venv-module + if executable is None: + executable = sys.executable -def get_fd_count(): - """ - Return the number of FDs open by this process. - """ - import psutil - return psutil.Process().num_fds() + if not executable: + raise ValueError + + try: + base_executable = sys._base_executable + except AttributeError: + base_executable = None + + if base_executable and base_executable != executable: + return base_executable + + # Python 2.x only has sys.base_prefix if running outside a virtualenv. + try: + sys.base_prefix + except AttributeError: + # Python 2.x outside a virtualenv + return executable + + # Python 3.3+ has sys.base_prefix. In a virtualenv it differs to sys.prefix. + if sys.base_prefix == sys.prefix: + return executable + + while executable.startswith(sys.prefix) and stat.S_ISLNK(os.lstat(executable).st_mode): + dirname = os.path.dirname(executable) + target = os.path.join(dirname, os.readlink(executable)) + executable = os.path.abspath(os.path.normpath(target)) + print(executable) + + if executable.startswith(sys.base_prefix): + return executable + + # Virtualenvs record details in pyvenv.cfg + parser = configparser.RawConfigParser() + with io.open(os.path.join(sys.prefix, 'pyvenv.cfg'), encoding='utf-8') as f: + content = u'[virtualenv]\n' + f.read() + try: + parser.read_string(content) + except AttributeError: + parser.readfp(io.StringIO(content)) + + # virtualenv style pyvenv.cfg includes the base executable. + # venv style pyvenv.cfg doesn't. + try: + return parser.get(u'virtualenv', u'base-executable') + except configparser.NoOptionError: + pass + + basename = os.path.basename(executable) + home = parser.get(u'virtualenv', u'home') + return os.path.join(home, basename) def data_path(suffix): @@ -79,28 +155,15 @@ def data_path(suffix): return path -def subprocess__check_output(*popenargs, **kwargs): - # Missing from 2.6. - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, _ = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output - - -def Popen__terminate(proc): - os.kill(proc.pid, signal.SIGTERM) - - -if hasattr(subprocess, 'check_output'): - subprocess__check_output = subprocess.check_output - -if hasattr(subprocess.Popen, 'terminate'): - Popen__terminate = subprocess.Popen.terminate +def retry(fn, on, max_attempts, delay): + for i in range(max_attempts): + try: + return fn() + except on: + if i >= max_attempts - 1: + raise + else: + time.sleep(delay) def threading__thread_is_alive(thread): @@ -115,6 +178,13 @@ def threading__thread_is_alive(thread): return thread.isAlive() +def threading_thread_name(thread): + try: + return thread.name # Available in Python 2.6+ + except AttributeError: + return thread.getName() # Deprecated in Python 3.10+ + + def wait_for_port( host, port, @@ -144,7 +214,7 @@ def wait_for_port( if not pattern: # Success: We connected & there's no banner check to perform. - sock.shutdown(socket.SHUTD_RDWR) + sock.shutdown(socket.SHUT_RDWR) sock.close() return @@ -327,13 +397,17 @@ def stop(self): return self.raw() -class TestCase(unittest2.TestCase): +class TestCase(unittest.TestCase): @classmethod def setUpClass(cls): # This is done in setUpClass() so we have a chance to run before any # Broker() instantiations in setUp() etc. mitogen.fork.on_fork() - cls._fd_count_before = get_fd_count() + cls._fds_before = psutil.Process().open_files() + # Ignore children started by external packages - in particular + # multiprocessing.resource_tracker.main()`, started when some Ansible + # versions instantiate a `multithreading.Lock()`. + cls._children_before = frozenset(psutil.Process().children()) super(TestCase, cls).setUpClass() ALLOWED_THREADS = set([ @@ -344,7 +418,7 @@ def setUpClass(cls): def _teardown_check_threads(self): counts = {} for thread in threading.enumerate(): - name = thread.getName() + name = threading_thread_name(thread) # Python 2.4: enumerate() may return stopped threads. assert \ not threading__thread_is_alive(thread) \ @@ -360,10 +434,23 @@ def _teardown_check_threads(self): def _teardown_check_fds(self): mitogen.core.Latch._on_fork() - if get_fd_count() != self._fd_count_before: - import os; os.system('lsof +E -w -p %s | grep -vw mem' % (os.getpid(),)) - assert 0, "%s leaked FDs. Count before: %s, after: %s" % ( - self, self._fd_count_before, get_fd_count(), + fds_after = psutil.Process().open_files() + fds_leaked = len(self._fds_before) != len(fds_after) + if not fds_leaked: + return + else: + if sys.platform == 'linux': + subprocess.check_call( + 'lsof +E -w -p %i | grep -vw mem' % (os.getpid(),), + shell=True, + ) + else: + subprocess.check_call( + 'lsof -w -p %i | grep -vw mem' % (os.getpid(),), + shell=True, + ) + assert 0, "%s leaked FDs: %s\nBefore:\t%s\nAfter:\t%s" % ( + self, fds_leaked, self._fds_before, fds_after, ) # Some class fixtures (like Ansible MuxProcess) start persistent children @@ -374,19 +461,39 @@ def _teardown_check_zombies(self): if self.no_zombie_check: return + # pid=0: Wait for any child process in the same process group as us. + # WNOHANG: Don't block if no processes ready to report status. try: pid, status = os.waitpid(0, os.WNOHANG) - except OSError: - return # ECHILD + except OSError as e: + # ECHILD: there are no child processes in our group. + if e.errno == errno.ECHILD: + return + raise if pid: assert 0, "%s failed to reap subprocess %d (status %d)." % ( self, pid, status ) - print('') - print('Children of unit test process:') - os.system('ps uww --ppid ' + str(os.getpid())) + children_after = frozenset(psutil.Process().children()) + children_leaked = children_after.difference(self._children_before) + if not children_leaked: + return + + print('Leaked children of unit test process:') + subprocess.check_call( + ['ps', '-o', 'user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,command', '-ww', '-p', + ','.join(str(p.pid) for p in children_leaked), + ], + ) + if self._children_before: + print('Pre-existing children of unit test process:') + subprocess.check_call( + ['ps', '-o', 'user,pid,%cpu,%mem,vsz,rss,tty,stat,start,time,command', '-ww', '-p', + ','.join(str(p.pid) for p in self._children_before), + ], + ) assert 0, "%s leaked still-running subprocesses." % (self,) def tearDown(self): @@ -411,6 +518,7 @@ def assertRaises(self, exc, func, *args, **kwargs): def get_docker_host(): + # Duplicated in ci_lib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -420,22 +528,24 @@ def get_docker_host(): class DockerizedSshDaemon(object): - def _get_container_port(self): - s = subprocess__check_output(['docker', 'port', self.container_name]) - for line in s.decode().splitlines(): - dport, proto, baddr, bport = self.PORT_RE.match(line).groups() - if dport == '22' and proto == 'tcp': - self.port = int(bport) - - self.host = self.get_host() - if self.port is None: + PORT_RE = re.compile( + # e.g. 0.0.0.0:32771, :::32771, [::]:32771' + r'(?P[0-9.]+|::|\[[a-f0-9:.]+\]):(?P[0-9]+)', + ) + + @classmethod + def get_port(cls, container): + s = subprocess.check_output(['docker', 'port', container, '22/tcp']) + m = cls.PORT_RE.search(s.decode()) + if not m: raise ValueError('could not find SSH port in: %r' % (s,)) + return int(m.group('port')) def start_container(self): try: - subprocess__check_output(['docker', '--version']) + subprocess.check_output(['docker', '--version']) except Exception: - raise unittest2.SkipTest('Docker binary is unavailable') + raise unittest.SkipTest('Docker binary is unavailable') self.container_name = 'mitogen-test-%08x' % (random.getrandbits(64),) args = [ @@ -447,58 +557,66 @@ def start_container(self): '--name', self.container_name, self.image, ] - subprocess__check_output(args) - self._get_container_port() + subprocess.check_output(args) + self.port = self.get_port(self.container_name) + + def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE): + # Code duplicated in ci_lib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) + d = distro_pattern.match(distro_spec).groupdict(default=None) - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): - if '-' in mitogen_test_distro: - distro, _py3 = mitogen_test_distro.split('-') - else: - distro = mitogen_test_distro - _py3 = None + self.distro = d['distro'] + self.family = d['family'] - if _py3 == 'py3': + if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' else: self.python_path = '/usr/bin/python' - self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - - # 22/tcp -> 0.0.0.0:32771 - self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)') - self.port = None - - self.start_container() - - def get_host(self): - return get_docker_host() + self.image = image_template % d + self.host = get_docker_host() def wait_for_sshd(self): - wait_for_port(self.get_host(), self.port, pattern='OpenSSH') + wait_for_port(self.host, self.port, pattern='OpenSSH') def check_processes(self): - args = ['docker', 'exec', self.container_name, 'ps', '-o', 'comm='] + # Get Accounting name (ucomm) & command line (args) of each process + # in the container. No truncation (-ww). No column headers (foo=). + ps_output = subprocess.check_output([ + 'docker', 'exec', self.container_name, + 'ps', '-w', '-w', '-o', 'ucomm=', '-o', 'args=', + ]) + ps_lines = ps_output.decode().splitlines() + processes = [tuple(line.split(None, 1)) for line in ps_lines] counts = {} - for comm in subprocess__check_output(args).decode().splitlines(): - comm = comm.strip() - counts[comm] = counts.get(comm, 0) + 1 + for ucomm, _ in processes: + counts[ucomm] = counts.get(ucomm, 0) + 1 if counts != {'ps': 1, 'sshd': 1}: assert 0, ( 'Docker container %r contained extra running processes ' 'after test completed: %r' % ( self.container_name, - counts + processes, ) ) def close(self): args = ['docker', 'rm', '-f', self.container_name] - subprocess__check_output(args) + subprocess.check_output(args) class BrokerMixin(object): broker_class = mitogen.master.Broker + + # Flag for tests that shutdown the broker themself + # e.g. unix_test.ListenerTest broker_shutdown = False def setUp(self): @@ -533,28 +651,52 @@ class DockerMixin(RouterMixin): def setUpClass(cls): super(DockerMixin, cls).setUpClass() if os.environ.get('SKIP_DOCKER_TESTS'): - raise unittest2.SkipTest('SKIP_DOCKER_TESTS is set') + raise unittest.SkipTest('SKIP_DOCKER_TESTS is set') - # we want to be able to override test distro for some tests that need a different container spun up - daemon_args = {} - if hasattr(cls, 'mitogen_test_distro'): - daemon_args['mitogen_test_distro'] = cls.mitogen_test_distro - - cls.dockerized_ssh = DockerizedSshDaemon(**daemon_args) + # cls.dockerized_ssh is injected by dynamically generating TestCase + # subclasses. + # TODO Bite the bullet, switch to e.g. pytest + cls.dockerized_ssh.start_container() cls.dockerized_ssh.wait_for_sshd() @classmethod def tearDownClass(cls): - cls.dockerized_ssh.check_processes() + retry( + cls.dockerized_ssh.check_processes, + on=AssertionError, + max_attempts=5, + delay=0.1, + ) cls.dockerized_ssh.close() super(DockerMixin, cls).tearDownClass() + @property + def docker_ssh_default_kwargs(self): + return { + 'hostname': self.dockerized_ssh.host, + 'port': self.dockerized_ssh.port, + 'check_host_keys': 'ignore', + 'ssh_debug_level': 3, + # https://www.openssh.com/legacy.html + # ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. + # Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). + # Duplicated cases in + # - tests/ansible/ansible.cfg + # - tests/ansible/integration/connection_delegation/delegate_to_template.yml + # - tests/ansible/integration/connection_delegation/stack_construction.yml + # - tests/ansible/integration/process/unix_socket_cleanup.yml + # - tests/ansible/integration/ssh/variables.yml + # - tests/testlib.py + 'ssh_args': [ + '-o', 'HostKeyAlgorithms +ssh-rsa', + '-o', 'PubkeyAcceptedKeyTypes +ssh-rsa', + ], + 'python_path': self.dockerized_ssh.python_path, + } + def docker_ssh(self, **kwargs): - kwargs.setdefault('hostname', self.dockerized_ssh.host) - kwargs.setdefault('port', self.dockerized_ssh.port) - kwargs.setdefault('check_host_keys', 'ignore') - kwargs.setdefault('ssh_debug_level', 3) - kwargs.setdefault('python_path', self.dockerized_ssh.python_path) + for k, v in self.docker_ssh_default_kwargs.items(): + kwargs.setdefault(k, v) return self.router.ssh(**kwargs) def docker_ssh_any(self, **kwargs): diff --git a/tests/timer_test.py b/tests/timer_test.py index 749405a43..f8f203a98 100644 --- a/tests/timer_test.py +++ b/tests/timer_test.py @@ -1,8 +1,7 @@ - -import time - -import mock -import unittest2 +try: + from unittest import mock +except ImportError: + import mock import mitogen.core import mitogen.parent @@ -19,49 +18,49 @@ def setUp(self): class GetTimeoutTest(TimerListMixin, testlib.TestCase): def test_empty(self): - self.assertEquals(None, self.list.get_timeout()) + self.assertEqual(None, self.list.get_timeout()) def test_one_event(self): self.list.schedule(2, lambda: None) self.list._now = lambda: 1 - self.assertEquals(1, self.list.get_timeout()) + self.assertEqual(1, self.list.get_timeout()) def test_two_events_same_moment(self): self.list.schedule(2, lambda: None) self.list.schedule(2, lambda: None) self.list._now = lambda: 1 - self.assertEquals(1, self.list.get_timeout()) + self.assertEqual(1, self.list.get_timeout()) def test_two_events(self): self.list.schedule(2, lambda: None) self.list.schedule(3, lambda: None) self.list._now = lambda: 1 - self.assertEquals(1, self.list.get_timeout()) + self.assertEqual(1, self.list.get_timeout()) def test_two_events_expired(self): self.list.schedule(2, lambda: None) self.list.schedule(3, lambda: None) self.list._now = lambda: 3 - self.assertEquals(0, self.list.get_timeout()) + self.assertEqual(0, self.list.get_timeout()) def test_two_events_in_past(self): self.list.schedule(2, lambda: None) self.list.schedule(3, lambda: None) self.list._now = lambda: 30 - self.assertEquals(0, self.list.get_timeout()) + self.assertEqual(0, self.list.get_timeout()) def test_two_events_in_past(self): self.list.schedule(2, lambda: None) self.list.schedule(3, lambda: None) self.list._now = lambda: 30 - self.assertEquals(0, self.list.get_timeout()) + self.assertEqual(0, self.list.get_timeout()) def test_one_cancelled(self): t1 = self.list.schedule(2, lambda: None) t2 = self.list.schedule(3, lambda: None) self.list._now = lambda: 0 t1.cancel() - self.assertEquals(3, self.list.get_timeout()) + self.assertEqual(3, self.list.get_timeout()) def test_two_cancelled(self): t1 = self.list.schedule(2, lambda: None) @@ -69,30 +68,30 @@ def test_two_cancelled(self): self.list._now = lambda: 0 t1.cancel() t2.cancel() - self.assertEquals(None, self.list.get_timeout()) + self.assertEqual(None, self.list.get_timeout()) class ScheduleTest(TimerListMixin, testlib.TestCase): def test_in_past(self): self.list._now = lambda: 30 timer = self.list.schedule(29, lambda: None) - self.assertEquals(29, timer.when) - self.assertEquals(0, self.list.get_timeout()) + self.assertEqual(29, timer.when) + self.assertEqual(0, self.list.get_timeout()) def test_in_future(self): self.list._now = lambda: 30 timer = self.list.schedule(31, lambda: None) - self.assertEquals(31, timer.when) - self.assertEquals(1, self.list.get_timeout()) + self.assertEqual(31, timer.when) + self.assertEqual(1, self.list.get_timeout()) def test_same_moment(self): self.list._now = lambda: 30 timer = self.list.schedule(31, lambda: None) timer2 = self.list.schedule(31, lambda: None) - self.assertEquals(31, timer.when) - self.assertEquals(31, timer2.when) - self.assertTrue(timer is not timer2) - self.assertEquals(1, self.list.get_timeout()) + self.assertEqual(31, timer.when) + self.assertEqual(31, timer2.when) + self.assertIsNot(timer, timer2) + self.assertEqual(1, self.list.get_timeout()) class ExpireTest(TimerListMixin, testlib.TestCase): @@ -101,7 +100,7 @@ def test_in_past(self): self.assertTrue(timer.active) self.list._now = lambda: 30 self.list.expire() - self.assertEquals(1, len(timer.func.mock_calls)) + self.assertEqual(1, len(timer.func.mock_calls)) self.assertFalse(timer.active) def test_in_future(self): @@ -109,7 +108,7 @@ def test_in_future(self): self.assertTrue(timer.active) self.list._now = lambda: 28 self.list.expire() - self.assertEquals(0, len(timer.func.mock_calls)) + self.assertEqual(0, len(timer.func.mock_calls)) self.assertTrue(timer.active) def test_same_moment(self): @@ -119,8 +118,8 @@ def test_same_moment(self): self.assertTrue(timer2.active) self.list._now = lambda: 29 self.list.expire() - self.assertEquals(1, len(timer.func.mock_calls)) - self.assertEquals(1, len(timer2.func.mock_calls)) + self.assertEqual(1, len(timer.func.mock_calls)) + self.assertEqual(1, len(timer2.func.mock_calls)) self.assertFalse(timer.active) self.assertFalse(timer2.active) @@ -128,11 +127,11 @@ def test_cancelled(self): self.list._now = lambda: 29 timer = self.list.schedule(29, mock.Mock()) timer.cancel() - self.assertEquals(None, self.list.get_timeout()) + self.assertEqual(None, self.list.get_timeout()) self.list._now = lambda: 29 self.list.expire() - self.assertEquals(0, len(timer.func.mock_calls)) - self.assertEquals(None, self.list.get_timeout()) + self.assertEqual(0, len(timer.func.mock_calls)) + self.assertEqual(None, self.list.get_timeout()) class CancelTest(TimerListMixin, testlib.TestCase): @@ -143,7 +142,7 @@ def test_single_cancel(self): timer.cancel() self.assertFalse(timer.active) self.list.expire() - self.assertEquals(0, len(timer.func.mock_calls)) + self.assertEqual(0, len(timer.func.mock_calls)) def test_double_cancel(self): self.list._now = lambda: 29 @@ -153,7 +152,7 @@ def test_double_cancel(self): timer.cancel() self.assertFalse(timer.active) self.list.expire() - self.assertEquals(0, len(timer.func.mock_calls)) + self.assertEqual(0, len(timer.func.mock_calls)) @mitogen.core.takes_econtext @@ -195,7 +194,3 @@ def test_child_upgrade(self): finally: router.broker.shutdown() router.broker.join() - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index f30a233ec..ab9f4e19e 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -1,11 +1,7 @@ - -import logging -import time - -import unittest2 +import os +import unittest import mitogen.core -import mitogen.master import testlib import simple_pkg.ping @@ -14,6 +10,10 @@ # TODO: this is a joke. 2/3 interop is one of the hardest bits to get right. # There should be 100 tests in this file. +@unittest.skipIf( + os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21, + "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)", +) class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: python_path = 'python2' @@ -25,9 +25,5 @@ def test_succeeds(self): target = self.router.local(python_path=self.python_path) spare2, = target.call(simple_pkg.ping.ping, spare) - self.assertEquals(spare.context_id, spare2.context_id) - self.assertEquals(spare.name, spare2.name) - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(spare.context_id, spare2.context_id) + self.assertEqual(spare.name, spare2.name) diff --git a/tests/types_test.py b/tests/types_test.py index 8e441c651..0229b94cc 100644 --- a/tests/types_test.py +++ b/tests/types_test.py @@ -1,4 +1,3 @@ - import sys try: @@ -8,7 +7,7 @@ from StringIO import StringIO as StringIO from StringIO import StringIO as BytesIO -import unittest2 +import unittest import mitogen.core from mitogen.core import b @@ -29,26 +28,26 @@ def make(self): def test_repr(self): blob = self.make() - self.assertEquals('[blob: 128 bytes]', repr(blob)) + self.assertEqual('[blob: 128 bytes]', repr(blob)) def test_decays_on_constructor(self): blob = self.make() - self.assertEquals(b('x') * 128, mitogen.core.BytesType(blob)) + self.assertEqual(b('x') * 128, mitogen.core.BytesType(blob)) def test_decays_on_write(self): blob = self.make() io = BytesIO() io.write(blob) - self.assertEquals(128, io.tell()) - self.assertEquals(b('x') * 128, io.getvalue()) + self.assertEqual(128, io.tell()) + self.assertEqual(b('x') * 128, io.getvalue()) def test_message_roundtrip(self): blob = self.make() msg = mitogen.core.Message.pickled(blob) blob2 = msg.unpickle() - self.assertEquals(type(blob), type(blob2)) - self.assertEquals(repr(blob), repr(blob2)) - self.assertEquals(mitogen.core.BytesType(blob), + self.assertEqual(type(blob), type(blob2)) + self.assertEqual(repr(blob), repr(blob2)) + self.assertEqual(mitogen.core.BytesType(blob), mitogen.core.BytesType(blob2)) @@ -60,26 +59,26 @@ def make(self): def test_repr(self): secret = self.make() - self.assertEquals('[secret]', repr(secret)) + self.assertEqual('[secret]', repr(secret)) def test_decays_on_constructor(self): secret = self.make() - self.assertEquals('password', mitogen.core.UnicodeType(secret)) + self.assertEqual('password', mitogen.core.UnicodeType(secret)) def test_decays_on_write(self): secret = self.make() io = StringIO() io.write(secret) - self.assertEquals(8, io.tell()) - self.assertEquals('password', io.getvalue()) + self.assertEqual(8, io.tell()) + self.assertEqual('password', io.getvalue()) def test_message_roundtrip(self): secret = self.make() msg = mitogen.core.Message.pickled(secret) secret2 = msg.unpickle() - self.assertEquals(type(secret), type(secret2)) - self.assertEquals(repr(secret), repr(secret2)) - self.assertEquals(mitogen.core.b(secret), + self.assertEqual(type(secret), type(secret2)) + self.assertEqual(repr(secret), repr(secret2)) + self.assertEqual(mitogen.core.b(secret), mitogen.core.b(secret2)) @@ -88,32 +87,32 @@ class KwargsTest(testlib.TestCase): def test_empty(self): kw = self.klass({}) - self.assertEquals({}, kw) - self.assertEquals('Kwargs({})', repr(kw)) + self.assertEqual({}, kw) + self.assertEqual('Kwargs({})', repr(kw)) klass, (dct,) = kw.__reduce__() - self.assertTrue(klass is self.klass) - self.assertTrue(type(dct) is dict) - self.assertEquals({}, dct) + self.assertIs(klass, self.klass) + self.assertIs(type(dct), dict) + self.assertEqual({}, dct) - @unittest2.skipIf(condition=(sys.version_info >= (2, 6)), + @unittest.skipIf(condition=(sys.version_info >= (2, 6)), reason='py<2.6 only') def test_bytes_conversion(self): kw = self.klass({u'key': 123}) - self.assertEquals({'key': 123}, kw) - self.assertEquals("Kwargs({'key': 123})", repr(kw)) + self.assertEqual({'key': 123}, kw) + self.assertEqual("Kwargs({'key': 123})", repr(kw)) - @unittest2.skipIf(condition=not mitogen.core.PY3, + @unittest.skipIf(condition=not mitogen.core.PY3, reason='py3 only') def test_unicode_conversion(self): kw = self.klass({mitogen.core.b('key'): 123}) - self.assertEquals({u'key': 123}, kw) - self.assertEquals("Kwargs({'key': 123})", repr(kw)) + self.assertEqual({u'key': 123}, kw) + self.assertEqual("Kwargs({'key': 123})", repr(kw)) klass, (dct,) = kw.__reduce__() - self.assertTrue(klass is self.klass) - self.assertTrue(type(dct) is dict) - self.assertEquals({u'key': 123}, dct) + self.assertIs(klass, self.klass) + self.assertIs(type(dct), dict) + self.assertEqual({u'key': 123}, dct) key, = dct - self.assertTrue(type(key) is mitogen.core.UnicodeType) + self.assertIs(type(key), mitogen.core.UnicodeType) class AdornedUnicode(mitogen.core.UnicodeType): @@ -125,24 +124,20 @@ class ToTextTest(testlib.TestCase): def test_bytes(self): s = self.func(mitogen.core.b('bytes')) - self.assertEquals(mitogen.core.UnicodeType, type(s)) - self.assertEquals(s, u'bytes') + self.assertEqual(mitogen.core.UnicodeType, type(s)) + self.assertEqual(s, u'bytes') def test_unicode(self): s = self.func(u'text') - self.assertEquals(mitogen.core.UnicodeType, type(s)) - self.assertEquals(s, u'text') + self.assertEqual(mitogen.core.UnicodeType, type(s)) + self.assertEqual(s, u'text') def test_adorned_unicode(self): s = self.func(AdornedUnicode(u'text')) - self.assertEquals(mitogen.core.UnicodeType, type(s)) - self.assertEquals(s, u'text') + self.assertEqual(mitogen.core.UnicodeType, type(s)) + self.assertEqual(s, u'text') def test_integer(self): s = self.func(123) - self.assertEquals(mitogen.core.UnicodeType, type(s)) - self.assertEquals(s, u'123') - - -if __name__ == '__main__': - unittest2.main() + self.assertEqual(mitogen.core.UnicodeType, type(s)) + self.assertEqual(s, u'123') diff --git a/tests/unix_test.py b/tests/unix_test.py index cf3e595f3..e251a7ade 100644 --- a/tests/unix_test.py +++ b/tests/unix_test.py @@ -1,12 +1,9 @@ - import os import socket import subprocess import sys import time -import unittest2 - import mitogen import mitogen.master import mitogen.service @@ -68,17 +65,13 @@ class ListenerTest(testlib.RouterMixin, testlib.TestCase): def test_constructor_basic(self): listener = self.klass.build_stream(router=self.router) - capture = testlib.LogCapturer() - capture.start() - try: - self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path)) - os.unlink(listener.protocol.path) - # ensure we catch 0 byte read error log message - self.broker.shutdown() - self.broker.join() - self.broker_shutdown = True - finally: - capture.stop() + self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path)) + os.unlink(listener.protocol.path) + + # ensure we catch 0 byte read error log message + self.broker.shutdown() + self.broker.join() + self.broker_shutdown = True class ClientTest(testlib.TestCase): @@ -98,12 +91,12 @@ def _try_connect(self, path): def _test_simple_client(self, path): router, context = self._try_connect(path) try: - self.assertEquals(0, context.context_id) - self.assertEquals(1, mitogen.context_id) - self.assertEquals(0, mitogen.parent_id) + self.assertEqual(0, context.context_id) + self.assertEqual(1, mitogen.context_id) + self.assertEqual(0, mitogen.parent_id) resp = context.call_service(service_name=MyService, method_name='ping') - self.assertEquals(mitogen.context_id, resp['src_id']) - self.assertEquals(0, resp['auth_id']) + self.assertEqual(mitogen.context_id, resp['src_id']) + self.assertEqual(0, resp['auth_id']) finally: router.broker.shutdown() router.broker.join() @@ -133,7 +126,8 @@ def _test_simple_server(cls, path): def test_simple(self): path = mitogen.unix.make_socket_path() proc = subprocess.Popen( - [sys.executable, __file__, 'ClientTest_server', path] + [sys.executable, __file__, 'ClientTest_server', path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) try: self._test_simple_client(path) @@ -142,11 +136,11 @@ def test_simple(self): mitogen.context_id = 0 mitogen.parent_id = None mitogen.parent_ids = [] - proc.wait() + b_stdout, _ = proc.communicate() + self.assertEqual(proc.returncode, 0) + self.assertEqual(b_stdout.decode(), '') if __name__ == '__main__': if len(sys.argv) == 3 and sys.argv[1] == 'ClientTest_server': ClientTest._test_simple_server(path=sys.argv[2]) - else: - unittest2.main() diff --git a/tests/utils_test.py b/tests/utils_test.py index b5204a3c9..372469612 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -1,13 +1,4 @@ -#!/usr/bin/env python - -import os -import tempfile - -import unittest2 -import mock - import mitogen.core -import mitogen.parent import mitogen.master import mitogen.utils from mitogen.core import b @@ -21,6 +12,7 @@ def func0(router): @mitogen.utils.with_router def func(router): + "Docstring of func" return router @@ -40,6 +32,10 @@ def test_with_broker(self): self.assertIsInstance(router, mitogen.master.Router) self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread)) + def test_with_broker_preserves_attributes(self): + self.assertEqual(func.__doc__, 'Docstring of func') + self.assertEqual(func.__name__, 'func') + class Dict(dict): pass class List(list): pass @@ -48,6 +44,44 @@ class Unicode(mitogen.core.UnicodeType): pass class Bytes(mitogen.core.BytesType): pass +class StubbornBytes(mitogen.core.BytesType): + """ + A binary string type that persists through `bytes(...)`. + + Stand-in for `AnsibleUnsafeBytes()` in Ansible 7-9 (core 2.14-2.16), after + fixes/mitigations for CVE-2023-5764. + """ + if mitogen.core.PY3: + def __bytes__(self): return self + def __str__(self): return self.decode() + else: + def __str__(self): return self + def __unicode__(self): return self.decode() + + def decode(self, encoding='utf-8', errors='strict'): + s = super(StubbornBytes).encode(encoding=encoding, errors=errors) + return StubbornText(s) + + +class StubbornText(mitogen.core.UnicodeType): + """ + A text string type that persists through `unicode(...)` or `str(...)`. + + Stand-in for `AnsibleUnsafeText()` in Ansible 7-9 (core 2.14-2.16), after + following fixes/mitigations for CVE-2023-5764. + """ + if mitogen.core.PY3: + def __bytes__(self): return self.encode() + def __str__(self): return self + else: + def __str__(self): return self.encode() + def __unicode__(self): return self + + def encode(self, encoding='utf-8', errors='strict'): + s = super(StubbornText).encode(encoding=encoding, errors=errors) + return StubbornBytes(s) + + class CastTest(testlib.TestCase): def test_dict(self): self.assertEqual(type(mitogen.utils.cast({})), dict) @@ -95,10 +129,15 @@ def test_bytes(self): self.assertEqual(type(mitogen.utils.cast(b(''))), mitogen.core.BytesType) self.assertEqual(type(mitogen.utils.cast(Bytes())), mitogen.core.BytesType) + def test_stubborn_types_raise(self): + stubborn_bytes = StubbornBytes(b('abc')) + self.assertIs(stubborn_bytes, mitogen.core.BytesType(stubborn_bytes)) + self.assertRaises(TypeError, mitogen.utils.cast, stubborn_bytes) + + stubborn_text = StubbornText(u'abc') + self.assertIs(stubborn_text, mitogen.core.UnicodeType(stubborn_text)) + self.assertRaises(TypeError, mitogen.utils.cast, stubborn_text) + def test_unknown(self): self.assertRaises(TypeError, mitogen.utils.cast, set()) self.assertRaises(TypeError, mitogen.utils.cast, 4j) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tox.ini b/tox.ini index eda4e567f..ea988aa0c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,40 +1,64 @@ -# This file is a local convenience. It is not a substitute for the full CI -# suite, and does not cover the full range of Python versions for Mitogen. - -# I use this on Ubuntu 20.04, with the following additions +# This configuration drives both CI and local development. +# I use this locally on Ubuntu 22.04, with the following additions # # sudo add-apt-repository ppa:deadsnakes/ppa -# sudo apt update -# sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass +# sudo apt install lib{ldap2,sasl2,ssl}-dev python{2,2.7,3}{,-dev} python3.{7..13}{,-dev,-venv} python-is-python3 sshpass tox -# Last version to support each python version +# Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv +# ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== +# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 +# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 +# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 +# 2.7 <= 2.11 <= 2.16 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² +# 3.5 <= 2.11 <= 2.15 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² +# 3.6 <= 2.11 <= 2.16 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² +# 3.7 <= 2.12 <= 2.17 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0 +# 3.8 <= 2.12 +# 3.9 <= 2.15 +# 3.10 <= 2.17 +# 3.11 +# 3.12 >= 2.13¹ +# +# Notes +# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13). +# Python 3.12 removed support for find_module(), replaced by find_spec(). +# In Ansible <= 4.x ansible.module_utils.six lacks find_spec(). +# https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3 +# In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder +# lacks find_spec(). https://github.com/ansible/ansible/pull/76225 +# +# Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15). +# Python 3.12 removed deprecated httplib.HTTPSConnection() arguments. +# https://github.com/ansible/ansible/pull/80751 # -# tox vir'env pip ansible ansible coverage -# control target -# ========== ======== ======== ======== ======== ======== ======== -# python2.4 1.4 1.8 1.1 2.3? -# python2.5 1.6.1 1.9.1 1.3.1 ??? -# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 4.5.4 -# python2.7 20.3 2.10 -# python3.5 2.10 -# python3.6 2.10 -# python3.7 2.10 +# 2. Higher virtualenv versions cannot run under this Python version. They can +# still generate virtual environments for it. -# pip --no-python-version-warning -# pip --disable-pip-version-check +# Ansible Dependency +# ================== ====================== +# ansible <= 2.9 +# ansible == 2.10.* ansible-base ~= 2.10.0 +# ansible == 3.* ansible-base ~= 2.10.0 +# ansible == 4.* ansible-core ~= 2.11.0 +# ansible == 5.* ansible-core ~= 2.12.0 +# ansible == 6.* ansible-core ~= 2.13.0 +# ansible == 7.x ansible-core ~= 2.14.0 +# ansible == 8.x ansible-core ~= 2.15.0 +# ansible == 9.x ansible-core ~= 2.16.0 +# ansible == 10.x ansible-core ~= 2.17.0 +# ansible == 11.x ansible-core ~= 2.18.0 + +# See also +# - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix [tox] envlist = init, - py{27,35,39}-mode_ansible-distros_centos, - py{27,35,39}-mode_ansible-distros_debian, - py{27,35,39}-mode_ansible-distros_ubuntu, - py{27,35,39}-mode_mitogen-distro_centos{6,7,8}, - py{27,35,39}-mode_mitogen-distro_debian{9,10,11}, - py{27,35,39}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{27,36}-mode_ansible-ansible{2.10,3,4}, + py{311}-mode_ansible-ansible{2.10,3,4,5}, + py{313}-mode_ansible-ansible{6,7,8,9,10,11}, + py{27,36,313}-mode_mitogen, report, -requires = - tox-factor [testenv] basepython = @@ -45,56 +69,93 @@ basepython = py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 + py311: python3.11 + py312: python3.12 + py313: python3.13 +deps = + -r{toxinidir}/tests/requirements.txt + mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt + ansible2.10: ansible~=2.10.0 + ansible3: ansible~=3.0 + ansible4: ansible~=4.0 + ansible5: ansible~=5.0 + ansible6: ansible~=6.0 + ansible7: ansible~=7.0 + ansible8: ansible~=8.0 + ansible9: ansible~=9.0 + ansible10: ansible~=10.0 + ansible11: ansible>=11.0 install_command = - python -m pip --no-python-version-warning install {opts} {packages} + python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = - mode_ansible: {toxinidir}/.ci/ansible_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py - mode_mitogen: {toxinidir}/.ci/mitogen_install.py commands = mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py + mode_localhost: {toxinidir}/.ci/localhost_ansible_tests.py mode_mitogen: {toxinidir}/.ci/mitogen_tests.py passenv = ANSIBLE_* HOME + MITOGEN_* setenv = - ANSIBLE_SKIP_TAGS = requires_local_sudo + # See also azure-pipelines.yml + ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 - VER=2.10.5 - ansible2.3: VER=2.3.3.0 - ansible2.4: VER=2.4.6.0 - ansible2.8: VER=2.8.3 - ansible2.9: VER=2.9.6 - ansible2.10: VER=2.10.0 - distro_centos5: DISTRO=centos5 - distro_centos6: DISTRO=centos6 - distro_centos7: DISTRO=centos7 - distro_centos8: DISTRO=centos8 - distro_debian9: DISTRO=debian9 - distro_debian10: DISTRO=debian10 - distro_debian11: DISTRO=debian11 - distro_ubuntu1604: DISTRO=ubuntu1604 - distro_ubuntu1804: DISTRO=ubuntu1804 - distro_ubuntu2004: DISTRO=ubuntu2004 - distros_centos: DISTROS=centos6 centos7 centos8 - distros_centos5: DISTROS=centos5 - distros_centos6: DISTROS=centos6 - distros_centos7: DISTROS=centos7 - distros_centos8: DISTROS=centos8 - distros_debian: DISTROS=debian9 debian10 debian11 - distros_debian9: DISTROS=debian9 - distros_debian10: DISTROS=debian10 - distros_debian11: DISTROS=debian11 - distros_ubuntu: DISTROS=ubuntu1604 ubuntu1804 ubuntu2004 - distros_ubuntu1604: DISTROS=ubuntu1604 - distros_ubuntu1804: DISTROS=ubuntu1804 - distros_ubuntu2004: DISTROS=ubuntu2004 + PIP_CONSTRAINT={toxinidir}/tests/constraints.txt + # Ansible 6 - 8 (ansible-core 2.13 - 2.15) require Python 2.7 or >= 3.5 on targets + ansible6: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible7: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible8: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets + ansible9: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets + ansible10: MITOGEN_TEST_DISTRO_SPECS=debian10-py3 debian11-py3 ubuntu2004-py3 + # Ansible 11 (ansible-core 2.18) requires Python >= 3.8 on targets + ansible11: MITOGEN_TEST_DISTRO_SPECS=debian11-py3 ubuntu2004-py3 + distros_centos: MITOGEN_TEST_DISTRO_SPECS=centos6 centos7 centos8 + distros_centos5: MITOGEN_TEST_DISTRO_SPECS=centos5 + distros_centos6: MITOGEN_TEST_DISTRO_SPECS=centos6 + distros_centos7: MITOGEN_TEST_DISTRO_SPECS=centos7 + distros_centos8: MITOGEN_TEST_DISTRO_SPECS=centos8 + distros_debian: MITOGEN_TEST_DISTRO_SPECS=debian9 debian10 debian11 + distros_debian9: MITOGEN_TEST_DISTRO_SPECS=debian9 + distros_debian10: MITOGEN_TEST_DISTRO_SPECS=debian10 + distros_debian11: MITOGEN_TEST_DISTRO_SPECS=debian11 + distros_ubuntu: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 ubuntu1804 ubuntu2004 + distros_ubuntu1604: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 + distros_ubuntu1804: MITOGEN_TEST_DISTRO_SPECS=ubuntu1804 + distros_ubuntu2004: MITOGEN_TEST_DISTRO_SPECS=ubuntu2004 mode_ansible: MODE=ansible + mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive + mode_ansible: ANSIBLE_CALLBACK_WHITELIST=profile_tasks + mode_ansible: ANSIBLE_CALLBACKS_ENABLED=profile_tasks mode_debops_common: MODE=debops_common + mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive mode_mitogen: MODE=mitogen - strategy_linear: STRATEGY=linear + strategy_linear: ANSIBLE_STRATEGY=linear +allowlist_externals = + # Added: Tox 3.18: Tox 4.0+ + *_install.py + *_tests.py + aws + docker + docker-credential-secretservice + echo + gpg2 + pass +whitelist_externals = + # Deprecated: Tox 3.18+; Removed: Tox 4.0 + *_install.py + *_tests.py + aws + docker + docker-credential-secretservice + echo + gpg2 + pass [testenv:init] basepython = python3 @@ -114,7 +175,9 @@ whitelist_externals = echo [testenv:docs] -basepython = python3 +basepython = python3.8 changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +deps = + -r docs/requirements.txt