add testframework and version test as example

.github/workflows/build-ci.yml

@@ -4,8 +4,9 @@ on:
   pull_request:
     paths:
       - "**/Dockerfile"
-      - "**/docker-entrypoint.py"
+      - "**/entrypoint.py"
       - "**/PLATFORMS"
+      - "tests/"
       - "tools/genmatrix.js"
       - ".github/workflows/build-ci.yml"
 
@@ -38,6 +39,8 @@ jobs:
     if: ${{ fromJson(needs.gen-matrix.outputs.matrix) }}
     needs: gen-matrix
     name: build
+    env:
+      image_tag: local/ci:${{ github.run_id }}
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -62,8 +65,16 @@ jobs:
           builder: ${{ steps.buildx.outputs.name }}
           push: false
           load: true
+          tags: ${{ env.image_tag }}
           platforms: ${{ matrix.platform }}
           context: ./${{ matrix.version }}/${{ matrix.variant }}
           file: ./${{ matrix.version }}/${{ matrix.variant }}/Dockerfile
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+      - name: Run integration tests
+        run: |
+          python3 -m tests.integration_runner \
+                  --platform ${{ matrix.platform }} \
+                  --image ${{ env.image_tag }} \
+                  --version ${{ matrix.version }}

tests/integration/framework/docker_runner.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 The Dogecoin Core developers
+"""
+Test framework for end-to-end docker tests
+"""
+
+import subprocess
+import sys
+
+class DockerRunner:
+    """Run docker containers for testing"""
+
+    def __init__(self, platform, image, verbose):
+        """Sets platform and image for all tests ran with this instance"""
+        self.platform = platform
+        self.image = image
+        self.verbose = verbose
+
+    def construct_docker_command(self, envs, args):
+        """
+        Construct a docker command with env and args
+        """
+        command = ["docker", "run", "--platform", self.platform]
+
+        for env in envs:
+            command.append("-e")
+            command.append(env)
+
+        command.append(self.image)
+
+        for arg in args:
+            command.append(arg)
+
+        return command
+
+    def run_interactive_command(self, envs, args):
+        """
+        Run our target docker image with a list of
+        environment variables and a list of arguments
+        """
+        command = self.construct_docker_command(envs, args)
+
+        if self.verbose:
+            print(f"Running command: { ' '.join(command) }")
+
+        try:
+            output = subprocess.run(command, capture_output=True, check=True)
+        except subprocess.CalledProcessError as docker_err:
+            print(f"Error while running command: { ' '.join(command) }", file=sys.stderr)
+            print(docker_err, file=sys.stderr)
+            print(docker_err.stderr.decode("utf-8"), file=sys.stderr)
+            print(docker_err.stdout.decode("utf-8"), file=sys.stdout)
+
+            raise docker_err
+
+        return output

tests/integration/framework/test_runner.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 The Dogecoin Core developers
+"""
+Base class to define and run Dogecoin Core Docker tests with
+"""
+
+import argparse
+import sys
+
+from .docker_runner import DockerRunner
+
+class TestConfigurationError(Exception):
+    """Raised when the test is configured inconsistently"""
+
+class TestRunner:
+    """Base class to define and run Dogecoin Core Docker tests with"""
+    def __init__(self):
+        """Make sure there is an options object"""
+        self.options = {}
+
+    def add_options(self, parser):
+        """Allow adding options in tests"""
+
+    def run_test(self):
+        """Actual test, must be implemented by the final class"""
+        raise NotImplementedError
+
+    def run_command(self, envs, args):
+        """Run a docker command with env and args"""
+        assert self.options.platform is not None
+        assert self.options.image is not None
+
+        runner = DockerRunner(self.options.platform,
+            self.options.image, self.options.verbose)
+
+        return runner.run_interactive_command(envs, args)
+
+    def main(self):
+        """main loop"""
+        parser = argparse.ArgumentParser()
+        parser.add_argument("--platform", dest="platform", required=True,
+            help="The platform to use for testing, eg: 'linux/amd64'")
+        parser.add_argument("--image", dest="image", required=True,
+            help="The image or tag to execute tests against, eg: 'verywowimage'")
+        parser.add_argument("--verbose", dest="verbose", default=False, action="store_true",
+            help="Verbosely output actions taken and print docker logs, regardless of outcome")
+
+        self.add_options(parser)
+        self.options = parser.parse_args()
+
+        self.run_test()
+        print("Tests successful")
+        sys.exit(0)

tests/integration/version.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 The Dogecoin Core developers
+"""
+Test the version installed to be the expected version
+"""
+
+import re
+
+from .framework.test_runner import TestRunner
+
+class VersionTest(TestRunner):
+    """Versions test"""
+
+    def __init__(self):
+        """Constructor"""
+        TestRunner.__init__(self)
+        self.version_expr = None
+
+    def add_options(self, parser):
+        """Add test-specific --version option"""
+        parser.add_argument("--version", dest="version", required=True,
+            help="The version that is expected to be installed, eg: '1.14.5'")
+
+    def run_test(self):
+        """Check the version of each executable"""
+
+        self.version_expr = re.compile(f".*{ self.options.version }.*")
+
+        # check dogecoind with only env
+        dogecoind = self.run_command(["VERSION=1"], [])
+        self.ensure_version_on_first_line(dogecoind.stdout)
+
+        # check dogecoin-cli
+        dogecoincli = self.run_command([], ["dogecoin-cli", "-?"])
+        self.ensure_version_on_first_line(dogecoincli.stdout)
+
+        # check dogecoin-tx
+        dogecointx = self.run_command([], ["dogecoin-tx", "-?"])
+        self.ensure_version_on_first_line(dogecointx.stdout)
+
+        # make sure that we find version errors
+        caught_error = False
+        try:
+            self.ensure_version_on_first_line("no version here".encode('utf-8'))
+        except AssertionError:
+            caught_error = True
+
+        if not caught_error:
+            raise AssertionError("Failed to catch a missing version")
+
+    def ensure_version_on_first_line(self, cmd_output):
+        """Assert that the version is contained in the first line of output string"""
+        first_line = cmd_output.decode("utf-8").split("\n")[0]
+
+        if re.match(self.version_expr, first_line) is None:
+            text = f"Could not find version { self.options.version } in { first_line }"
+            raise AssertionError(text)
+
+if __name__ == '__main__':
+    VersionTest().main()

tests/integration_runner.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 The Dogecoin Core developers
+"""
+Runs the integration tests
+"""
+
+import subprocess
+import sys
+
+from .integration.framework.test_runner import TestRunner
+
+def print_test_output(test_name, stdout, stderr=None):
+    """prints output from a test, including from stderr if provided"""
+    print("\n")
+    print(test_name)
+    print("----------------------")
+
+    if stderr is not None:
+        print(stderr.decode("utf-8"), file=sys.stderr)
+
+    print(stdout.decode("utf-8"))
+
+class IntegrationRunner(TestRunner):
+    """Runs the integration tests"""
+
+    def __init__(self):
+        """Initializes the failure tracker and test result map"""
+        TestRunner.__init__(self)
+        self.found_failure = False
+        self.result_map = {}
+
+
+    def add_options(self, parser):
+        """Add test-specific --version option"""
+        parser.add_argument("--version", dest="version", required=True,
+            help="The version that is expected to be installed, eg: '1.14.5'")
+
+    def run_test(self):
+        """Run all specified tests and inherit any failures"""
+
+        #List of tests to run
+        tests = [
+            [ "version", [ "--version", self.options.version ] ],
+        ]
+
+        for test in tests:
+            self.result_map[test[0]] = self.run_individual_test(test)
+
+        self.print_summary()
+
+        if self.found_failure:
+            sys.exit(1)
+
+    def run_individual_test(self, test):
+        """Run the actual test"""
+        command = [
+            "/usr/bin/env", "python3",
+            "-m", f"tests.integration.{ test[0] }",
+            "--platform", self.options.platform,
+            "--image", self.options.image,
+        ]
+
+        if len(test) > 1 and len(test[1]) > 0:
+            for arg in test[1]:
+                command.append(arg)
+
+        if self.options.verbose:
+            command.append("--verbose")
+
+        try:
+            output = subprocess.run(command, capture_output=True, check=True)
+        except subprocess.CalledProcessError as test_err:
+            self.found_failure = True
+            print_test_output(test[0], test_err.stdout, test_err.stderr)
+            return False
+
+        if self.options.verbose:
+            print_test_output(test[0], output.stdout)
+
+        return True
+
+    def print_summary(self):
+        """Print a summary to stdout"""
+        print("\n")
+        print(f"RESULTS: for { self.options.image } on { self.options.platform }")
+
+        successes = 0
+        failures = 0
+        for test, result in self.result_map.items():
+            if result:
+                successes += 1
+                result_str = "Success"
+            else:
+                failures += 1
+                result_str = "Failure"
+
+            print(f"{ test }: { result_str }")
+
+        sum_str = f"{ successes } successful tests and { failures } failures"
+        print(f"\nFinished test suite with { sum_str }")
+
+if __name__ == '__main__':
+    IntegrationRunner().main()

tools/genmatrix.js

@@ -6,7 +6,9 @@ const fs = require('fs');
 // we want to make decisions based on these changing
 const CI_FILES = [
   '.github/workflows/build-ci.yml',
-  'tools/genmatrix.js'
+  'tools/genmatrix.js',
+  'tests/integration',
+  'tests/integration_runner.py',
 ];
 
 // Regular expression for detecting version numbering of Dogecoin Core
@@ -128,6 +130,13 @@ class MatrixBuilder {
 
 }
 
+// Checks if any of the CI files have been changed
+const checkCIFilesChanged = (changedFiles) => {
+  return CI_FILES.some(file => changedFiles.some(changedFile => {
+    return changedFile.match(new RegExp(file));
+  }));
+}
+
 const generateMatrix = (baseDir, changedFiles) => {
 
   const builder = new MatrixBuilder(baseDir);
@@ -136,8 +145,9 @@ const generateMatrix = (baseDir, changedFiles) => {
   builder.readMatrix();
 
   // If the CI has changed, test all builds, otherwise, just build what's changed
-  const ciHasChanged = CI_FILES.some(file => changedFiles.indexOf(file) !== -1);
-  if (!ciHasChanged) {
+  if (checkCIFilesChanged(changedFiles)) {
+    console.log("CI files have changed, running all builds!");
+  } else {
     builder.filterIncludedVariants(changedFiles);
   }