diff --git a/.gitignore b/.gitignore index affdf25f..04f741da 100644 --- a/.gitignore +++ b/.gitignore @@ -12,11 +12,9 @@ /target/ /tmp/ -/.vagrant/ - -/target/ - /build-linux/data/ /build-linux/gnupg/ +/build-linux/gpg-deb/ +/build-linux/gpg-rpm/ /build-linux/target/ diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 907c4c4e..00000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -python 2.7.17 diff --git a/Dockerfile-centos-7 b/Dockerfile-centos-7 index 87148832..96671add 100644 --- a/Dockerfile-centos-7 +++ b/Dockerfile-centos-7 @@ -2,10 +2,10 @@ FROM centos/systemd ARG FPM_VERSION=1.11.0 ARG PYTHON_VERSION=3 +ARG DOCKER_WORKDIR=/usr/share/pdagent ENV PYTHON_VERSION ${PYTHON_VERSION} - ENV container docker -ENV DOCKER_DEV_ENV true +ENV DOCKER_WORKDIR ${DOCKER_WORKDIR} RUN yum install -y -q centos-release-scl RUN yum install -y -q createrepo @@ -26,12 +26,7 @@ RUN source /opt/rh/rh-ruby23/enable && \ /opt/rh/rh-ruby23/root/usr/bin/gem install -q --no-ri --no-rdoc -v $FPM_VERSION fpm RUN yum install -y python${PYTHON_VERSION} -WORKDIR /pd-agent-install -COPY . /pd-agent-install -RUN mkdir -p /pd-agent-install/build-linux/release -RUN cd /pd-agent-install/build-linux && ./make_rpm.sh /pd-agent-install/build-linux/gnupg /pd-agent-install/build-linux/release -RUN echo "Installing rpm package" - -RUN ./scripts/build.sh centos +COPY . $DOCKER_WORKDIR +WORKDIR $DOCKER_WORKDIR CMD ["/usr/sbin/init"] diff --git a/Dockerfile-ubuntu b/Dockerfile-ubuntu index 3ae37868..7713dead 100644 --- a/Dockerfile-ubuntu +++ b/Dockerfile-ubuntu @@ -3,10 +3,11 @@ FROM ubuntu:${UBUNTU_VERSION} ARG FPM_VERSION=1.11.0 ARG PYTHON_VERSION=3 +ARG DOCKER_WORKDIR=/usr/share/pdagent ENV PYTHON_VERSION ${PYTHON_VERSION} ENV container docker ENV DEBIAN_FRONTEND noninteractive -ENV DOCKER_DEV_ENV true +ENV DOCKER_WORKDIR ${DOCKER_WORKDIR} RUN apt-get update -y -qq RUN apt-get install -y -q apt-utils @@ -41,13 +42,7 @@ RUN rm -f /lib/systemd/system/multi-user.target.wants/* \ VOLUME [ "/sys/fs/cgroup" ] -WORKDIR /pd-agent-install -COPY . /pd-agent-install -RUN mkdir -p /pd-agent-install/build-linux/release -RUN cd /pd-agent-install/build-linux && ./make_deb.sh /pd-agent-install/build-linux/gnupg /pd-agent-install/build-linux/release -RUN echo "Installing debian package" - -# Installing the debian package will also enable pdagent in systemd -RUN ./scripts/build.sh ubuntu +COPY . $DOCKER_WORKDIR +WORKDIR $DOCKER_WORKDIR CMD ["/lib/systemd/systemd"] diff --git a/Makefile b/Makefile index d67fefac..35c8f2d1 100644 --- a/Makefile +++ b/Makefile @@ -1,52 +1,74 @@ include build-linux/make_common.env export $(shell sed 's/=.*//' build-linux/make_common.env) -main: build - +.PHONY: build build: ubuntu centos +.PHONY: ubuntu ubuntu: target/deb target/tmp/GPG-KEY-pagerduty -centos: target/rpm target/tmp/GPG-KEY-pagerduty +.PHONY: centos +centos: target/rpm target/tmp/GPG-KEY-RPM-pagerduty +.PHONY: build-ubuntu build-ubuntu: docker build . \ -t pdagent-ubuntu \ -f Dockerfile-ubuntu \ --build-arg FPM_VERSION="${FPM_VERSION}" \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ - --build-arg UBUNTU_VERSION="${UBUNTU_VERSION}" + --build-arg UBUNTU_VERSION="${UBUNTU_VERSION}" \ + --build-arg DOCKER_WORKDIR="${DOCKER_WORKDIR}" +.PHONY: build-centos build-centos: docker build . \ -t pdagent-centos \ -f Dockerfile-centos-7 \ --build-arg FPM_VERSION="${FPM_VERSION}" \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ - --build-arg CENTOS_VERSION="${CENTOS_VERSION}" + --build-arg CENTOS_VERSION="${CENTOS_VERSION}" \ + --build-arg DOCKER_WORKDIR="${DOCKER_WORKDIR}" target/deb: build-ubuntu docker run \ - -v `pwd`:/usr/share/pdagent \ + -v `pwd`:${DOCKER_WORKDIR} \ -it pdagent-ubuntu \ - /bin/sh -c "/bin/sh build-linux/make_deb.sh /usr/share/pdagent/build-linux/gnupg /usr/share/pdagent/target" + /bin/sh -c "/bin/sh build-linux/make_deb.sh ${DOCKER_WORKDIR}/build-linux/gpg-deb ${DOCKER_WORKDIR}/target" target/rpm: build-centos docker run \ - -v `pwd`:/usr/share/pdagent \ + -v `pwd`:${DOCKER_WORKDIR} \ -it pdagent-centos \ - /bin/sh -c "/bin/sh build-linux/make_rpm.sh /usr/share/pdagent/build-linux/gnupg /usr/share/pdagent/target" + /bin/sh -c "/bin/sh build-linux/make_rpm.sh ${DOCKER_WORKDIR}/build-linux/gpg-rpm ${DOCKER_WORKDIR}/target" target/tmp/GPG-KEY-pagerduty: docker run \ - -v `pwd`:/usr/share/pdagent \ + -v `pwd`:${DOCKER_WORKDIR} \ + -it pdagent-ubuntu \ + /bin/sh -c "mkdir -p ${DOCKER_WORKDIR}/target/tmp; gpg --armor --export --homedir ${DOCKER_WORKDIR}/build-linux/gpg-deb > ${DOCKER_WORKDIR}/target/tmp/GPG-KEY-pagerduty" + +target/tmp/GPG-KEY-RPM-pagerduty: + docker run \ + -v `pwd`:${DOCKER_WORKDIR} \ -it pdagent-ubuntu \ - /bin/sh -c "mkdir -p /usr/share/pdagent/target/tmp; gpg --armor --export --homedir /usr/share/pdagent/build-linux/gnupg > /usr/share/pdagent/target/tmp/GPG-KEY-pagerduty" + /bin/sh -c "mkdir -p ${DOCKER_WORKDIR}/target/tmp; gpg --armor --export --homedir ${DOCKER_WORKDIR}/build-linux/gpg-rpm > ${DOCKER_WORKDIR}/target/tmp/GPG-KEY-RPM-pagerduty" .PHONY: test test: find unit_tests -name "test_*.py" | xargs python run-tests.py +.PHONY: test-integration +test-integration: test-integration-ubuntu test-integration-centos + +.PHONY: test-integration-ubuntu +test-integration-ubuntu: ubuntu + scripts/full-integration-test.sh ubuntu + +.PHONY: test-integration-centos +test-integration-centos: centos + scripts/full-integration-test.sh centos + .PHONY: clean clean: rm -rf dist diff --git a/README.md b/README.md index 68fbcc70..c1fe3fae 100644 --- a/README.md +++ b/README.md @@ -63,15 +63,15 @@ For IDE setup instructions see [PyDev Setup](pydev-setup.md) or [IDEA Setup](ide ### Build Tools -To perform a complete automated build, you'll need to install Vagrant and Scons. See [Vagrant Setup](vagrant-setup.md) and [Scons Setup](scons-setup.md) for instructions. +To perform a complete automated build, you'll need to install Docker and `make`. ### Running Unit Tests You can run the unit tests with the following command: -`scons test-local` +`make test` -To run them without installing SCons, use the `run-tests.py` test runner, e.g.: +To run them without using `make`, use the `run-tests.py` test runner, e.g.: `python run-tests.py unit_tests/test_*.py unit_tests/thirdparty/test_*.py` @@ -98,41 +98,12 @@ make ubuntu make centos ``` -3. Run integration tests on the packages as follows: - * Edit the file `pdagenttestinteg/util.sh` and change the line `SVC_KEY=CHANGEME` to a real PagerDuty Service API Key in your pdt test account. - * Run the command: -``` -scons test-integration -``` - -This will run the integration tests on the various VMs using the packagesbuilt in the previous step. Note that the tests will trigger and resolve some incidents when they run. - If you want to build packages by hand, follow the instructions in the [Build Linux Howto](build-linux/howto.md). Similarly, you can check the SCons targets using `scons -h` for instructions on performing specific builds tasks and on specific VMs. -#### Some helpful scripts - -``` -scripts/rev_pkgs.sh -``` - -This will remove the installed pdagent packages from the vagrant build machines, `agent-minimal-centos65` and `agent-minimal-ubuntu1204`, and run `scons local-repo gpg-home=build-linux/gnupg` to install them again. Run this anytime you revise a package artifact like `build-linux/deb/postinst`. - -``` -scripts/kill_pids.sh -``` - -This will kill stray pdagent processes and cleanup the pidfile on all vagrant machines. Run this -if your changes are causing integration tests to fail due to improper process managment via `service` or `systemctl`. - -``` -scripts/setup_upgrade_test.sh -``` - -This will vagrant destroy, up and install the latest public repo pdagent package on machines for upgrade testing via `scons test-integration`. #License Copyright (c) 2013-2014, PagerDuty, Inc. diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 28c87858..00000000 --- a/Vagrantfile +++ /dev/null @@ -1,50 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -_bento_centos65 = { - "box" => "bento_centos65", - "box_url" => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_centos-6.5_chef-provisionerless.box", -} -_bento_centos72 = { - "box" => "bento_centos72", - "box_url" => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_centos-7.2_chef-provisionerless.box", -} -_bento_ubuntu1204 = { - "box" => "bento_ubuntu1204", - "box_url" => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_ubuntu-12.04_chef-provisionerless.box", -} -_bento_ubuntu1404 = { - "box" => "bento_ubuntu1404", - "box_url" => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_ubuntu-14.04_chef-provisionerless.box", -} -_bento_ubuntu1604 = { - "box" => "bento_ubuntu1604", - "box_url" => "http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_ubuntu-16.04_chef-provisionerless.box", -} - -vms = { - "agent-minimal-centos65" => _bento_centos65, - "agent-minimal-centos72" => _bento_centos72, - "agent-minimal-ubuntu1204" => _bento_ubuntu1204, - "agent-minimal-ubuntu1404" => _bento_ubuntu1404, - "agent-minimal-ubuntu1604" => _bento_ubuntu1604 -} - -Vagrant.configure("2") do |conf_outer| - - vms.each do |name, conf| - - conf_outer.vm.define name do |config| - - config.vm.box = conf["box"] - config.vm.box_url = conf["box_url"] - - ## no longer necessary with Vagrant 2.1.2? - # Public/bridged network so VM can install packages from the internet - # config.vm.network :public_network, :bridge => "en0: Wi-Fi (AirPort)" - - end # conf_outer.vm.define - - end # vms.each - -end # Vagrant.configure diff --git a/build-linux/deb/postinst b/build-linux/deb/postinst index 4f340e78..3546cb65 100755 --- a/build-linux/deb/postinst +++ b/build-linux/deb/postinst @@ -40,11 +40,7 @@ install_init () { install_systemd () { cp /var/lib/pdagent/scripts/pdagent.service /lib/systemd/system systemctl enable pdagent - if [ -z $DOCKER_DEV_ENV ]; then - # Prevent starting pdagent in Docker environment because systemd - # will do that after running the Docker systemd process - systemctl start pdagent - fi + systemctl start pdagent } if [ "$1" = "configure" ]; then diff --git a/build-linux/howto.md b/build-linux/howto.md index 10e9ee09..cfae23fd 100644 --- a/build-linux/howto.md +++ b/build-linux/howto.md @@ -1,8 +1,8 @@ # Linux Build Instructions -These instructions assume you're running this on a Mac with Vagrant installed, -and that the project directory ends up mounted in the VM at `/vagrant`. (this -should happen automatically) +These instructions assume you're running this on a Mac with Docker installed, +and that the project directory ends up mounted in the VM at +`/usr/share/pdagent`. (this should happen automatically) ## One-time setup of Development GPG keys: @@ -26,17 +26,15 @@ the following instructions accordingly. Building the .deb: ``` -scons local-repo \ - gpg-home=build-linux/gnupg \ - virt=agent-minimal-ubuntu1204 +make ubuntu ``` Install & test the .deb: ``` -vagrant ssh agent-minimal-ubuntu1204 +docker run -it pdagent-ubuntu /bin/bash -sudo apt-key add /vagrant/target/tmp/GPG-KEY-pagerduty -sudo sh -c 'echo "deb file:///vagrant/target deb/" \ +sudo apt-key add /usr/share/pdagent/target/tmp/GPG-KEY-pagerduty +sudo sh -c 'echo "deb file:///usr/share/pdagent/target deb/" \ >/etc/apt/sources.list.d/pdagent.list' sudo apt-get update sudo apt-get install pdagent @@ -60,22 +58,20 @@ Rerun the test commands to ensure files are gone Building the .rpm: ``` -scons local-repo \ - gpg-home=/gpg/path/used/earlier \ - virt=agent-minimal-centos65 +make centos ``` Install & test the .rpm: ``` -vagrant ssh agent-minimal-centos65 +docker run -it pdagent-centos /bin/bash sudo sh -c 'cat >/etc/yum.repos.d/pdagent.repo < - -OS=$1 -TEST_FILE=$2 -make build-${OS} -if [ $OS == "ubuntu" ] -then - docker run -d --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro -it pdagent-ubuntu -elif [ $OS == 'centos' ] -then - docker run -d -p 5000:80 --privileged=true --tmpfs /tmp --tmpfs /run -v /sys/fs/cgroup:/sys/fs/cgroup:ro -it pdagent-centos -fi -docker exec -it $(docker ps -q -f ancestor=pdagent-${OS}) /bin/bash ./pdagenttestinteg/${TEST_FILE} diff --git a/scripts/setup_upgrade_test.sh b/scripts/setup_upgrade_test.sh deleted file mode 100755 index 5ae96b57..00000000 --- a/scripts/setup_upgrade_test.sh +++ /dev/null @@ -1,5 +0,0 @@ -scripts/setup_upgrade_test_centos65.sh -scripts/setup_upgrade_test_centos72.sh -scripts/setup_upgrade_test_ubuntu1204.sh -scripts/setup_upgrade_test_ubuntu1404.sh -scripts/setup_upgrade_test_ubuntu1604.sh diff --git a/scripts/setup_upgrade_test_centos65.sh b/scripts/setup_upgrade_test_centos65.sh deleted file mode 100755 index bc156c39..00000000 --- a/scripts/setup_upgrade_test_centos65.sh +++ /dev/null @@ -1,3 +0,0 @@ -vagrant destroy -f agent-minimal-centos65 -vagrant up agent-minimal-centos65 -vagrant ssh agent-minimal-centos65 -c 'bash /vagrant/scripts/install_pub_centos_pkg.sh' diff --git a/scripts/setup_upgrade_test_centos72.sh b/scripts/setup_upgrade_test_centos72.sh deleted file mode 100755 index 33f9b340..00000000 --- a/scripts/setup_upgrade_test_centos72.sh +++ /dev/null @@ -1,3 +0,0 @@ -vagrant destroy -f agent-minimal-centos72 -vagrant up agent-minimal-centos72 -vagrant ssh agent-minimal-centos72 -c 'bash /vagrant/scripts/install_pub_centos_pkg.sh' diff --git a/scripts/setup_upgrade_test_ubuntu1204.sh b/scripts/setup_upgrade_test_ubuntu1204.sh deleted file mode 100755 index cc54246c..00000000 --- a/scripts/setup_upgrade_test_ubuntu1204.sh +++ /dev/null @@ -1,3 +0,0 @@ -vagrant destroy -f agent-minimal-ubuntu1204 -vagrant up agent-minimal-ubuntu1204 -vagrant ssh agent-minimal-ubuntu1204 -c 'bash /vagrant/scripts/install_pub_ubuntu_pkg.sh' diff --git a/scripts/setup_upgrade_test_ubuntu1404.sh b/scripts/setup_upgrade_test_ubuntu1404.sh deleted file mode 100755 index fdbf0f9a..00000000 --- a/scripts/setup_upgrade_test_ubuntu1404.sh +++ /dev/null @@ -1,3 +0,0 @@ -vagrant destroy -f agent-minimal-ubuntu1404 -vagrant up agent-minimal-ubuntu1404 -vagrant ssh agent-minimal-ubuntu1404 -c 'bash /vagrant/scripts/install_pub_ubuntu_pkg.sh' diff --git a/scripts/setup_upgrade_test_ubuntu1604.sh b/scripts/setup_upgrade_test_ubuntu1604.sh deleted file mode 100755 index 2f026900..00000000 --- a/scripts/setup_upgrade_test_ubuntu1604.sh +++ /dev/null @@ -1,3 +0,0 @@ -vagrant destroy -f agent-minimal-ubuntu1604 -vagrant up agent-minimal-ubuntu1604 -vagrant ssh agent-minimal-ubuntu1604 -c 'bash /vagrant/scripts/install_pub_ubuntu_pkg.sh' diff --git a/vagrant-setup.md b/vagrant-setup.md deleted file mode 100644 index 6b6e9c7a..00000000 --- a/vagrant-setup.md +++ /dev/null @@ -1,67 +0,0 @@ -# Vagrant Setup - -Vagrant setup notes for OS X 10.8 (Mountain Lion). - -## Installation/Upgrade - -1. Install Vagrant. See instructions at http://www.vagrantup.com/ - -NOTE: If you experience issues in the 'Configuring and enabling network interfaces...' phase when bringing up centos instances, upgrade your vagrant to the latest. My upgrade from v1.9.0 to v2.1.2 fixed this. - -2. Install VirtualBox. See instructions at https://www.virtualbox.org/ - -## Managing VMs - -Run the command `vagrant status` in the project directory to list the VMs -defined in the project's `Vagrantfile`. - -For example: - -``` -vagrant status -Current machine states: - -agent-minimal-centos65 not created (virtualbox) -agent-minimal-centos72 not created (virtualbox) -agent-minimal-ubuntu1204 running (virtualbox) -agent-minimal-ubuntu1404 running (virtualbox) -agent-minimal-ubuntu1604 running (virtualbox) - -This environment represents multiple VMs. The VMs are all listed -above with their current state. For more information about a specific -VM, run `vagrant status NAME`. -``` - -You can bring up all VM at the same time by running `vagrant up` or bring up -an individual VM by specifying its name. For example: - -`vagrant up agent-minimal-ubuntu1204` - -**Note**: the first time that you bring up a VM may require a large download of the "box" that the VM is based on. The box downloads will be cached in the -directory `~/.vagrant.d/boxes`. - -You can manage the VMs using the Vagrant commands: `up`, `halt`, `reload`, -`suspend` and `resume`. - -## Accessing the VM - -You can ssh into a VM by using `vagrant ssh `. For example: - -`vagrant ssh agent-minimal-ubuntu1204` - -## Deleting VMs - -You can delete Vagrant VMs by using the `vagrant destroy` command: - -``` -vagrant destroy -Are you sure you want to destroy the 'agent-minimal-ubuntu1204' VM? [y/N] y -[agent-minimal-ubuntu1204] Forcing shutdown of VM... -[agent-minimal-ubuntu1204] Destroying VM and associated drives... -``` - -## Vagrant cache - -Vagrant caches VM downloads in `~/.vagrant.d/boxes`. These are reused when you -destroy and rebuild a VM in the same project or in another project. You may -want to check for and clean out old downloads here to recover disk space.