diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f9c6b087..a968c4cc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -119,6 +119,17 @@ jobs: - name: Execute salt-api tests run: tests/salt-api/test.sh + - name: Execute gitfs tests + env: + GITFS_KEYS_DIR: tests/gitfs/data/keys/gitfs + run: | + mkdir -p "${GITFS_KEYS_DIR}" + echo "${{ secrets.TESTS_REPO_PRIVATE_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh + chmod 600 "${GITFS_KEYS_DIR}"/gitfs_ssh + echo "${{ secrets.TESTS_REPO_PUBLIC_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub + chmod 644 "${GITFS_KEYS_DIR}"/gitfs_ssh.pub + tests/gitfs/test.sh + - name: Cleanup run: | docker stop registry diff --git a/.gitignore b/.gitignore index d70f4093..8c922aa9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ +# Always included +!assets/** + # development /config/ /roots/ /keys/ /logs/ /3pfs/ -.vscode/ \ No newline at end of file +.vscode/ + +# tests +/tests/**/keys/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b01faa7c..b327d7d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ This file only reflects the changes that are made in this image. Please refer to the [Salt 3004 Release Notes](https://docs.saltstack.com/en/latest/topics/releases/3004.html) for the list of changes in SaltStack. +**3004_1** + +- Install `libssh2 1.10.0` from source +- Install `libgit2 1.3.0` from source +- Install `pygit2 1.7.0` from pip repositories +- Change Docker base image to `ubuntu:hirsute-20210917` +- Upgrade Python to version `3.9` + **3004** - Upgrade `salt-master` to `3004` *Silicon* diff --git a/Dockerfile b/Dockerfile index 335e6781..28c2b32f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ -FROM ubuntu:focal-20211006 +FROM ubuntu:hirsute-20210917 ARG BUILD_DATE ARG VCS_REF # https://github.com/saltstack/salt/releases ENV SALT_VERSION="3004" \ - PYTHON_VERSION="3.8" + PYTHON_VERSION="3.9" -ENV IMAGE_VERSION="${SALT_VERSION}" +ENV IMAGE_VERSION="${SALT_VERSION}_1" ENV SALT_DOCKER_DIR="/etc/docker-salt" \ SALT_ROOT_DIR="/etc/salt" \ @@ -32,7 +32,7 @@ WORKDIR ${SALT_BUILD_DIR} # hadolint ignore=DL3008 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - sudo ca-certificates apt-transport-https wget locales openssh-client \ + sudo ca-certificates openssl apt-transport-https wget locales openssh-client \ python${PYTHON_VERSION} python3-dev libpython3-dev \ python3-pip python3-setuptools python3-wheel \ supervisor logrotate git gettext-base tzdata \ diff --git a/README.md b/README.md index a516bed2..7ad8e162 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ This keys must be placed inside `/home/salt/data/keys` directory. You can create an ssh key for pygit2 with the following command: ```sh -ssh-keygen -f gitfs_pygit2 -C 'gitfs@example.com' +ssh-keygen -f gitfs_ssh -C 'gitfs@example.com' ``` Place it wherever you want inside the container and specify its path with the configuration parameters: `gitfs_pubkey` and `gitfs_privkey` in your `.conf` file. @@ -334,13 +334,19 @@ gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub **Important Note** +By default, this image has been tested with RSA 4096 ssh keys generated with `ssh-keygen`. + If you get the following error while using `gitfs` with `pygit2` ```plain _pygit2.GitError: Failed to authenticate SSH session: Unable to send userauth-publickey request ``` -look if your private key hash empty lines at the bottom of the file and suppress them for solving the error. +you may have to recreate your ssh key adding the parameter: `-m PEM`: + +```sh +ssh-keygen -m PEM -f gitfs_ssh -C 'gitfs@example.com' +``` ### 3rd Party Formulas @@ -507,6 +513,7 @@ Below you can find a list with the available options that can be used to customi | `SALT_API_SERVICE_ENABLED` | Enable `salt-api` service. Default: `false` | | `SALT_API_USER` | Set username for `salt-api` service. Default: `salt_api` | | `SALT_API_USER_PASS` | `SALT_API_USER` password. Required if `SALT_API_SERVICE_ENBALED` is `true` and `SALT_API_USER` is not empty. _Unset_ by default | +| `SALT_API_CERT_CN` | Common name in the request. Default: `localhost` | | `SALT_MASTER_SIGN_PUBKEY` | Sign the master auth-replies with a cryptographic signature of the master's public key. Possible values: 'True' or 'False'. Default: `False` | | `SALT_MASTER_USE_PUBKEY_SIGNATURE` | Instead of computing the signature for each auth-reply, use a pre-calculated signature. This option requires `SALT_MASTER_SIGN_PUBKEY` set to 'True'. Possible values: 'True' or 'False'. Default: `True` | | `SALT_MASTER_SIGN_KEY_NAME` | The customizable name of the signing-key-pair without suffix. Default: `master_sign` | @@ -579,7 +586,7 @@ Where `salt-service` is one of: `salt-master` os `salt-api` (if `SALT_API_SERVIC [saltproject_badge]: https://img.shields.io/badge/Salt-v3004-lightgrey.svg?logo=Saltstack [saltproject_release_notes]: https://docs.saltproject.io/en/latest/topics/releases/3004.html "Salt Project Release Notes" -[ubuntu_badge]: https://img.shields.io/badge/ubuntu-focal--20211006-E95420.svg?logo=Ubuntu +[ubuntu_badge]: https://img.shields.io/badge/ubuntu-hirsute--20210917-E95420.svg?logo=Ubuntu [ubuntu_hub_docker]: https://hub.docker.com/_/ubuntu/ "Ubuntu Image" [github_publish_badge]: https://img.shields.io/github/workflow/status/cdalvaro/docker-salt-master/Publish%20Docker%20image?label=build&logo=GitHub&logoColor=%23181717 [github_publish_workflow]: https://github.com/cdalvaro/docker-salt-master/actions?query=workflow%3A%22Publish+Docker+image%22 diff --git a/assets/build/functions.sh b/assets/build/functions.sh index 300ddea3..5d71e639 100755 --- a/assets/build/functions.sh +++ b/assets/build/functions.sh @@ -2,7 +2,45 @@ set -e -# Execute a command as SALT_USER +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_debug +# DESCRIPTION: Echo debug information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_debug() { + if [[ "${DEBUG}" == 'true' || "${ECHO_DEBUG}" == 'true' ]]; then + echo " * DEBUG: $*" + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_info +# DESCRIPTION: Echo information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_info() { + echo " * INFO: $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_warn +# DESCRIPTION: Echo warning information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_warn() { + echo " * WARN: $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_error +# DESCRIPTION: Echo errors to stderr. +#---------------------------------------------------------------------------------------------------------------------- +function log_error() +{ + (>&2 echo " * ERROR: $*") +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: exec_as_salt +# DESCRIPTION: Execute the pass command as the `SALT_USER` user. +#---------------------------------------------------------------------------------------------------------------------- function exec_as_salt() { if [[ $(whoami) == "${SALT_USER}" ]]; then @@ -11,3 +49,171 @@ function exec_as_salt() sudo -HEu "${SALT_USER}" "$@" fi } + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM 32-bits or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm32() +{ + uname -m | grep -qE 'armv7l' +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM 64-bits or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm64() +{ + uname -m | grep -qE 'arm64|aarch64' +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm() +{ + is_arm32 || is_arm64 +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: install_pkgs +# DESCRIPTION: Install packages using apt-get install. +#---------------------------------------------------------------------------------------------------------------------- +function install_pkgs() +{ + apt-get install --no-install-recommends --yes $@ +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: download +# DESCRIPTION: Download the content from the given URL and save it into the specified file. +#---------------------------------------------------------------------------------------------------------------------- +function download() +{ + local URL="$1" + local FILE_NAME="$2" + + local WGET_ARGS=(--quiet) + is_arm32 && WGET_ARGS+=(--no-check-certificate) + + log_info "Downloading ${FILE_NAME} from ${URL} ..." + wget ${WGET_ARGS[@]} -O "${FILE_NAME}" "${URL}" + if [[ -f "${FILE_NAME}" ]]; then + log_debug "Success!" + else + log_error "Failed to download ${URL}" + exit 1 + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: check_sha256 +# DESCRIPTION: Compute the SHA256 hash for the given file and check if it matches the expected one. +#---------------------------------------------------------------------------------------------------------------------- +function check_sha256() +{ + local FILE="${1}" + local SHA256="${2}" + + log_info "Checking ${FILE} SHA256 hash ..." + if echo "${SHA256} ${FILE}" | shasum -a 256 -c --status -; then + log_debug "SHA256 hash for ${FILE} matches! (${SHA256})" + else + local HASH=$(shasum -a 256 "${FILE}" | awk '{print $1}') + log_error "SHA256 checksum mismatch for ${FILE}" + log_error "Expected: ${SHA256}" + log_error " Got: ${HASH}" + exit 1 + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: extract +# DESCRIPTION: Extract the given .tar.gz into the current directory. +#---------------------------------------------------------------------------------------------------------------------- +function extract() +{ + local FILE="${1}" + log_info "Unpacking file: ${FILE}" + tar xzf "${FILE}" --strip-components 1 +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: build_and_install +# DESCRIPTION: Build and install the given package from the current directory using cmake. +#---------------------------------------------------------------------------------------------------------------------- +function build_and_install() +{ + local PACKAGE_NAME="${1}"; shift + local CMAKE_ARGS=( + -Wno-dev + -DCMAKE_BUILD_TYPE=Release + ) + + # shellcheck disable=SC2206 + CMAKE_ARGS+=( $@ ) + + log_info "Building and installing ${PACKAGE_NAME} ..." + log_debug "CMAKE_ARGS: ${CMAKE_ARGS[@]}" + cmake ${CMAKE_ARGS[@]} . + cmake --build . --target install --config Release +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: install_libssh2 +# DESCRIPTION: Install libssh2 library. +#---------------------------------------------------------------------------------------------------------------------- +function install_libssh2() +{ + local LIBSSH2_VERSION=1.10.0 + local LIBSSH2_URL="https://github.com/libssh2/libssh2/releases/download/libssh2-${LIBSSH2_VERSION}/libssh2-${LIBSSH2_VERSION}.tar.gz" + local FILE_NAME="libssh2-${LIBSSH2_VERSION}.tar.gz" + local SHA256_SUM='2d64e90f3ded394b91d3a2e774ca203a4179f69aebee03003e5a6fa621e41d51' + + local CURRENT_DIR="$(pwd)" + local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}" + + download "${LIBSSH2_URL}" "${FILE_NAME}" + check_sha256 "${FILE_NAME}" "${SHA256_SUM}" + extract "${FILE_NAME}" + + _OPTS=( + -DLINT=OFF + -DBUILD_SHARED_LIBS=ON + -DCRYPTO_BACKEND=OpenSSL + -DENABLE_ZLIB_COMPRESSION=ON + -DENABLE_DEBUG_LOGGING=OFF + -DCLEAR_MEMORY=ON + ) + + build_and_install "libssh2 v${LIBSSH2_VERSION}" ${_OPTS[@]} + + cd "${CURRENT_DIR}" + rm -rf "${WORK_DIR}" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: install_libgit2 +# DESCRIPTION: Install libgit2 library. +#---------------------------------------------------------------------------------------------------------------------- +function install_libgit2() +{ + local LIBGIT2_VERSION=1.3.0 + local LIBGIT2_URL="https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz" + local FILE_NAME="libgit2-${LIBGIT2_VERSION}.tar.gz" + local SHA256_SUM='192eeff84596ff09efb6b01835a066f2df7cd7985e0991c79595688e6b36444e' + + local CURRENT_DIR="$(pwd)" + local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}" + + download "${LIBGIT2_URL}" "${FILE_NAME}" + check_sha256 "${FILE_NAME}" "${SHA256_SUM}" + extract "${FILE_NAME}" + + build_and_install "libgit2 v${LIBGIT2_VERSION}" -DBUILD_CLAR=OFF + + cd "${CURRENT_DIR}" + rm -rf "${WORK_DIR}" +} diff --git a/assets/build/install.sh b/assets/build/install.sh index 3ca59940..b68385e8 100755 --- a/assets/build/install.sh +++ b/assets/build/install.sh @@ -2,24 +2,27 @@ set -e -# shellcheck disable=SC1091 -source "${SALT_BUILD_DIR}/functions.sh" +export DEBIAN_FRONTEND=noninteractive -echo "Installing build dependencies ..." -BUILD_DEPENDENCIES=(make gcc g++ cmake pkg-config) +# shellcheck source=assets/build/functions.sh +FUNCTIONS_FILE="${SALT_BUILD_DIR}/functions.sh" +source "${FUNCTIONS_FILE}" -apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y "${BUILD_DEPENDENCIES[@]}" +log_info "Installing required packages and build dependencies ..." +REQUIRED_PACKAGES=( + libssl1.1 zlib1g libffi7 libpcre3 libgssapi3-heimdal +) + +BUILD_DEPENDENCIES=( + make gcc g++ cmake pkg-config libssl-dev zlib1g-dev libffi-dev + libpcre3-dev heimdal-dev +) -# Install arm build dependencies -if [[ "$(uname -i)" =~ ^(arm|aarch64) ]]; then - echo "Installing arm specific dependencies ..." - DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - libzmq3-dev libhttp-parser-dev libssl-dev libcurl4-openssl-dev -fi +apt-get update +install_pkgs "${REQUIRED_PACKAGES[@]}" "${BUILD_DEPENDENCIES[@]}" # Create salt user -echo "Creating ${SALT_USER} user ..." +log_info "Creating ${SALT_USER} user ..." useradd -d "${SALT_HOME}" -ms /bin/bash -U -G root,sudo,shadow "${SALT_USER}" # Set PATH @@ -28,14 +31,28 @@ PATH=/usr/local/sbin:/usr/local/bin:\$PATH EOF # Install python3 packages -echo "Installing python3 packages ..." -DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - python3-mako python3-pycryptodome python3-cherrypy3 python3-git python3-requests \ - python3-redis python3-gnupg python3-mysqldb python3-dateutil python3-libnacl python3-openssl \ - python3-pygit2 +log_info "Installing python3 packages ..." +install_pkgs --quiet \ + python3-mako python3-pycryptodome python3-cherrypy3 \ + python3-git python3-requests python3-redis python3-gnupg \ + python3-mysqldb python3-dateutil python3-libnacl python3-openssl pip3 install timelib==0.2.5 +# Install pygit2 package +install_libssh2 +install_libgit2 +pip3 install pygit2==1.7.0 + +# Downloading bootstrap-salt.sh script +BOOTSTRAP_VERSION='2021.09.17' +BOOTSTRAP_URL="https://raw.githubusercontent.com/saltstack/salt-bootstrap/v${BOOTSTRAP_VERSION}/bootstrap-salt.sh" +BOOTSTRAP_FILE='bootstrap-salt.sh' +BOOTSTRAP_SHA256='090d652cd6290debce0e3a4eded65086a4272e69446e711eb26f87160593b6a2' + +download "${BOOTSTRAP_URL}" "${BOOTSTRAP_FILE}" +check_sha256 "${BOOTSTRAP_FILE}" "${BOOTSTRAP_SHA256}" + # Bootstrap script options: # https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options ## -M: install Salt Master by default @@ -46,22 +63,17 @@ pip3 install timelib==0.2.5 ## -p: Extra-package to install ## -x: Changes the python version used to install a git version of salt SALT_BOOTSTRAP_OPTS=( -M -N -X -d -P -p salt-api -p salt-call -x "python${PYTHON_VERSION}" ) -_WGET_ARGS=() - -if [[ "$(uname -i)" == 'armv7l' ]]; then - ## -I: allow insecure connections while downloading any files - SALT_BOOTSTRAP_OPTS+=( -I ) - _WGET_ARGS+=( --no-check-certificate ) -fi - -echo "Installing saltstack ..." -echo "Option: ${SALT_BOOTSTRAP_OPTS[@]}" -wget ${_WGET_ARGS[@]} -O bootstrap-salt.sh https://bootstrap.saltstack.com -sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}" + +## -I: allow insecure connections while downloading any files +is_arm32 && SALT_BOOTSTRAP_OPTS+=( -I ) + +log_info "Installing saltstack ..." +log_debug "Options: ${SALT_BOOTSTRAP_OPTS[@]}" +sh "${BOOTSTRAP_FILE}" ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}" chown -R "${SALT_USER}": "${SALT_ROOT_DIR}" # Configure ssh -echo "Configuring ssh ..." +log_info "Configuring ssh ..." sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/ssh/ssh_config { echo " UserKnownHostsFile /dev/null" @@ -70,7 +82,7 @@ sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/ } >> /etc/ssh/ssh_config # Configure logrotate -echo "Configuring logrotate ..." +log_info "Configuring logrotate ..." # move supervisord.log file to ${SALT_LOGS_DIR}/supervisor/ sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;|" /etc/supervisor/supervisord.conf @@ -79,7 +91,7 @@ sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;| sed -i "s|^su root syslog$|su root root|" /etc/logrotate.conf # Configure supervisor -echo "Configuring supervisor ..." +log_info "Configuring supervisor ..." # configure supervisord to start salt-master cat > /etc/supervisor/conf.d/salt-master.conf <&2 echo "ERROR: $*") + (>&2 echo " * ERROR: $*") } -# Map salt user with host user +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: map_uidgid +# DESCRIPTION: Map salt user with host user. +#---------------------------------------------------------------------------------------------------------------------- function map_uidgid() { USERMAP_ORIG_UID=$(id -u "${SALT_USER}") @@ -45,9 +57,13 @@ function map_uidgid() fi } -# This function replaces placeholders with values -# $1: file with placeholders to replace -# $x: placeholders to replace +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: update_template +# DESCRIPTION: Replace placeholders with values. +# ARGUMENTS: +# - 1: Template file with placeholders to replace +# - @: Placeholder values +#---------------------------------------------------------------------------------------------------------------------- function update_template() { local FILE=${1?missing argument} @@ -74,7 +90,10 @@ function update_template() rm -f "${tmp_file}" } -# This function configures containers timezone +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_timezone +# DESCRIPTION: Configure containers timezone. +#---------------------------------------------------------------------------------------------------------------------- function configure_timezone() { echo "Configuring container timezone ..." @@ -94,7 +113,10 @@ function configure_timezone() fi } -# This function generates a master_sign key pair and its signature +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: gen_signed_keys +# DESCRIPTION: Generate a master_sign key pair and its signature. +#---------------------------------------------------------------------------------------------------------------------- function gen_signed_keys() { local key_name=${1:-master} @@ -108,7 +130,10 @@ function gen_signed_keys() echo -n "${GENERATED_KEYS_DIR}" } -# This function repairs keys permissions and creates keys if neaded +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: setup_salt_keys +# DESCRIPTION: Repair keys permissions and creates keys if neaded. +#---------------------------------------------------------------------------------------------------------------------- function setup_salt_keys() { echo "Setting up salt keys ..." @@ -135,7 +160,10 @@ function setup_salt_keys() find "${SALT_HOME}" -path "${SALT_KEYS_DIR}/*" -print0 | xargs -0 chown -h "${SALT_USER}": } -# This function configures ssh keys +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: setup_ssh_keys +# DESCRIPTION: Configure ssh keys. +#---------------------------------------------------------------------------------------------------------------------- function setup_ssh_keys() { echo "Configuring ssh ..." @@ -153,7 +181,10 @@ function setup_ssh_keys() fi } -# This function cofigures master service +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_salt_master +# DESCRIPTION: Configure master service. +#---------------------------------------------------------------------------------------------------------------------- function configure_salt_master() { echo "Configuring salt-master service ..." @@ -180,7 +211,10 @@ function configure_salt_master() SALT_MASTER_USE_PUBKEY_SIGNATURE } -# This function configures salt-api if service is set to be enabled +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: configure_salt_api +# DESCRIPTION: Configure salt-api if service is set to be enabled. +#---------------------------------------------------------------------------------------------------------------------- function configure_salt_api() { [[ ${SALT_API_SERVICE_ENABLED} == true ]] || return 0 @@ -208,10 +242,9 @@ function configure_salt_api() echo "Configuring salt-api service ..." CERTS_PATH=/etc/pki - SALT_API_KEY_FILE='docker-salt-master' - rm -rf "${CERTS_PATH}/tls/certs/*" - salt-call --local tls.create_self_signed_cert cacert_path="${CERTS_PATH}" CN="${SALT_API_KEY_FILE}" - chown "${SALT_USER}": "${CERTS_PATH}/tls/certs/${SALT_API_KEY_FILE}".{crt,key} + rm -rf "${CERTS_PATH}"/tls/certs/* + salt-call --local tls.create_self_signed_cert cacert_path="${CERTS_PATH}" CN="${SALT_API_CERT_CN}" + chown "${SALT_USER}": "${CERTS_PATH}/tls/certs/${SALT_API_CERT_CN}".{crt,key} cat >> "${SALT_ROOT_DIR}/master" < /etc/supervisor/conf.d/salt-api.conf < Removing ${CONTAINER_NAME} ..." - docker container rm --force "${CONTAINER_NAME}" -} +# https://stackoverflow.com/a/4774063/3398062 +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +# shellcheck source=assets/build/functions.sh +COMMON_FILE="${SCRIPT_PATH}/../lib/common.sh" +source "${COMMON_FILE}" trap cleanup EXIT # Run test instance echo "==> Starting docker-salt-master (${PLATFORM}) ..." -docker run --rm --detach --name "${CONTAINER_NAME}" \ - --publish 4505:4505 --publish 4506:4506 \ - --platform "${PLATFORM}" \ - "${IMAGE_NAME}" || ( echo "container started โŒ"; exit 1 ) -echo "container started โœ…" - -# Wait for salt-master bootup -echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." -sleep "${BOOTUP_WAIT_SECONDS}" +start_container_and_wait || error "container started" +ok "container started" # Check salt version echo "==> Checking salt version ..." -docker exec "${CONTAINER_NAME}" salt --versions -[[ "$(docker exec ${CONTAINER_NAME} salt --version)" == "salt $(cat VERSION)" ]] || ( echo "salt version โŒ"; exit 1 ) -echo "salt version โœ…" +docker-exec salt --versions +[[ "$(docker-exec salt --version)" == "salt $(cat VERSION)" ]] || error "salt version" +ok "salt version" # Test image calling healthcheck echo "==> Executing healthcheck ..." -docker exec "${CONTAINER_NAME}" /usr/local/sbin/healthcheck | grep -i 'true' || ( echo "healthcheck โŒ"; exit 1 ) -echo "healthcheck โœ…" +docker-exec /usr/local/sbin/healthcheck | grep -i 'true' || error "healthcheck" +ok "healthcheck" diff --git a/tests/gitfs/README.md b/tests/gitfs/README.md new file mode 100644 index 00000000..db35118d --- /dev/null +++ b/tests/gitfs/README.md @@ -0,0 +1,4 @@ +# GitFS Tests + +Checks: + - Clone a repository using a RSA 4096 ssh key generated with: `ssh-keygen -t rsa -b 4096 -C "salt@cdalvaro.test" -f gitfs_ssh` diff --git a/tests/gitfs/config/fileserver.conf b/tests/gitfs/config/fileserver.conf new file mode 100644 index 00000000..c848fb5b --- /dev/null +++ b/tests/gitfs/config/fileserver.conf @@ -0,0 +1,2 @@ +fileserver_backend: + - gitfs diff --git a/tests/gitfs/config/gitfs.conf b/tests/gitfs/config/gitfs.conf new file mode 100644 index 00000000..cbb8aad5 --- /dev/null +++ b/tests/gitfs/config/gitfs.conf @@ -0,0 +1,19 @@ +gitfs_provider: pygit2 +gitfs_base: main +gitfs_privkey: /home/salt/data/keys/gitfs/gitfs_ssh +gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub + +gitfs_remotes: + - git@github.com:cdalvaro/docker-salt-master-tests.git: + - root: tests/gitfs + +git_pillar_provider: pygit2 +git_pillar_base: main +git_pillar_branch: main + +ext_pillar: + - git: + - git@github.com:cdalvaro/docker-salt-master-tests.git: + - privkey: /home/salt/data/keys/gitfs/gitfs_ssh + - pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub + - root: tests/gitfs/pillar diff --git a/tests/gitfs/test.sh b/tests/gitfs/test.sh new file mode 100755 index 00000000..3600fc67 --- /dev/null +++ b/tests/gitfs/test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -e +[ "${DEBUG}" == true ] && set -vx + +echo "๐Ÿงช Running gitfs tests ..." + +# https://stackoverflow.com/a/4774063/3398062 +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck source=assets/build/functions.sh +COMMON_FILE="${SCRIPT_PATH}/../lib/common.sh" +source "${COMMON_FILE}" +trap cleanup EXIT + +export GITFS_KEYS_DIR=${GITFS_KEYS_DIR:-tests/gitfs/data/keys/gitfs} + +# Check gitfs keys are present +echo "==> Checking gitfs keys are present ..." +if [[ ! -f "${GITFS_KEYS_DIR}/gitfs_ssh" || ! -f "${GITFS_KEYS_DIR}/gitfs_ssh.pub" ]]; then + error "gitfs keys not found at ${GITFS_KEYS_DIR}" +fi +ok "gitfs keys" + +# Run test instance +echo "==> Starting docker-salt-master (${PLATFORM}) with RSA 4096 ssh key ..." +start_container_and_wait \ + --volume "$(pwd)/tests/gitfs/config":/home/salt/data/config:ro \ + --volume "$(pwd)/${GITFS_KEYS_DIR%%/gitfs}":/home/salt/data/keys \ +|| error "container started" +ok "container started" + +# Update repositories +echo "==> Updating gitfs repositories ..." +salt-run cache.clear_git_lock gitfs type=update +UPDATE_REPOS="$( salt-run fileserver.update )" +echo "${UPDATE_REPOS}" | grep -qi 'true' || error "update gitfs" +ok "update gitfs" + +# Check pillars +echo "==> Checking gitfs files ..." +FILE_LIST=$( salt-run fileserver.file_list ) +echo "${FILE_LIST}" +[[ "${FILE_LIST}" == *test.txt* ]] || error "gitfs files" +ok "gitfs files" diff --git a/tests/lib/common.sh b/tests/lib/common.sh new file mode 100644 index 00000000..0922eb3e --- /dev/null +++ b/tests/lib/common.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: IMAGE_NAME +# DESCRIPTION: The name and tag of the Docker image. Default: 'cdalvaro/docker-salt-master:latest'. +#---------------------------------------------------------------------------------------------------------------------- +export IMAGE_NAME=${IMAGE_NAME:-'cdalvaro/docker-salt-master:latest'} + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: CONTAINER_NAME +# DESCRIPTION: The name of the container. Default: 'salt-master'. +#---------------------------------------------------------------------------------------------------------------------- +export CONTAINER_NAME=salt-master + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: PLATFORM +# DESCRIPTION: The platform to run the tests on. Default: the current platform. +#---------------------------------------------------------------------------------------------------------------------- +export PLATFORM=${PLATFORM:-$(docker version --format='{{.Server.Os}}/{{.Server.Arch}}')} + +#--- ENV VARIABLE --------------------------------------------------------------------------------------------------- +# NAME: BOOTUP_WAIT_SECONDS +# DESCRIPTION: The number of seconds to wait for the container to boot up. Default: 60. +#---------------------------------------------------------------------------------------------------------------------- +export BOOTUP_WAIT_SECONDS=${BOOTUP_WAIT_SECONDS:-60} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: cleanup +# DESCRIPTION: Clean up tasks. +#---------------------------------------------------------------------------------------------------------------------- +function cleanup() +{ + echo "==> Removing ${CONTAINER_NAME} ..." + docker container rm --force "${CONTAINER_NAME}" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: docker-exec +# DESCRIPTION: Execute the given command inside the container. +#---------------------------------------------------------------------------------------------------------------------- +function docker-exec() +{ + docker exec "${CONTAINER_NAME}" "$@" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: salt-run +# DESCRIPTION: Execute the salt-run command inside the container. +#---------------------------------------------------------------------------------------------------------------------- +function salt-run() +{ + docker-exec salt-run "$@" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: salt-call +# DESCRIPTION: Execute the salt-call command inside the container. +#---------------------------------------------------------------------------------------------------------------------- +function salt-call() +{ + docker-exec salt-call "$@" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: master_log +# DESCRIPTION: Print salt-master log. +#---------------------------------------------------------------------------------------------------------------------- +function master_log() +{ + docker-exec cat data/logs/salt/master +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: wait_container +# DESCRIPTION: Wait for the container to boot up. +#---------------------------------------------------------------------------------------------------------------------- +function start_container_and_wait() +{ + # shellcheck disable=SC2206 + local DOCKER_ARGS=( $@ ) + + docker run --rm --detach --name "${CONTAINER_NAME}" \ + --publish 4505:4505 --publish 4506:4506 \ + --platform "${PLATFORM}" ${DOCKER_ARGS[@]} \ + "${IMAGE_NAME}" || return 1 + + echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." + sleep "${BOOTUP_WAIT_SECONDS}" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: ok +# DESCRIPTION: Print a successfull message. +#---------------------------------------------------------------------------------------------------------------------- +function ok() +{ + echo "โœ… $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: error +# DESCRIPTION: Print an error message, show the salt-master log and exit with code 1. +#---------------------------------------------------------------------------------------------------------------------- +function error() +{ + echo "๐Ÿ”ฅ $*" + master_log + return 1 +} diff --git a/tests/salt-api/test.sh b/tests/salt-api/test.sh index aef77313..5f661051 100755 --- a/tests/salt-api/test.sh +++ b/tests/salt-api/test.sh @@ -5,10 +5,13 @@ set -e echo "๐Ÿงช Running salt-api tests ..." -IMAGE_NAME=${IMAGE_NAME:-cdalvaro/docker-salt-master} -CONTAINER_NAME=salt_master -PLATFORM=${PLATFORM:-$(docker version --format='{{.Server.Os}}/{{.Server.Arch}}')} -BOOTUP_WAIT_SECONDS=${BOOTUP_WAIT_SECONDS:-60} +# https://stackoverflow.com/a/4774063/3398062 +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck source=assets/build/functions.sh +COMMON_FILE="${SCRIPT_PATH}/../lib/common.sh" +source "${COMMON_FILE}" +trap cleanup EXIT export SALTAPI_URL="https://localhost:8000/" export SALTAPI_USER=salt_api @@ -16,13 +19,6 @@ export SALTAPI_PASS=4wesome-Pass0rd export SALTAPI_EAUTH=pam export SALTAPI_TMP_DIR=${SALTAPI_TMP_DIR:-/tmp/salt-api} -function cleanup { - echo "==> Removing ${CONTAINER_NAME} ..." - docker container rm --force "${CONTAINER_NAME}" -} - -trap cleanup EXIT - # Create configuration files echo "==> Creating salt-api configuration file ..." mkdir -p "${SALTAPI_TMP_DIR}/config/" @@ -39,46 +35,41 @@ echo "salt-api config created โœ…" # Run test instance echo "==> Starting docker-salt-master (${PLATFORM}) with salt-api config ..." -docker run --rm --detach --name "${CONTAINER_NAME}" \ - --publish 4505:4505 --publish 4506:4506 --publish 8000:8000 \ +start_container_and_wait \ + --publish 8000:8000 \ --env SALT_API_SERVICE_ENABLED=true \ --env SALT_API_USER_PASS="${SALTAPI_PASS}" \ - --platform "${PLATFORM}" \ --volume "${SALTAPI_TMP_DIR}/config":/home/salt/data/config:ro \ - "${IMAGE_NAME}" || ( echo "container started โŒ"; exit 1 ) -echo "container started โœ…" - -# Wait for salt-master bootup -echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." -sleep "${BOOTUP_WAIT_SECONDS}" +|| error "container started" +ok "container started" - # Test salt-api authentication - echo "==> Getting salt-api token ..." +# Test salt-api authentication +echo "==> Getting salt-api token ..." SALTAPI_TOKEN=$(curl -sSk "${SALTAPI_URL%/}/login" \ - -H "Accept: application/x-yaml" \ - -d username="${SALTAPI_USER}" \ - -d password="${SALTAPI_PASS}" \ - -d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4) -[ -n "${SALTAPI_TOKEN}" ] || ( echo "salt-api token โŒ"; exit 1 ) -echo "salt-api token โœ…" + -H "Accept: application/x-yaml" \ + -d username="${SALTAPI_USER}" \ + -d password="${SALTAPI_PASS}" \ + -d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4) +[ -n "${SALTAPI_TOKEN}" ] || error "salt-api token" +ok "salt-api token" - # Test salt-api command - echo "==> Testing curl command ..." +# Test salt-api command +echo "==> Testing curl command ..." curl -sSk "${SALTAPI_URL}" \ - -H "Accept: application/x-yaml" \ - -H "X-Auth-Token: ${SALTAPI_TOKEN}" \ - -d client=runner \ - -d tgt='*' \ - -d fun=test.stream \ -| grep -i 'true' || ( echo "curl command โŒ"; exit 1 ) -echo "curl command โœ…" + -H "Accept: application/x-yaml" \ + -H "X-Auth-Token: ${SALTAPI_TOKEN}" \ + -d client=runner \ + -d tgt='*' \ + -d fun=test.stream \ +| grep -i 'true' || error "curl command" +ok "curl command" # Install salt-pepper echo "==> Installing salt-pepper ..." -pip3 install salt-pepper || ( echo "pepper installed โŒ"; exit 1 ) -echo "pepper installed โœ…" +pip3 install salt-pepper || error "pepper installed" +ok "pepper installed" # Test salt-pepper echo "==> Testing salt-pepper ..." -pepper -vvv --debug-http --ignore-ssl-errors --client runner test.stream|| ( echo "pepper test.stream โŒ"; exit 1 ) -echo "pepper test.stream โœ…" +pepper --client runner test.stream || error "pepper test.stream" +ok "pepper test.stream"