Merge pull request #108 from cdalvaro/bugfix/pygit2_1.7.0

Use latests version of pygit2

.github/workflows/build-and-test.yml

@@ -119,6 +119,17 @@ jobs:
       - name: Execute salt-api tests
         run: tests/salt-api/test.sh
 
+      - name: Execute gitfs tests
+        env:
+          GITFS_KEYS_DIR: tests/gitfs/data/keys/gitfs
+        run: |
+          mkdir -p "${GITFS_KEYS_DIR}"
+          echo "${{ secrets.TESTS_REPO_PRIVATE_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh
+          chmod 600 "${GITFS_KEYS_DIR}"/gitfs_ssh
+          echo "${{ secrets.TESTS_REPO_PUBLIC_KEY }}" > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub
+          chmod 644 "${GITFS_KEYS_DIR}"/gitfs_ssh.pub
+          tests/gitfs/test.sh
+
       - name: Cleanup
         run: |
           docker stop registry

.gitignore

@@ -1,7 +1,13 @@
+# Always included
+!assets/**
+
 # development
 /config/
 /roots/
 /keys/
 /logs/
 /3pfs/
-.vscode/
+.vscode/
+
+# tests
+/tests/**/keys/

CHANGELOG.md

@@ -4,6 +4,14 @@ This file only reflects the changes that are made in this image.
 Please refer to the [Salt 3004 Release Notes](https://docs.saltstack.com/en/latest/topics/releases/3004.html)
 for the list of changes in SaltStack.
 
+**3004_1**
+
+- Install `libssh2 1.10.0` from source
+- Install `libgit2 1.3.0` from source
+- Install `pygit2 1.7.0` from pip repositories
+- Change Docker base image to `ubuntu:hirsute-20210917`
+- Upgrade Python to version `3.9`
+
 **3004**
 
 - Upgrade `salt-master` to `3004` *Silicon*

Dockerfile

@@ -1,13 +1,13 @@
-FROM ubuntu:focal-20211006
+FROM ubuntu:hirsute-20210917
 
 ARG BUILD_DATE
 ARG VCS_REF
 
 # https://github.com/saltstack/salt/releases
 ENV SALT_VERSION="3004" \
-    PYTHON_VERSION="3.8"
+    PYTHON_VERSION="3.9"
 
-ENV IMAGE_VERSION="${SALT_VERSION}"
+ENV IMAGE_VERSION="${SALT_VERSION}_1"
 
 ENV SALT_DOCKER_DIR="/etc/docker-salt" \
     SALT_ROOT_DIR="/etc/salt" \
@@ -32,7 +32,7 @@ WORKDIR ${SALT_BUILD_DIR}
 # hadolint ignore=DL3008
 RUN apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \
-    sudo ca-certificates apt-transport-https wget locales openssh-client \
+    sudo ca-certificates openssl apt-transport-https wget locales openssh-client \
     python${PYTHON_VERSION} python3-dev libpython3-dev \
     python3-pip python3-setuptools python3-wheel \
     supervisor logrotate git gettext-base tzdata \

README.md

@@ -319,7 +319,7 @@ This keys must be placed inside `/home/salt/data/keys` directory.
 You can create an ssh key for pygit2 with the following command:
 
 ```sh
-ssh-keygen -f gitfs_pygit2 -C 'gitfs@example.com'
+ssh-keygen -f gitfs_ssh -C 'gitfs@example.com'
 ```
 
 Place it wherever you want inside the container and specify its path with the configuration parameters: `gitfs_pubkey` and `gitfs_privkey` in your `.conf` file.
@@ -334,13 +334,19 @@ gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub
 
 **Important Note**
 
+By default, this image has been tested with RSA 4096 ssh keys generated with `ssh-keygen`.
+
 If you get the following error while using `gitfs` with `pygit2`
 
 ```plain
 _pygit2.GitError: Failed to authenticate SSH session: Unable to send userauth-publickey request
 ```
 
-look if your private key hash empty lines at the bottom of the file and suppress them for solving the error.
+you may have to recreate your ssh key adding the parameter: `-m PEM`:
+
+```sh
+ssh-keygen -m PEM -f gitfs_ssh -C 'gitfs@example.com'
+```
 
 ### 3rd Party Formulas
 
@@ -507,6 +513,7 @@ Below you can find a list with the available options that can be used to customi
 | `SALT_API_SERVICE_ENABLED`         | Enable `salt-api` service. Default: `false`                                                                                                                                                                                |
 | `SALT_API_USER`                    | Set username for `salt-api` service. Default: `salt_api`                                                                                                                                                                   |
 | `SALT_API_USER_PASS`               | `SALT_API_USER` password. Required if `SALT_API_SERVICE_ENBALED` is `true` and `SALT_API_USER` is not empty. _Unset_ by default                                                                                            |
+| `SALT_API_CERT_CN`                 | Common name in the request. Default: `localhost` |
 | `SALT_MASTER_SIGN_PUBKEY`          | Sign the master auth-replies with a cryptographic signature of the master's public key. Possible values: 'True' or 'False'. Default: `False`                                                                               |
 | `SALT_MASTER_USE_PUBKEY_SIGNATURE` | Instead of computing the signature for each auth-reply, use a pre-calculated signature. This option requires `SALT_MASTER_SIGN_PUBKEY` set to 'True'. Possible values: 'True' or 'False'. Default: `True`                  |
 | `SALT_MASTER_SIGN_KEY_NAME`        | The customizable name of the signing-key-pair without suffix. Default: `master_sign`                                                                                                                                       |
@@ -579,7 +586,7 @@ Where `salt-service` is one of: `salt-master` os `salt-api` (if `SALT_API_SERVIC
 
 [saltproject_badge]: https://img.shields.io/badge/Salt-v3004-lightgrey.svg?logo=Saltstack
 [saltproject_release_notes]: https://docs.saltproject.io/en/latest/topics/releases/3004.html "Salt Project Release Notes"
-[ubuntu_badge]: https://img.shields.io/badge/ubuntu-focal--20211006-E95420.svg?logo=Ubuntu
+[ubuntu_badge]: https://img.shields.io/badge/ubuntu-hirsute--20210917-E95420.svg?logo=Ubuntu
 [ubuntu_hub_docker]: https://hub.docker.com/_/ubuntu/ "Ubuntu Image"
 [github_publish_badge]: https://img.shields.io/github/workflow/status/cdalvaro/docker-salt-master/Publish%20Docker%20image?label=build&logo=GitHub&logoColor=%23181717
 [github_publish_workflow]: https://github.com/cdalvaro/docker-salt-master/actions?query=workflow%3A%22Publish+Docker+image%22

assets/build/functions.sh

@@ -2,7 +2,45 @@
 
 set -e
 
-# Execute a command as SALT_USER
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  log_debug
+#   DESCRIPTION:  Echo debug information to stdout.
+#----------------------------------------------------------------------------------------------------------------------
+function log_debug() {
+  if [[ "${DEBUG}" == 'true' || "${ECHO_DEBUG}" == 'true' ]]; then
+    echo " *  DEBUG: $*"
+  fi
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  log_info
+#   DESCRIPTION:  Echo information to stdout.
+#----------------------------------------------------------------------------------------------------------------------
+function log_info() {
+  echo " *  INFO: $*"
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  log_warn
+#   DESCRIPTION:  Echo warning information to stdout.
+#----------------------------------------------------------------------------------------------------------------------
+function log_warn() {
+  echo " *  WARN: $*"
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  log_error
+#   DESCRIPTION:  Echo errors to stderr.
+#----------------------------------------------------------------------------------------------------------------------
+function log_error()
+{
+  (>&2 echo " *  ERROR: $*")
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  exec_as_salt
+#   DESCRIPTION:  Execute the pass command as the `SALT_USER` user.
+#----------------------------------------------------------------------------------------------------------------------
 function exec_as_salt()
 {
   if [[ $(whoami) == "${SALT_USER}" ]]; then
@@ -11,3 +49,171 @@ function exec_as_salt()
     sudo -HEu "${SALT_USER}" "$@"
   fi
 }
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  is_arm32
+#   DESCRIPTION:  Check whether the platform is ARM 32-bits or not.
+#----------------------------------------------------------------------------------------------------------------------
+function is_arm32()
+{
+  uname -m | grep -qE 'armv7l'
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  is_arm32
+#   DESCRIPTION:  Check whether the platform is ARM 64-bits or not.
+#----------------------------------------------------------------------------------------------------------------------
+function is_arm64()
+{
+  uname -m | grep -qE 'arm64|aarch64'
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  is_arm32
+#   DESCRIPTION:  Check whether the platform is ARM or not.
+#----------------------------------------------------------------------------------------------------------------------
+function is_arm()
+{
+  is_arm32 || is_arm64
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  install_pkgs
+#   DESCRIPTION:  Install packages using apt-get install.
+#----------------------------------------------------------------------------------------------------------------------
+function install_pkgs()
+{
+  apt-get install --no-install-recommends --yes $@
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  download
+#   DESCRIPTION:  Download the content from the given URL and save it into the specified file.
+#----------------------------------------------------------------------------------------------------------------------
+function download()
+{
+  local URL="$1"
+  local FILE_NAME="$2"
+
+  local WGET_ARGS=(--quiet)
+  is_arm32 && WGET_ARGS+=(--no-check-certificate)
+
+  log_info "Downloading ${FILE_NAME} from ${URL} ..."
+  wget ${WGET_ARGS[@]} -O "${FILE_NAME}" "${URL}"
+  if [[ -f "${FILE_NAME}" ]]; then
+    log_debug "Success!"
+  else
+    log_error "Failed to download ${URL}"
+    exit 1
+  fi
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  check_sha256
+#   DESCRIPTION:  Compute the SHA256 hash for the given file and check if it matches the expected one.
+#----------------------------------------------------------------------------------------------------------------------
+function check_sha256()
+{
+  local FILE="${1}"
+  local SHA256="${2}"
+
+  log_info "Checking ${FILE} SHA256 hash ..."
+  if echo "${SHA256}  ${FILE}" | shasum -a 256 -c --status -; then
+    log_debug "SHA256 hash for ${FILE} matches! (${SHA256})"
+  else
+    local HASH=$(shasum -a 256 "${FILE}" | awk '{print $1}')
+    log_error "SHA256 checksum mismatch for ${FILE}"
+    log_error "Expected: ${SHA256}"
+    log_error "     Got: ${HASH}"
+    exit 1
+  fi
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  extract
+#   DESCRIPTION:  Extract the given .tar.gz into the current directory.
+#----------------------------------------------------------------------------------------------------------------------
+function extract()
+{
+  local FILE="${1}"
+  log_info "Unpacking file: ${FILE}"
+  tar xzf "${FILE}" --strip-components 1
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  build_and_install
+#   DESCRIPTION:  Build and install the given package from the current directory using cmake.
+#----------------------------------------------------------------------------------------------------------------------
+function build_and_install()
+{
+  local PACKAGE_NAME="${1}"; shift
+  local CMAKE_ARGS=(
+    -Wno-dev
+    -DCMAKE_BUILD_TYPE=Release
+  )
+
+  # shellcheck disable=SC2206
+  CMAKE_ARGS+=( $@ )
+
+  log_info "Building and installing ${PACKAGE_NAME} ..."
+  log_debug "CMAKE_ARGS: ${CMAKE_ARGS[@]}"
+  cmake ${CMAKE_ARGS[@]} .
+  cmake --build . --target install --config Release
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  install_libssh2
+#   DESCRIPTION:  Install libssh2 library.
+#----------------------------------------------------------------------------------------------------------------------
+function install_libssh2()
+{
+  local LIBSSH2_VERSION=1.10.0
+  local LIBSSH2_URL="https://github.com/libssh2/libssh2/releases/download/libssh2-${LIBSSH2_VERSION}/libssh2-${LIBSSH2_VERSION}.tar.gz"
+  local FILE_NAME="libssh2-${LIBSSH2_VERSION}.tar.gz"
+  local SHA256_SUM='2d64e90f3ded394b91d3a2e774ca203a4179f69aebee03003e5a6fa621e41d51'
+
+  local CURRENT_DIR="$(pwd)"
+  local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}"
+
+  download "${LIBSSH2_URL}" "${FILE_NAME}"
+  check_sha256 "${FILE_NAME}" "${SHA256_SUM}"
+  extract "${FILE_NAME}"
+
+  _OPTS=(
+    -DLINT=OFF
+    -DBUILD_SHARED_LIBS=ON
+    -DCRYPTO_BACKEND=OpenSSL
+    -DENABLE_ZLIB_COMPRESSION=ON
+    -DENABLE_DEBUG_LOGGING=OFF
+    -DCLEAR_MEMORY=ON
+  )
+
+  build_and_install "libssh2 v${LIBSSH2_VERSION}" ${_OPTS[@]}
+
+  cd "${CURRENT_DIR}"
+  rm -rf "${WORK_DIR}"
+}
+
+#---  FUNCTION  -------------------------------------------------------------------------------------------------------
+#          NAME:  install_libgit2
+#   DESCRIPTION:  Install libgit2 library.
+#----------------------------------------------------------------------------------------------------------------------
+function install_libgit2()
+{
+  local LIBGIT2_VERSION=1.3.0
+  local LIBGIT2_URL="https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz"
+  local FILE_NAME="libgit2-${LIBGIT2_VERSION}.tar.gz"
+  local SHA256_SUM='192eeff84596ff09efb6b01835a066f2df7cd7985e0991c79595688e6b36444e'
+
+  local CURRENT_DIR="$(pwd)"
+  local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}"
+
+  download "${LIBGIT2_URL}" "${FILE_NAME}"
+  check_sha256 "${FILE_NAME}" "${SHA256_SUM}"
+  extract "${FILE_NAME}"
+
+  build_and_install "libgit2 v${LIBGIT2_VERSION}" -DBUILD_CLAR=OFF
+
+  cd "${CURRENT_DIR}"
+  rm -rf "${WORK_DIR}"
+}