Enabling container registry login and better logging when access is d…

…enied. (#41) * updating to enabling multiple runs and adding protections against syncing certificates to devcontainer feature * adding login capability
microsoft · Jun 21, 2024 · 29bcc3a · 29bcc3a
1 parent 260e2ce
commit 29bcc3a
Show file tree

Hide file tree

Showing 9 changed files with 251 additions and 74 deletions.
diff --git a/.github/workflows/devcontainer-feature-build-publish.yml b/.github/workflows/devcontainer-feature-build-publish.yml
@@ -90,6 +90,12 @@ jobs:
         echo "Copying all files to /var/spacedev..."
         ./.vscode/copy_to_spacedev.sh --output-dir ./.devcontainer/features/spacefx-dev/azure-orbital-space-sdk-setup
 
+        echo "...Cleaning './.devcontainer/features/spacefx-dev/azure-orbital-space-sdk-setup/chart/certs'..."
+        # Remove the certs that might've been generated
+        while read -r certFile; do
+          rm -f "${certFile}"
+        done < <(find "./.devcontainer/features/spacefx-dev/azure-orbital-space-sdk-setup/certs" -type f ! -name "*.json")
+
         # Build the devcontainer feature
         echo "Building the devcontainer feature..."
         devcontainer features package --force-clean-output-folder ./.devcontainer/features --output-folder ./output/spacefx-dev

diff --git a/.gitignore b/.gitignore
@@ -5,4 +5,8 @@ output
 .devcontainer/features/spacefx-dev/azure-orbital-space-sdk-setup
 not
 a
-tty
+tty
+certs/*/*.pem
+certs/*/*.crt
+certs/*/*.csr
+certs/*/*.key
diff --git a/.vscode/copy_to_spacedev.sh b/.vscode/copy_to_spacedev.sh
@@ -9,7 +9,7 @@
 ############################################################
 # Script variables
 ############################################################
-REPO_ROOT_DIR=$(dirname "$0")
+REPO_ROOT_DIR=$(git rev-parse --show-toplevel)
 OUTPUT_DIR=""
 ############################################################
 # Process the input options.
@@ -27,20 +27,41 @@ done
 ############################################################
 # Clean up the destination directory by removing the sensitive parts so they can be restaged
 ############################################################
-function clean_up_dest_directory(){
-  [[ -d "${OUTPUT_DIR}/logs" ]] &&  sudo rm -rf "${OUTPUT_DIR}/logs"
-  [[ -d "${OUTPUT_DIR}/tmp" ]] &&  sudo rm -rf "${OUTPUT_DIR}/tmp"
-  [[ -d "${OUTPUT_DIR}/output" ]] &&  sudo rm -rf "${OUTPUT_DIR}/output"
-  [[ -d "${OUTPUT_DIR}/xfer" ]] &&  sudo rm -rf "${OUTPUT_DIR}/xfer"
-  [[ -d "${OUTPUT_DIR}/plugins" ]] &&  sudo rm -rf "${OUTPUT_DIR}/plugins"
-  [[ -f "${OUTPUT_DIR}/chart/Charts.lock" ]] &&  sudo rm -f "${OUTPUT_DIR}/chart/Charts.lock"
-  [[ -f "${OUTPUT_DIR}/certs/*/*.crt" ]] &&  sudo rm -f "${OUTPUT_DIR}/certs/*/*.crt"
+function copy_directory_to_dest(){
+  local directory=""
+
+  while [[ "$#" -gt 0 ]]; do
+      case $1 in
+          --directory)
+              shift
+              directory=$1
+              ;;
+          *)
+              echo "Unknown parameter '$1'"
+              exit 1
+              ;;
+      esac
+      shift
+  done
+
+  echo "...copying '${REPO_ROOT_DIR}/${directory}' to '${OUTPUT_DIR}/${directory}'..."
+
+  sudo mkdir -p "${OUTPUT_DIR}/${directory}"
+  sudo rsync -a --update --no-links \
+        --exclude='/*.log' \
+        --exclude='/*.pem' \
+        --exclude='/*.csr' \
+        --exclude='/*.key' \
+        --exclude='/*.crt' \
+        "${REPO_ROOT_DIR}/${directory}/" "${OUTPUT_DIR}/${directory}/"
 
+  if [[ $? -gt 0 ]]; then
+    echo "...error copying '${REPO_ROOT_DIR}/${directory}' to '${OUTPUT_DIR}/${directory}'"
+    exit 1
+  fi
+
+  echo "...successfully copied '${REPO_ROOT_DIR}/${directory}' to '${OUTPUT_DIR}/${directory}'..."
 
-  while read -r shellFile; do
-    chmod +x ${shellFile}
-    chmod 777 ${shellFile}
-  done < <(find "${OUTPUT_DIR}" -iname "*.sh")
 }
 
 function main() {
@@ -56,32 +77,23 @@ function main() {
 
   [[ ! -d "${OUTPUT_DIR}" ]] && sudo mkdir -p "${OUTPUT_DIR}"
 
-  echo "...outputting to '${OUTPUT_DIR}'..."
-
-  eval "sudo rsync -a --update --no-links \
-        --exclude='/.devcontainer' \
-        --exclude='/.pipelines' \
-        --exclude='/.vscode' \
-        --exclude='/.git' \
-        --exclude='/.git*' \
-        --exclude='/docs' \
-        --exclude='/tmp' \
-        --exclude='/logs' \
-        --exclude='/output' \
-        --exclude='/owners.txt' \
-        --exclude='/*.md' \
-        --exclude='/LICENSE' \
-        --exclude='/*.log' \
-        --exclude='/*.gitignore' \
-        --exclude='/*.gitattributes' \
-        --exclude='/spacedev_cache' \
-        --exclude='/.shellcheckrc' \
-        --exclude='/tests' \
-        '${REPO_ROOT_DIR}/' '${OUTPUT_DIR}/'"
+  echo "Copying Azure Orbital Space SDK to '${OUTPUT_DIR}'..."
 
-  clean_up_dest_directory
+  copy_directory_to_dest --directory "build"
+  copy_directory_to_dest --directory "certs"
+  copy_directory_to_dest --directory "chart"
+  copy_directory_to_dest --directory "config"
+  copy_directory_to_dest --directory "env"
+  copy_directory_to_dest --directory "modules"
+  copy_directory_to_dest --directory "protos"
+  copy_directory_to_dest --directory "scripts"
 
-  echo "...successfully outputted to '${OUTPUT_DIR}'."
+  while read -r shellFile; do
+    chmod +x ${shellFile}
+    chmod 777 ${shellFile}
+  done < <(find "${OUTPUT_DIR}" -iname "*.sh")
+
+  echo "...successfully copied Azure Orbital Space SDK to '${OUTPUT_DIR}'."
 
   sudo chown -R "${USER:-$(id -un)}" "${OUTPUT_DIR}"
 
@@ -90,3 +102,5 @@ function main() {
 
 main
 
+
+
diff --git a/config/0_spacesdk-base.yaml b/config/0_spacesdk-base.yaml
@@ -50,4 +50,6 @@ config:
       push_enabled: true
       pull_enabled: true
       login_enabled: true
+      login_username_file: ${HOME}/.ssh/ghcr_username
+      login_password_file: ${HOME}/.ssh/ghcr_password
   hostDirectoryMounts:
diff --git a/modules/m_50_spacefx-config.sh b/modules/m_50_spacefx-config.sh
@@ -31,8 +31,9 @@ function _generate_spacefx_config_json() {
     create_directory "${SPACEFX_DIR}/tmp/config"
 
     if [[ "${SPACEFX_CHANNEL}" != "stable" ]]; then
-        [[ ! -f "${SPACEFX_DIR}/config/channels/${SPACEFX_CHANNEL}.yaml" ]] && exit_with_error "Channel '${SPACEFX_CHANNEL}' does not exist. Please update the channel in spacefx.env and try again."
-        run_a_script "cp ${SPACEFX_DIR}/config/channels/${SPACEFX_CHANNEL}.yaml ${SPACEFX_DIR}/config/${SPACEFX_CHANNEL}.yaml"
+        debug_log "Channel '${SPACEFX_CHANNEL}' detected.  Copying channel config '${SPACEFX_DIR}/config/channels/${SPACEFX_CHANNEL}.yaml' to '${SPACEFX_DIR}/config/${SPACEFX_CHANNEL}.yaml'."
+        [[ ! -f "${SPACEFX_DIR}/config/channels/${SPACEFX_CHANNEL}.yaml" ]] && exit_with_error "Channel config '${SPACEFX_DIR}/config/channels/${SPACEFX_CHANNEL}.yaml' does not exist. Please update the channel in spacefx.env and try again."
+        run_a_script "cp ${SPACEFX_DIR}/config/channels/${SPACEFX_CHANNEL}.yaml ${SPACEFX_DIR}/config/${SPACEFX_CHANNEL}.yaml" --disable_log
     fi
 
     # Build the JSON output from the configuration in yq

diff --git a/modules/m_60_container_registries.sh b/modules/m_60_container_registries.sh
@@ -217,19 +217,29 @@ function find_registry_for_image(){
     fi
 
     info_log "Locating registry for '${container_image}'..."
-
     run_a_script "jq -r '.config.containerRegistries[] | select(.pull_enabled == true) | @base64' ${SPACEFX_DIR}/tmp/config/spacefx-config.json" container_registries --disable_log
 
     REGISTRY_IMAGE_NAME=""
 
     for row in $container_registries; do
         parse_json_line --json "${row}" --property ".url" --result container_registry
+        parse_json_line --json "${row}" --property ".login_enabled" --result login_enabled
+        parse_json_line --json "${row}" --property ".login_username_file" --result login_username_file
+        parse_json_line --json "${row}" --property ".login_password_file" --result login_password_file
 
         check_for_repo_prefix --registry "${container_registry}" --repo "${container_image}" --result _find_registry_for_image_repo
 
         info_log "Checking container registry '${container_registry}' for image '${_find_registry_for_image_repo}'..."
 
-        run_a_script "regctl image manifest ${container_registry}/${_find_registry_for_image_repo}" --ignore_error --disable_log
+        if [[ "${login_enabled}" == "true" ]]; then
+            login_to_container_registry --container_registry "${container_registry}" --container_registry_username_file "${login_username_file}" --container_registry_password_file "${login_password_file}"
+        fi
+
+        run_a_script "regctl image manifest ${container_registry}/${_find_registry_for_image_repo}" _find_registry_for_image_result --ignore_error --disable_log
+
+        if [[ "${_find_registry_for_image_result}" == *"unauthorized"* ]]; then
+            exit_with_error "Unauthorized to access image to container registry '${container_registry}'.  Please login with docker login '${container_registry}', regctl registry login '${container_registry}' --user <username> --pass <password>, or use the config login_username_file and login_password_file configuration options"
+        fi
 
         if [[ "${RETURN_CODE}" -eq 0 ]]; then
             info_log "...image '${container_image}' FOUND in container registry '${container_registry}' (as '${_find_registry_for_image_repo}')"
@@ -243,6 +253,103 @@ function find_registry_for_image(){
     eval "$return_result_var='$REGISTRY_IMAGE_NAME'"
 }
 
+############################################################
+# Login to container registry
+############################################################
+function login_to_container_registry(){
+    info_log "START: ${FUNCNAME[0]}"
+
+    local container_registry=""
+    local container_registry_username_file=""
+    local container_registry_password_file=""
+
+    local is_logged_in=false
+
+    while [[ "$#" -gt 0 ]]; do
+        case $1 in
+        --container_registry)
+            shift
+            container_registry=$1
+            ;;
+        --container_registry_username_file)
+            shift
+            container_registry_username_file=$1
+            ;;
+        --container_registry_password_file)
+            shift
+            container_registry_password_file=$1
+            ;;
+        esac
+        shift
+    done
+
+    [[ -z "${container_registry}" ]] && exit_with_error "--container_registry empty.  Please supply a container registry to login to"
+
+    info_log "container_registry_username_file '${container_registry_username_file}'..."
+
+    is_cmd_available "docker" HAS_DOCKER
+
+    if [[ "${HAS_DOCKER}" == true ]]; then
+        trace_log "Docker detected.  Checking if we're already logged in to '${container_registry}'..."
+
+        if [[ -f "${HOME}/.docker/config.json" ]]; then
+            run_a_script "jq -r '.auths | has(\"${container_registry}\")' ${HOME}/.docker/config.json" is_logged_in
+        fi
+
+        if [[ "${is_logged_in}" == false ]]; then
+            run_a_script "docker logout ${container_registry}" --ignore_error --disable_log
+            run_a_script "docker logout ${container_registry}" --ignore_error --disable_log --no_sudo
+
+            [[ -z "${container_registry_username_file}" ]] && exit_with_error "--container_registry_username_file empty. Please supply a container registry username file to login to"
+            [[ -z "${container_registry_password_file}" ]] && exit_with_error "--container_registry_password_file empty.  Please supply a container registry password file to login to"
+            [[ ! -f "${container_registry_username_file}" ]] && exit_with_error "Unable to login to '${container_registry}'.  Username file '${container_registry_username_file}' not found"
+            [[ ! -f "${container_registry_password_file}" ]] && exit_with_error "Unable to login to '${container_registry}'.  Password file '${container_registry_password_file}' not found"
+
+            run_a_script "cat ${container_registry_username_file}" container_registry_username --disable_log
+            run_a_script "cat ${container_registry_password_file}" container_registry_password --disable_log
+            run_a_script "docker login ${container_registry} --username '${container_registry_username}' --password '${container_registry_password}'" --disable_log
+            run_a_script "docker login ${container_registry} --username '${container_registry_username}' --password '${container_registry_password}'" --disable_log --no_sudo
+
+            is_logged_in=true
+        else
+            info_log "Already logged in to '${container_registry}' with Docker."
+            info_log "END: ${FUNCNAME[0]}"
+            return
+        fi
+    fi
+
+    # This will allow us to login with regctl if docker is not available
+    is_cmd_available "regctl" HAS_REGCTL
+    if [[ "${HAS_REGCTL}" == true ]]; then
+        trace_log "Regctl detected.  Checking if we're already logged in to '${container_registry}'..."
+
+        if [[ -f "${HOME}/.regctl/config.json" ]]; then
+            run_a_script "jq -r '.hosts | has(\"${container_registry}\")' ${HOME}/.regctl/config.json" is_logged_in --disable_log
+        fi
+
+        if [[ "${is_logged_in}" == false ]]; then
+            run_a_script "regctl registry logout ${container_registry}" --ignore_error --disable_log
+            run_a_script "regctl registry logout ${container_registry}" --ignore_error --disable_log --no_sudo
+
+            [[ -z "${container_registry_username_file}" ]] && exit_with_error "--container_registry_username_file empty. Please supply a container registry username file to login to"
+            [[ -z "${container_registry_password_file}" ]] && exit_with_error "--container_registry_password_file empty.  Please supply a container registry password file to login to"
+            [[ ! -f "${container_registry_username_file}" ]] && exit_with_error "Unable to login to '${container_registry}'.  Username file '${container_registry_username_file}' not found"
+            [[ ! -f "${container_registry_password_file}" ]] && exit_with_error "Unable to login to '${container_registry}'.  Password file '${container_registry_password_file}' not found"
+
+            run_a_script "cat ${container_registry_username_file}" container_registry_username --disable_log
+            run_a_script "cat ${container_registry_password_file}" container_registry_password --disable_log
+            run_a_script "regctl registry login ${container_registry} --user '${container_registry_username}' --pass '${container_registry_password}'" --disable_log
+            run_a_script "regctl registry login ${container_registry} --user '${container_registry_username}' --pass '${container_registry_password}'" --disable_log --no_sudo
+
+            is_logged_in=true
+        else
+            trace_log "Already logged in to '${container_registry}' with Regctl."
+        fi
+    fi
+
+
+    info_log "END: ${FUNCNAME[0]}"
+}
 
 ############################################################
 # Push a local image to a repository

diff --git a/modules/m_70_certificates.sh b/modules/m_70_certificates.sh
@@ -170,7 +170,7 @@ function generate_certificate() {
         return
     fi
 
-    create_directory ${output_dir}
+    create_directory "${output_dir}"
 
     if [[ -f "${output_dir}/${cert_name}.crt" ]]; then
         debug_log "Removing out-of-date '${output_dir}/${cert_name}.crt'"
@@ -182,7 +182,7 @@ function generate_certificate() {
 
     cd "${output_dir}" || exit_with_error "Failed to cd to '${output_dir}'"
 
-    run_a_script "cfssl gencert -ca=${SPACEFX_DIR}/certs/ca/ca.spacefx.local.pem -ca-key=${SPACEFX_DIR}/certs/ca/ca.spacefx.local.key -config=${cert_config} -profile=server ${cert_profile} | cfssljson -bare ${cert_name}" --no_log_results
+    run_a_script "cfssl gencert -ca=${SPACEFX_DIR}/certs/ca/ca.spacefx.local.pem -ca-key=${SPACEFX_DIR}/certs/ca/ca.spacefx.local.key -config=${cert_config} -profile=server ${cert_profile} | cfssljson -bare ${cert_name}" --disable_log
 
     cd - || exit_with_error "Failed to cd back"