From 3252180568e6ea7ec241382d3816fcce596c79dc Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:40:02 -0400 Subject: [PATCH 01/55] feat(portability): add support for RISC-V architecture --- .github/workflows/validation.yaml | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 7f8141a6f..b470d80e1 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -399,6 +399,38 @@ jobs: with: name: openbsd-source-bitcoin-debug.log path: /home/runner/work/nodebuilder/nodebuilder/debug.log + + run-nodebuilder-riscv: + name: Test source on RISC-V + needs: [changes, shell-lint, yaml-lint] + if: >- + ${{ + github.event.action == 'prerelease' || + github.event.action == 'published' || + needs.changes.outputs.ci == 'true' || + needs.changes.outputs.dependencies == 'true' || + needs.changes.outputs.shell == 'true' + }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { arch: riscv64, distro: ubuntu_latest } + steps: + - uses: actions/checkout@v4 + - name: Test the console output + uses: uraimo/run-on-arch-action@v2 + timeout-minutes: 180 + with: + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + githubToken: ${{ github.token }} + run: | + uname -a + sh -x ./nodebuilder + [ "$?" -gt 0 ] && echo "nodebuilder exited with status $?" >&2 && exit 1 + find / -name debug.log 2> /dev/null | grep bitcoin || true run-nodebuilder-docker: name: Docker image for ${{ matrix.container }} From 2b5d8305ec94a963dbefd1b431b9b5cd1ab9aa16 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:58:51 -0400 Subject: [PATCH 02/55] Update validation.yaml --- .github/workflows/validation.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index b470d80e1..344de8b8b 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -428,6 +428,7 @@ jobs: githubToken: ${{ github.token }} run: | uname -a + id -u sh -x ./nodebuilder [ "$?" -gt 0 ] && echo "nodebuilder exited with status $?" >&2 && exit 1 find / -name debug.log 2> /dev/null | grep bitcoin || true From 0da357fbcf55813c8fae5ffd2a441d58d204b27e Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:56:33 -0400 Subject: [PATCH 03/55] update sudo --- nodebuilder | 1 + 1 file changed, 1 insertion(+) diff --git a/nodebuilder b/nodebuilder index bf1ed2d28..eacd835de 100755 --- a/nodebuilder +++ b/nodebuilder @@ -524,6 +524,7 @@ ensure_sudo_dependency() apk --quiet add sudo ;; debian | ubuntu) + apt-get update > /dev/null apt-get install -y sudo > /dev/null ;; fedora* | rhel | centos* | rocky | ol) From cd0f7b7330261787a23977683caf56edd3c17ba8 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:06:44 -0400 Subject: [PATCH 04/55] libatomic1 dependency on riscv ubuntu --- nodebuilder | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nodebuilder b/nodebuilder index eacd835de..a7f02fa7e 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1011,6 +1011,8 @@ install_runtime_dependencies_aptget() dependencies=$(curl --fail --silent --show-error --location --retry 5 "${RUNTIME_DEPENDENCIES_URL}") [ -z "${dependencies:-}" ] && throw_error 'The list of dependencies is empty.' printf '%s\n' "${dependencies}" | xargs sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends > /dev/null 2>&1 + [ "${TARGET_ARCHITECTURE}" = 'riscv64' ] && + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends libatomic1 > /dev/null 2>&1 } install_runtime_dependencies_darwin() From 0d8a47f851dfd7dc9f50692894e5420ef08fb166 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:11:00 -0400 Subject: [PATCH 05/55] Update validation.yaml --- .github/workflows/validation.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 344de8b8b..472a15a9a 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -416,6 +416,7 @@ jobs: fail-fast: false matrix: include: + - { arch: riscv64, distro: alpine_latest } - { arch: riscv64, distro: ubuntu_latest } steps: - uses: actions/checkout@v4 From 5f15e6cb8d130aecf8d57a4c44a0fb081348abfb Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 09:55:20 -0400 Subject: [PATCH 06/55] Debug the alpine compile --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index a7f02fa7e..9f04f3e3d 100755 --- a/nodebuilder +++ b/nodebuilder @@ -97,7 +97,7 @@ compile_bitcoin_from_source() gmake --jobs "${SYS_CORES_PLUS_ONE}" > /dev/null 2>&1 ;; *) - make --jobs "${SYS_CORES_PLUS_ONE}" > /dev/null 2>&1 + make --jobs "${SYS_CORES_PLUS_ONE}" ;; esac From bf04487685ca3335ce21e241c83cf0b06d01d240 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 11:50:29 -0400 Subject: [PATCH 07/55] bump the timeout minutes for riscv64 runs --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 472a15a9a..3c6568b6e 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -422,7 +422,7 @@ jobs: - uses: actions/checkout@v4 - name: Test the console output uses: uraimo/run-on-arch-action@v2 - timeout-minutes: 180 + timeout-minutes: 330 with: arch: ${{ matrix.arch }} distro: ${{ matrix.distro }} From 5f2b02846ac8a3cd25b2c374230d2f29923c05ae Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:10:13 -0400 Subject: [PATCH 08/55] add aarch64 architectures --- .github/workflows/validation.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 3c6568b6e..a7ecd6330 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -416,6 +416,9 @@ jobs: fail-fast: false matrix: include: + - { arch: aarch64, distro: alpine_latest } + - { arch: aarch64, distro: fedora_latest } + - { arch: aarch64, distro: ubuntu_latest } - { arch: riscv64, distro: alpine_latest } - { arch: riscv64, distro: ubuntu_latest } steps: @@ -491,8 +494,7 @@ jobs: matrix: container: [Gentoo] include: - - container: Gentoo - dockerfile: Dockerfile_gentoo + - { container: Gentoo, dockerfile: Dockerfile_gentoo } steps: - uses: actions/checkout@v4 - name: Check the current OS version From 494bb7e8228439ba77dea4b7c5280ab61dedfde0 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:20:00 -0400 Subject: [PATCH 09/55] add powerpc 64 --- .github/workflows/validation.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index a7ecd6330..001f6bab4 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -419,6 +419,8 @@ jobs: - { arch: aarch64, distro: alpine_latest } - { arch: aarch64, distro: fedora_latest } - { arch: aarch64, distro: ubuntu_latest } + - { arch: ppc64le, distro: alpine_latest } + - { arch: ppc64le, distro: ubuntu_latest } - { arch: riscv64, distro: alpine_latest } - { arch: riscv64, distro: ubuntu_latest } steps: From 25c0ab6b22961ff996af3469603b2e8814e39026 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:24:02 -0400 Subject: [PATCH 10/55] refactor the if statement --- nodebuilder | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 9f04f3e3d..09a3eb4d5 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1011,8 +1011,9 @@ install_runtime_dependencies_aptget() dependencies=$(curl --fail --silent --show-error --location --retry 5 "${RUNTIME_DEPENDENCIES_URL}") [ -z "${dependencies:-}" ] && throw_error 'The list of dependencies is empty.' printf '%s\n' "${dependencies}" | xargs sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends > /dev/null 2>&1 - [ "${TARGET_ARCHITECTURE}" = 'riscv64' ] && + if [ "${TARGET_ARCHITECTURE}" = 'riscv64' ]; then sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends libatomic1 > /dev/null 2>&1 + fi } install_runtime_dependencies_darwin() From fd8210642b4b47afcc77a9810dc34f86285922ad Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:39:16 -0400 Subject: [PATCH 11/55] Add artifact --- .github/workflows/validation.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 001f6bab4..8b0d94e7b 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -392,7 +392,7 @@ jobs: run: | sh -x ./test/test_nodebuilder --ref "${GITHUB_SHA}" [ "$?" -gt 0 ] && echo "test_nodebuilder exited with status $?" >&2 && exit 1 - [ -f "/home/bitcoin/.bitcoin/debug.log" ] && cp "/home/bitcoin/.bitcoin/debug.log" . + [ -f '/home/bitcoin/.bitcoin/debug.log' ] && cp '/home/bitcoin/.bitcoin/debug.log' . find / -name debug.log 2> /dev/null | grep bitcoin || true - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 @@ -432,12 +432,26 @@ jobs: arch: ${{ matrix.arch }} distro: ${{ matrix.distro }} githubToken: ${{ github.token }} - run: | + setup: | + echo "${TERM:-TERM is not found}" uname -a id -u + df -h + commmand -v tput + tput colors + date -u + mkdir -p "${PWD}/artifacts" + dockerRunArgs: --volume "${PWD}/artifacts:/artifacts" + run: | sh -x ./nodebuilder [ "$?" -gt 0 ] && echo "nodebuilder exited with status $?" >&2 && exit 1 find / -name debug.log 2> /dev/null | grep bitcoin || true + cp '${HOME}/.bitcoin/debug.log' '/artifacts/' + - name: Save Bitcoin Core log as artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.distro }}-${{ matrix.arch }}-bitcoin-debug.log + path: /artifacts/debug.log run-nodebuilder-docker: name: Docker image for ${{ matrix.container }} From 7c7db45f55eb4708105d35e794f9dd1affe710d8 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:46:50 -0400 Subject: [PATCH 12/55] handle ppc64le naming issue --- nodebuilder | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 09a3eb4d5..99b5aad49 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1550,7 +1550,15 @@ readonly BYTES_TO_KIB=1024 readonly KIB_TO_MIB="${BYTES_TO_KIB}" readonly MIB_TO_GIB="${BYTES_TO_KIB}" -TARGET_ARCHITECTURE="$(uname -m)" && readonly TARGET_ARCHITECTURE +case "$(uname -m") in + ppc64le) + TARGET_ARCHITECTURE='powerpc64' + ;; + *) + TARGET_ARCHITECTURE="$(uname -m)" + ;; +esac && readonly TARGET_ARCHITECTURE + TARGET_KERNEL="$(uname -s)" && readonly TARGET_KERNEL TARGET_OPERATING_SYSTEM="$(get_operating_system)" && readonly TARGET_OPERATING_SYSTEM TARGET_OPERATING_SYSTEM_RELEASE="$(uname -r)" && readonly TARGET_OPERATING_SYSTEM_RELEASE From 74c2e547bba9ba00cb472df089e1fa6f02e51a51 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:49:43 -0400 Subject: [PATCH 13/55] Update nodebuilder --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 99b5aad49..b0cad8012 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1550,7 +1550,7 @@ readonly BYTES_TO_KIB=1024 readonly KIB_TO_MIB="${BYTES_TO_KIB}" readonly MIB_TO_GIB="${BYTES_TO_KIB}" -case "$(uname -m") in +case "$(uname -m)" in ppc64le) TARGET_ARCHITECTURE='powerpc64' ;; From e87c8a7c16e8547c2518a80ca29f1432e9b5af0f Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:54:00 -0400 Subject: [PATCH 14/55] fix broken quotes --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 4d6a24588..8322af1c9 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -445,7 +445,7 @@ jobs: sh -x ./nodebuilder [ "$?" -gt 0 ] && echo "nodebuilder exited with status $?" >&2 && exit 1 find / -name debug.log 2> /dev/null | grep bitcoin || true - cp '${HOME}/.bitcoin/debug.log' '/artifacts/' + cp "${HOME}/.bitcoin/debug.log" '/artifacts/' - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: From 6bc6e4aa2ea5a111f58584653e6fd903b59dd0bb Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 16:56:42 -0400 Subject: [PATCH 15/55] Update validation.yaml --- .github/workflows/validation.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 8322af1c9..53f92889f 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -436,7 +436,6 @@ jobs: uname -a id -u df -h - commmand -v tput tput colors date -u mkdir -p "${PWD}/artifacts" From 5aac22c8ea064e3617daf65d5a6b94d2b94005dc Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:08:40 -0400 Subject: [PATCH 16/55] Update nodebuilder --- nodebuilder | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nodebuilder b/nodebuilder index 259acc28d..361eb46c9 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1005,8 +1005,15 @@ install_runtime_dependencies_aptget() dependencies=$(curl --fail --silent --show-error --location --retry 5 "${RUNTIME_DEPENDENCIES_URL}") [ -z "${dependencies:-}" ] && throw_error 'The list of dependencies is empty.' printf '%s\n' "${dependencies}" | xargs sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends > /dev/null 2>&1 - if [ "${TARGET_ARCHITECTURE}" = 'riscv64' ]; then - sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends libatomic1 > /dev/null 2>&1 + case "${TARGET_ARCHITECTURE}" in + powerpc64) + # TODO: make this a CI step not in the script + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends qemu-user-static > /dev/null 2>&1 + ;; + riscv64) + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends libatomic1 > /dev/null 2>&1 + ;; + fi } From fb62bedb4c1699a4dffe728de4f97e0ad0a6d815 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:12:51 -0400 Subject: [PATCH 17/55] fix syntax --- nodebuilder | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder b/nodebuilder index 361eb46c9..c6d49d6dd 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1013,8 +1013,8 @@ install_runtime_dependencies_aptget() riscv64) sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends libatomic1 > /dev/null 2>&1 ;; - - fi + *) ;; + esac } install_runtime_dependencies_darwin() From 2e445e7ef25a71e8f1b796f5d0ed8e7f28d15c4f Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:18:48 -0400 Subject: [PATCH 18/55] add noninteractive for apt --- nodebuilder | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nodebuilder b/nodebuilder index c6d49d6dd..2ee034dcb 100755 --- a/nodebuilder +++ b/nodebuilder @@ -458,7 +458,7 @@ ensure_curl_dependency() sudo apk --quiet add curl ;; debian | ubuntu) - sudo apt-get install -y curl > /dev/null + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y curl > /dev/null ;; fedora* | rhel | centos* | rocky | ol) sudo dnf install --allowerasing --assumeyes curl > /dev/null @@ -493,7 +493,7 @@ ensure_curl_dependency() if command -v apk > /dev/null; then sudo apk --quiet add curl elif command -v apt-get > /dev/null; then - sudo apt-get -qq install -y curl > /dev/null + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y curl > /dev/null elif command -v dnf > /dev/null; then sudo dnf install --allowerasing --assumeyes curl > /dev/null elif command -v pacman > /dev/null; then @@ -1220,7 +1220,7 @@ install_system_updates_aptget() { check_dpkg_lock readonly STDERR_INSTALL_LOG_FILE="${TEMP_DIRECTORY}/stderr_install.log" - sudo apt-get -qq update && sudo NEEDRESTART_MODE=a apt-get -qq dist-upgrade --assume-yes > /dev/null 2> "${STDERR_INSTALL_LOG_FILE}" + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update && sudo NEEDRESTART_MODE=a apt-get -qq dist-upgrade --assume-yes > /dev/null 2> "${STDERR_INSTALL_LOG_FILE}" grep -v 'apt-utils' "${STDERR_INSTALL_LOG_FILE}" >&2 || true rm "${STDERR_INSTALL_LOG_FILE}" } From 1c0039d22babc64e184750b1e8c65d8a2a74d0a7 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:24:14 -0400 Subject: [PATCH 19/55] Update nodebuilder --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 2ee034dcb..872898ffe 100755 --- a/nodebuilder +++ b/nodebuilder @@ -525,7 +525,7 @@ ensure_sudo_dependency() ;; debian | ubuntu) apt-get update > /dev/null - apt-get install -y sudo > /dev/null + DEBIAN_FRONTEND=noninteractive apt-get install -y sudo > /dev/null ;; fedora* | rhel | centos* | rocky | ol) dnf install --allowerasing --assumeyes sudo > /dev/null From 345aaf4fc85daeab5f9bb53f96f8afc3e32d7837 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:26:32 -0400 Subject: [PATCH 20/55] improve readability --- nodebuilder | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 872898ffe..35016618d 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1220,7 +1220,8 @@ install_system_updates_aptget() { check_dpkg_lock readonly STDERR_INSTALL_LOG_FILE="${TEMP_DIRECTORY}/stderr_install.log" - sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update && sudo NEEDRESTART_MODE=a apt-get -qq dist-upgrade --assume-yes > /dev/null 2> "${STDERR_INSTALL_LOG_FILE}" + sudo apt-get -qq update && + sudo NEEDRESTART_MODE=a apt-get -qq dist-upgrade --assume-yes > /dev/null 2> "${STDERR_INSTALL_LOG_FILE}" grep -v 'apt-utils' "${STDERR_INSTALL_LOG_FILE}" >&2 || true rm "${STDERR_INSTALL_LOG_FILE}" } From 438aacdcec1643ee654ecd12246680c4ebb12557 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:34:39 -0400 Subject: [PATCH 21/55] Update nodebuilder --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 35016618d..4a069b864 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1008,7 +1008,7 @@ install_runtime_dependencies_aptget() case "${TARGET_ARCHITECTURE}" in powerpc64) # TODO: make this a CI step not in the script - sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends qemu-user-static > /dev/null 2>&1 + sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends libc6:ppc64el > /dev/null 2>&1 ;; riscv64) sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install --assume-yes --no-install-recommends libatomic1 > /dev/null 2>&1 From 789658875af8684995312aded647c06b50173721 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 18:55:43 -0400 Subject: [PATCH 22/55] suppress stdout during make --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 4a069b864..b4cfcc8d1 100755 --- a/nodebuilder +++ b/nodebuilder @@ -97,7 +97,7 @@ compile_bitcoin_from_source() gmake --jobs "${SYS_CORES_PLUS_ONE}" > /dev/null 2>&1 ;; *) - make --jobs "${SYS_CORES_PLUS_ONE}" + make --jobs "${SYS_CORES_PLUS_ONE}" > /dev/null 2>&1 ;; esac From f5f7eaf1fa07a64ad47ec6b8661484657594d92f Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 18:58:44 -0400 Subject: [PATCH 23/55] remove fedora --- .github/workflows/validation.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 53f92889f..7664f8e74 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -416,7 +416,6 @@ jobs: matrix: include: - { arch: aarch64, distro: alpine_latest } - - { arch: aarch64, distro: fedora_latest } - { arch: aarch64, distro: ubuntu_latest } - { arch: ppc64le, distro: alpine_latest } - { arch: ppc64le, distro: ubuntu_latest } From aa0b8ccee86fa75aef739e8aaef9caaa4c68a594 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 5 Oct 2024 20:05:25 -0400 Subject: [PATCH 24/55] Update validation.yaml --- .github/workflows/validation.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 7664f8e74..3489181c4 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -417,8 +417,6 @@ jobs: include: - { arch: aarch64, distro: alpine_latest } - { arch: aarch64, distro: ubuntu_latest } - - { arch: ppc64le, distro: alpine_latest } - - { arch: ppc64le, distro: ubuntu_latest } - { arch: riscv64, distro: alpine_latest } - { arch: riscv64, distro: ubuntu_latest } steps: From b9b2427a27be2cfc9ac6b1e73f1eee6b508a1bd3 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sun, 6 Oct 2024 07:19:13 -0400 Subject: [PATCH 25/55] fix the df command with a general solution --- nodebuilder | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/nodebuilder b/nodebuilder index b4cfcc8d1..86b0dc29c 100755 --- a/nodebuilder +++ b/nodebuilder @@ -640,17 +640,7 @@ ensure_xargs_dependency() get_free_space_in_mib() { - case "${TARGET_KERNEL}" in - Darwin | FreeBSD | NetBSD) - /bin/df -m "${BITCOIN_DATA_DIRECTORY}" | awk 'NR==2 {print $4}' - ;; - OpenBSD) - BLOCKSIZE=1M df "${BITCOIN_DATA_DIRECTORY}" | awk 'NR==2 {print $4}' - ;; - *) - df --output=avail --block-size='1MiB' "${BITCOIN_DATA_DIRECTORY}" | sed 1d - ;; - esac + BLOCKSIZE=1M df "${BITCOIN_DATA_DIRECTORY}" | awk 'NR==2 {print $4}' } get_log_timestamp() From 5d2e965ce276f44c18a9b837556390725113d6e7 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sun, 6 Oct 2024 07:25:52 -0400 Subject: [PATCH 26/55] refactor the synced data check into a function --- nodebuilder | 53 +++++++++++++++++++---------------------------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/nodebuilder b/nodebuilder index 86b0dc29c..cd7435975 100755 --- a/nodebuilder +++ b/nodebuilder @@ -638,6 +638,21 @@ ensure_xargs_dependency() fi } +get_directory_size_in_mib() +{ + if [ $# -ne 1 ]; then + throw_error 'get_directory_size_in_mib() requires 1 argument.' + fi + + readonly directory_to_check="$1" + + if [ -d "${directory_to_check}/" ]; then + BLOCKSIZE=1M du -d0 "${directory_to_check}" | cut -f1 + else + printf '0\n' + fi +} + get_free_space_in_mib() { BLOCKSIZE=1M df "${BITCOIN_DATA_DIRECTORY}" | awk 'NR==2 {print $4}' @@ -1708,42 +1723,12 @@ if [ ! -f "${BITCOIN_CORE_CONFIG_FILE}" ]; then set_bitcoin_core_option 'server' '1' fi -# Synced data and prune checks -readonly BITCOIN_CORE_BLOCKS_DIRECTORY="${BITCOIN_DATA_DIRECTORY}/blocks" -if [ -d "${BITCOIN_CORE_BLOCKS_DIRECTORY}"/ ]; then - case "${TARGET_KERNEL}" in - Darwin | FreeBSD | NetBSD) - INITIAL_BLOCKS_SIZE_IN_MIB="$(du -d0 -m "${BITCOIN_CORE_BLOCKS_DIRECTORY}" | cut -f1)" - ;; - OpenBSD) - INITIAL_BLOCKS_SIZE_IN_MIB="$(BLOCKSIZE=1M du -d0 "${BITCOIN_CORE_BLOCKS_DIRECTORY}" | cut -f1)" - ;; - *) - INITIAL_BLOCKS_SIZE_IN_MIB="$(du -d0 --block-size='1MiB' "${BITCOIN_CORE_BLOCKS_DIRECTORY}" | cut -f1)" - ;; - esac -else - INITIAL_BLOCKS_SIZE_IN_MIB=0 -fi +INITIAL_BLOCKS_SIZE_IN_MIB="$(get_directory_size_in_mib "${BITCOIN_DATA_DIRECTORY}/blocks")" readonly INITIAL_BLOCKS_SIZE_IN_MIB -readonly BITCOIN_CORE_CHAINSTATE_DIRECTORY="${BITCOIN_DATA_DIRECTORY}/chainstate" -if [ -d "${BITCOIN_CORE_CHAINSTATE_DIRECTORY}"/ ]; then - case "${TARGET_KERNEL}" in - Darwin | FreeBSD | NetBSD) - INITIAL_CHAINSTATE_SIZE_IN_MIB="$(du -d0 -m "${BITCOIN_CORE_CHAINSTATE_DIRECTORY}" | cut -f1)" - ;; - OpenBSD) - INITIAL_CHAINSTATE_SIZE_IN_MIB="$(BLOCKSIZE=1M du -d0 "${BITCOIN_CORE_CHAINSTATE_DIRECTORY}" | cut -f1)" - ;; - *) - INITIAL_CHAINSTATE_SIZE_IN_MIB="$(du -d0 --block-size='1MiB' "${BITCOIN_CORE_CHAINSTATE_DIRECTORY}" | cut -f1)" - ;; - esac -else - INITIAL_CHAINSTATE_SIZE_IN_MIB=0 -fi +INITIAL_CHAINSTATE_SIZE_IN_MIB="$(get_directory_size_in_mib "${BITCOIN_DATA_DIRECTORY}/chainstate")" readonly INITIAL_CHAINSTATE_SIZE_IN_MIB -readonly INITIAL_DATA_ALREADY_SYNCED_IN_MIB=$((INITIAL_BLOCKS_SIZE_IN_MIB + INITIAL_CHAINSTATE_SIZE_IN_MIB)) +INITIAL_DATA_ALREADY_SYNCED_IN_MIB="$((INITIAL_BLOCKS_SIZE_IN_MIB + INITIAL_CHAINSTATE_SIZE_IN_MIB))" +readonly INITIAL_DATA_ALREADY_SYNCED_IN_MIB log_info "Found data already synced... $((INITIAL_DATA_ALREADY_SYNCED_IN_MIB / MIB_TO_GIB)) GiB." free_space_in_mib="$(get_free_space_in_mib)" From 68e0dafc417fb1c4eec61addfe9b1be0dd4fbde7 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sun, 6 Oct 2024 09:31:38 -0400 Subject: [PATCH 27/55] add debug steps --- .github/workflows/validation.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 3489181c4..4be002149 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -441,7 +441,11 @@ jobs: sh -x ./nodebuilder [ "$?" -gt 0 ] && echo "nodebuilder exited with status $?" >&2 && exit 1 find / -name debug.log 2> /dev/null | grep bitcoin || true - cp "${HOME}/.bitcoin/debug.log" '/artifacts/' + cp "/root/.bitcoin/debug.log" '/artifacts/' + - name: find the debug log + run: find / -name debug.log + - name: find the artifact debug log + run: find / -name debug.log | grep artifacts - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: From e61b286533d423250d713bf6b92cbb14ae128c36 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sun, 6 Oct 2024 14:49:37 -0400 Subject: [PATCH 28/55] fix artifacts --- .github/workflows/validation.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 4be002149..c89ce0bb1 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -443,14 +443,14 @@ jobs: find / -name debug.log 2> /dev/null | grep bitcoin || true cp "/root/.bitcoin/debug.log" '/artifacts/' - name: find the debug log - run: find / -name debug.log + run: find / -name debug.log 2> /dev/null || true - name: find the artifact debug log - run: find / -name debug.log | grep artifacts + run: find / -name debug.log 2> /dev/null | grep artifacts || true - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: name: ${{ matrix.distro }}-${{ matrix.arch }}-bitcoin-debug.log - path: /artifacts/debug.log + path: /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log run-nodebuilder-docker: name: Docker image for ${{ matrix.container }} From bed681eef9774d978e74cc11ae5138e740bfe8b3 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:49:06 -0400 Subject: [PATCH 29/55] try fixing artifact permissions --- .github/workflows/validation.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index c89ce0bb1..c90e340ae 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -444,8 +444,12 @@ jobs: cp "/root/.bitcoin/debug.log" '/artifacts/' - name: find the debug log run: find / -name debug.log 2> /dev/null || true - - name: find the artifact debug log - run: find / -name debug.log 2> /dev/null | grep artifacts || true + - name: check permissions of artifact log + run: ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log + - name: Modify the permissions + run: chmod 777 /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log + - name: chekc permissions again + run: ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: From 1bdc389d1587e7bcb06740e4111ed02a41f5d813 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:13:54 -0400 Subject: [PATCH 30/55] try to fix permissions --- .github/workflows/validation.yaml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index c90e340ae..a21b524c7 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -436,6 +436,7 @@ jobs: tput colors date -u mkdir -p "${PWD}/artifacts" + mkdir -p "${HOME}/artifacts" dockerRunArgs: --volume "${PWD}/artifacts:/artifacts" run: | sh -x ./nodebuilder @@ -443,13 +444,25 @@ jobs: find / -name debug.log 2> /dev/null | grep bitcoin || true cp "/root/.bitcoin/debug.log" '/artifacts/' - name: find the debug log - run: find / -name debug.log 2> /dev/null || true + run: | + find / -name debug.log 2> /dev/null || true + - name: copy debug.log artifact to home directory + run: | + cp /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log ${HOME}/artifacts/ + - name: find both of the debug logs + run: | + find / -name debug.log 2> /dev/null || true - name: check permissions of artifact log - run: ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log + run: | + ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log + ls -l ${HOME}/debug.log - name: Modify the permissions - run: chmod 777 /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log - - name: chekc permissions again - run: ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log + run: | + chmod 777 ${HOME}/artifacts/debug.log + - name: check permissions again + run: | + ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log + ls -l ${HOME}/debug.log - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: From 56e1411201058c46115562a5e03bb6c4b0062008 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:38:15 -0400 Subject: [PATCH 31/55] fix shellcheck --- .github/workflows/validation.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index a21b524c7..4d9135055 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -448,21 +448,21 @@ jobs: find / -name debug.log 2> /dev/null || true - name: copy debug.log artifact to home directory run: | - cp /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log ${HOME}/artifacts/ + cp /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log "${HOME}/artifacts/" - name: find both of the debug logs run: | find / -name debug.log 2> /dev/null || true - name: check permissions of artifact log run: | ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log - ls -l ${HOME}/debug.log + ls -l "${HOME}/debug.log" - name: Modify the permissions run: | - chmod 777 ${HOME}/artifacts/debug.log + chmod 777 "${HOME}/artifacts/debug.log" - name: check permissions again run: | ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log - ls -l ${HOME}/debug.log + ls -l "${HOME}/debug.log" - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: From 4be98c47b8e92f7436b194c740cbe57d3b5951b9 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:46:07 -0400 Subject: [PATCH 32/55] Update validation.yaml --- .github/workflows/validation.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 4d9135055..2d479e17a 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -443,6 +443,7 @@ jobs: [ "$?" -gt 0 ] && echo "nodebuilder exited with status $?" >&2 && exit 1 find / -name debug.log 2> /dev/null | grep bitcoin || true cp "/root/.bitcoin/debug.log" '/artifacts/' + chmod o+r /artifacts/debug.log - name: find the debug log run: | find / -name debug.log 2> /dev/null || true From ddd02ce3b6d9009f1da1cad047731565f0fabb01 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:46:13 -0400 Subject: [PATCH 33/55] Update validation.yaml --- .github/workflows/validation.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 2d479e17a..6a152e31c 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -455,15 +455,15 @@ jobs: find / -name debug.log 2> /dev/null || true - name: check permissions of artifact log run: | - ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log - ls -l "${HOME}/debug.log" + ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log || true + ls -l "${HOME}/debug.log" || true - name: Modify the permissions run: | chmod 777 "${HOME}/artifacts/debug.log" - name: check permissions again run: | - ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log - ls -l "${HOME}/debug.log" + ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log || true + ls -l "${HOME}/debug.log" || true - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: From f67f89430c9f7a14fe15cbceebc70491be45e27c Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:03:38 -0400 Subject: [PATCH 34/55] clean up --- .github/workflows/validation.yaml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 6a152e31c..6c305036f 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -444,26 +444,6 @@ jobs: find / -name debug.log 2> /dev/null | grep bitcoin || true cp "/root/.bitcoin/debug.log" '/artifacts/' chmod o+r /artifacts/debug.log - - name: find the debug log - run: | - find / -name debug.log 2> /dev/null || true - - name: copy debug.log artifact to home directory - run: | - cp /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log "${HOME}/artifacts/" - - name: find both of the debug logs - run: | - find / -name debug.log 2> /dev/null || true - - name: check permissions of artifact log - run: | - ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log || true - ls -l "${HOME}/debug.log" || true - - name: Modify the permissions - run: | - chmod 777 "${HOME}/artifacts/debug.log" - - name: check permissions again - run: | - ls -l /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log || true - ls -l "${HOME}/debug.log" || true - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: From 5730dd361b5568726ff2a6782cde0a11b116ddee Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Tue, 8 Oct 2024 07:30:24 -0400 Subject: [PATCH 35/55] test console output --- .github/workflows/validation.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 6c305036f..161c13a78 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -392,15 +392,14 @@ jobs: sh -x ./test/test_nodebuilder --ref "${GITHUB_SHA}" [ "$?" -gt 0 ] && echo "test_nodebuilder exited with status $?" >&2 && exit 1 [ -f '/home/bitcoin/.bitcoin/debug.log' ] && cp '/home/bitcoin/.bitcoin/debug.log' . - find / -name debug.log 2> /dev/null | grep bitcoin || true - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 with: name: openbsd-source-bitcoin-debug.log path: /home/runner/work/nodebuilder/nodebuilder/debug.log - run-nodebuilder-riscv: - name: Test source on RISC-V + run-nodebuilder-qemu: + name: Test on QEMU needs: [changes, shell-lint, yaml-lint] if: >- ${{ @@ -436,13 +435,10 @@ jobs: tput colors date -u mkdir -p "${PWD}/artifacts" - mkdir -p "${HOME}/artifacts" dockerRunArgs: --volume "${PWD}/artifacts:/artifacts" run: | - sh -x ./nodebuilder - [ "$?" -gt 0 ] && echo "nodebuilder exited with status $?" >&2 && exit 1 - find / -name debug.log 2> /dev/null | grep bitcoin || true - cp "/root/.bitcoin/debug.log" '/artifacts/' + sh -x ./test/test_nodebuilder --ref "${GITHUB_SHA}" + [ -f /root/.bitcoin/debug.log ] && cp /root/.bitcoin/debug.log /artifacts/ chmod o+r /artifacts/debug.log - name: Save Bitcoin Core log as artifact uses: actions/upload-artifact@v4 From 46a9ea01f2eea8718fc4ad9602529c91dcb86e5a Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:23:21 -0400 Subject: [PATCH 36/55] check for git before validating reference name --- test/test_nodebuilder | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test_nodebuilder b/test/test_nodebuilder index 528d35d78..550c4f337 100755 --- a/test/test_nodebuilder +++ b/test/test_nodebuilder @@ -91,12 +91,14 @@ validate_bitcoin_version() validate_git_ref_short_name() { - [ "$(git rev-parse --is-inside-work-tree 2> /dev/null)" = 'true' ] || - throw_error 'Not inside a Git repository.' "${LINENO}" - [ "$(basename "$(git rev-parse --show-toplevel)")" = 'nodebuilder' ] || - throw_error 'Not in the nodebuilder repository.' "${LINENO}" - git rev-parse --quiet --verify "$1" || - throw_error "$1 is not a valid branch, tag, or commit." "${LINENO}" + if command -v git > /dev/null 2>&1; then + [ "$(git rev-parse --is-inside-work-tree 2> /dev/null)" = 'true' ] || + throw_error 'Not inside a Git repository.' "${LINENO}" + [ "$(basename "$(git rev-parse --show-toplevel)")" = 'nodebuilder' ] || + throw_error 'Not in the nodebuilder repository.' "${LINENO}" + git rev-parse --quiet --verify "$1" || + throw_error "$1 is not a valid branch, tag, or commit." "${LINENO}" + fi } readonly VALID_BITCOIN_VERSION_LIST='0.9.5 0.10.0 0.10.1 0.10.2 0.10.3 0.10.4 \ From 382a2b052a40bf69f59e6438ea838b49afa1a769 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Wed, 9 Oct 2024 05:03:43 -0400 Subject: [PATCH 37/55] remove stdout --- test/test_nodebuilder | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_nodebuilder b/test/test_nodebuilder index 550c4f337..6a0dc6bfa 100755 --- a/test/test_nodebuilder +++ b/test/test_nodebuilder @@ -185,8 +185,7 @@ fi # Execute the script with optional command line arguments printf 'Executing command: %s\n' "${RUN_NODEBUILDER_COMMAND}" if ! ${RUN_NODEBUILDER_COMMAND} \ - > "${STDOUT_TEST_FILENAME}" \ - 2> "${STDERR_TEST_FILENAME}"; then + > "${STDOUT_TEST_FILENAME}"; then throw_error "Failed to execute nodebuilder command." fi From 1870beb145344ac0a0227aafcaeca207224af149 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:45:10 -0400 Subject: [PATCH 38/55] Update nodebuilder --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 49d73db89..b7caf6cc6 100755 --- a/nodebuilder +++ b/nodebuilder @@ -525,7 +525,7 @@ ensure_sudo_dependency() ;; debian | ubuntu) apt-get update > /dev/null - DEBIAN_FRONTEND=noninteractive apt-get install -y sudo > /dev/null + DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --quiet sudo > /dev/null ;; fedora* | rhel | centos* | rocky | ol) dnf install --allowerasing --assumeyes sudo > /dev/null From 50eca8180d556560a002926af3f32de07204b684 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Thu, 10 Oct 2024 06:58:56 -0400 Subject: [PATCH 39/55] handle case were file stderr doesnt exist --- test/test_nodebuilder | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/test_nodebuilder b/test/test_nodebuilder index 6a0dc6bfa..16716eff1 100755 --- a/test/test_nodebuilder +++ b/test/test_nodebuilder @@ -28,9 +28,15 @@ get_operating_system() handle_exit() { kill_tail_process - [ -f nodebuilder ] && rm nodebuilder - [ -f "${STDOUT_TEST_FILENAME}" ] && rm "${STDOUT_TEST_FILENAME}" - [ -f "${STDERR_TEST_FILENAME}" ] && rm "${STDERR_TEST_FILENAME}" + if [ -f nodebuilder ]; then + rm nodebuilder + fi + if [ -f "${STDOUT_TEST_FILENAME}" ]; then + rm "${STDOUT_TEST_FILENAME}" + fi + if [ -f "${STDERR_TEST_FILENAME}" ]; then + rm "${STDERR_TEST_FILENAME}" + fi } handle_error() From 8e793d08e66604006d85efb05fd5bf4e2828e86c Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:50:59 -0400 Subject: [PATCH 40/55] try to suppress apt-utils --- nodebuilder | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index b7caf6cc6..9f8c022c5 100755 --- a/nodebuilder +++ b/nodebuilder @@ -524,8 +524,11 @@ ensure_sudo_dependency() apk --quiet add sudo ;; debian | ubuntu) + readonly STDERR_INSTALL_LOG_FILE="${TEMP_DIRECTORY}/stderr_install.log" apt-get update > /dev/null - DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --quiet sudo > /dev/null + DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --quiet sudo > /dev/null 2> "${STDERR_INSTALL_LOG_FILE}" + grep -v 'apt-utils' "${STDERR_INSTALL_LOG_FILE}" >&2 || true + rm "${STDERR_INSTALL_LOG_FILE}" ;; fedora* | rhel | centos* | rocky | ol) dnf install --allowerasing --assumeyes sudo > /dev/null From 8c376791c70ad62965b22d0b55d2683fe8da1384 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 11 Oct 2024 07:40:52 -0400 Subject: [PATCH 41/55] reorder the login with first IBD status check --- nodebuilder | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder b/nodebuilder index 7b9cc7381..10924ee24 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1869,9 +1869,9 @@ if [ "${ibd_status}" = 'true' ]; then log_info 'Disabling system sleep, suspend, and hibernate.' sudo systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target > /dev/null 2>&1 fi - log_info "Close this Terminal window by clicking on the 'X'." - log_info "This screen will refresh in ${SLEEP_TIME_SECONDS} seconds." fi + log_info "Close this Terminal window by clicking on the 'X'." + log_info "This screen will refresh in ${SLEEP_TIME_SECONDS} seconds." sleep "${SLEEP_TIME_SECONDS}" else log_info 'Your node has already synced the blockchain.' From 79d6e746f9eecf02b1cc02534074acbfd821888c Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 11 Oct 2024 13:28:30 -0400 Subject: [PATCH 42/55] use two-dimensional matrix --- .github/workflows/validation.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 8673dbc15..a7e58faba 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -413,11 +413,8 @@ jobs: strategy: fail-fast: false matrix: - include: - - { arch: aarch64, distro: alpine_latest } - - { arch: aarch64, distro: ubuntu_latest } - - { arch: riscv64, distro: alpine_latest } - - { arch: riscv64, distro: ubuntu_latest } + arch: [aarch64, riscv64] + distro: [alpine_latest, ubuntu_latest] steps: - uses: actions/checkout@v4 - name: Test the console output From 7922aafdd4ae3f9c5f1d5cded4ab93f8f13c7927 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:03:11 -0400 Subject: [PATCH 43/55] use braces for variable expansion --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index ff586aabc..9ca629b72 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1568,7 +1568,7 @@ readonly KIB_TO_MIB="${BYTES_TO_KIB}" readonly MIB_TO_GIB="${BYTES_TO_KIB}" TARGET_ARCHITECTURE=$(uname -m) -case "$TARGET_ARCHITECTURE" in +case "${TARGET_ARCHITECTURE}" in ppc64le) TARGET_ARCHITECTURE='powerpc64' ;; From 9b58640c36ee9d99b48506f0765500f646437a52 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:01:53 -0400 Subject: [PATCH 44/55] lowercase job name --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 74baaf573..510203c7c 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -400,7 +400,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/debug.log run-nodebuilder-qemu: - name: Test on QEMU + name: Test on qemu needs: [changes, shell-lint, yaml-lint] if: >- ${{ From 7e730d0ab3448741983cc2df186e90a565c4425c Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 07:22:52 -0400 Subject: [PATCH 45/55] add job-purpose to CI job name --- .github/workflows/validation.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 510203c7c..3a81d16ca 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -400,7 +400,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/debug.log run-nodebuilder-qemu: - name: Test on qemu + name: Test {matrix.job-purpose} needs: [changes, shell-lint, yaml-lint] if: >- ${{ @@ -416,6 +416,11 @@ jobs: matrix: arch: [aarch64, riscv64] distro: [alpine_latest, ubuntu_latest] + include: + - distro: alpine_latest + job-purpose: source + - distro: ubuntu_latest + job-purpose: package steps: - uses: actions/checkout@v4 - name: Test the console output From f1a88ab6e34c521c3ca6c407dfff79ffa744638c Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 07:45:28 -0400 Subject: [PATCH 46/55] Update validation.yaml --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 3a81d16ca..b5b36f66e 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -400,7 +400,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/debug.log run-nodebuilder-qemu: - name: Test {matrix.job-purpose} + name: Test ${{ matrix.job-purpose }} needs: [changes, shell-lint, yaml-lint] if: >- ${{ From 555ca3048356df8f8db92a9dd51820a4f9575c1f Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 07:51:15 -0400 Subject: [PATCH 47/55] update name and reorder matrix --- .github/workflows/validation.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index b5b36f66e..8b283e293 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -400,7 +400,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/debug.log run-nodebuilder-qemu: - name: Test ${{ matrix.job-purpose }} + name: Test ${{ matrix.job-purpose }} on ${{ matrix.arch }} qemu needs: [changes, shell-lint, yaml-lint] if: >- ${{ @@ -414,13 +414,13 @@ jobs: strategy: fail-fast: false matrix: + distro: [ubuntu_latest, alpine_latest] arch: [aarch64, riscv64] - distro: [alpine_latest, ubuntu_latest] include: - - distro: alpine_latest - job-purpose: source - distro: ubuntu_latest job-purpose: package + - distro: alpine_latest + job-purpose: source steps: - uses: actions/checkout@v4 - name: Test the console output From 5347b493dcf777452c3daf26e37700e208a84b65 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 07:59:24 -0400 Subject: [PATCH 48/55] improve the name --- .github/workflows/validation.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 8b283e293..51ceba031 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -400,7 +400,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/debug.log run-nodebuilder-qemu: - name: Test ${{ matrix.job-purpose }} on ${{ matrix.arch }} qemu + name: Test ${{ matrix.job-purpose }} on ${{ matrix.arch }} ${{ matrix.distro-name }} needs: [changes, shell-lint, yaml-lint] if: >- ${{ @@ -418,8 +418,10 @@ jobs: arch: [aarch64, riscv64] include: - distro: ubuntu_latest + distro-name: Ubuntu job-purpose: package - distro: alpine_latest + distro-name: Alpine job-purpose: source steps: - uses: actions/checkout@v4 From 5153fbf222509e18bb21ca73ae8ae9802f1f07b6 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 08:05:47 -0400 Subject: [PATCH 49/55] slight reorder --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 51ceba031..ad9357dcd 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -431,6 +431,7 @@ jobs: with: arch: ${{ matrix.arch }} distro: ${{ matrix.distro }} + dockerRunArgs: --volume "${PWD}/artifacts:/artifacts" githubToken: ${{ github.token }} setup: | echo "${TERM:-TERM is not found}" @@ -440,7 +441,6 @@ jobs: tput colors date -u mkdir -p "${PWD}/artifacts" - dockerRunArgs: --volume "${PWD}/artifacts:/artifacts" run: | sh -x ./test/test_nodebuilder --ref "${GITHUB_SHA}" [ -f /root/.bitcoin/debug.log ] && cp /root/.bitcoin/debug.log /artifacts/ From f993b015fbd6a1883fb7282d474dfaadd39b2def Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 08:08:39 -0400 Subject: [PATCH 50/55] Update validation.yaml --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index ad9357dcd..231df397b 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -400,7 +400,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/debug.log run-nodebuilder-qemu: - name: Test ${{ matrix.job-purpose }} on ${{ matrix.arch }} ${{ matrix.distro-name }} + name: Test ${{ matrix.job-purpose }} on ${{ matrix.distro-name }} ${{ matrix.arch }} needs: [changes, shell-lint, yaml-lint] if: >- ${{ From 81f290a75f907e22bd64979a8a6846bafbe9a43e Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 08:17:32 -0400 Subject: [PATCH 51/55] remove distro name from job name --- .github/workflows/validation.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 76b763ffd..95fc81f5a 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -406,7 +406,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/debug.log run-nodebuilder-qemu: - name: Test ${{ matrix.job-purpose }} on ${{ matrix.distro-name }} ${{ matrix.arch }} + name: Test ${{ matrix.job-purpose }} on ${{ matrix.arch }} needs: [changes, shell-lint, yaml-lint] if: >- ${{ @@ -424,10 +424,8 @@ jobs: arch: [aarch64, riscv64] include: - distro: ubuntu_latest - distro-name: Ubuntu job-purpose: package - distro: alpine_latest - distro-name: Alpine job-purpose: source steps: - uses: actions/checkout@v4 From 13edf2a327b051cc8c782b863fbe6eaec20b6340 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 12:28:52 -0400 Subject: [PATCH 52/55] rename the dockerfile jobs --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 95fc81f5a..3bd297a21 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -456,7 +456,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log run-nodebuilder-docker: - name: Docker image for ${{ matrix.container }} + name: Test docker image for ${{ matrix.container }} needs: [changes, dockerfile-lint, shell-lint, yaml-lint] if: >- github.event.action != 'prerelease' && From d726275fcc6992336f701a40a9f117a13e515759 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 12:31:12 -0400 Subject: [PATCH 53/55] try another name --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 3bd297a21..79a4e9db3 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -456,7 +456,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log run-nodebuilder-docker: - name: Test docker image for ${{ matrix.container }} + name: Test ${{ matrix.container }} docker image needs: [changes, dockerfile-lint, shell-lint, yaml-lint] if: >- github.event.action != 'prerelease' && From d69fb53891278e70d5032915ad434764401b8d23 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 12:46:27 -0400 Subject: [PATCH 54/55] another name variant --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 79a4e9db3..1b6861124 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -456,7 +456,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log run-nodebuilder-docker: - name: Test ${{ matrix.container }} docker image + name: Build Dockerfile for ${{ matrix.container }} needs: [changes, dockerfile-lint, shell-lint, yaml-lint] if: >- github.event.action != 'prerelease' && From bb0542925182daaa965ebfcf586de630419624cf Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Sat, 12 Oct 2024 12:51:38 -0400 Subject: [PATCH 55/55] Update validation.yaml --- .github/workflows/validation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 1b6861124..b452c0d8a 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -456,7 +456,7 @@ jobs: path: /home/runner/work/nodebuilder/nodebuilder/artifacts/debug.log run-nodebuilder-docker: - name: Build Dockerfile for ${{ matrix.container }} + name: Build docker image for ${{ matrix.container }} needs: [changes, dockerfile-lint, shell-lint, yaml-lint] if: >- github.event.action != 'prerelease' &&