From d0835dc7aa2ea6885798170302ea33de57af9e15 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 13:05:14 +0100
Subject: [PATCH 01/37] [experiment] test code build runner

---
 src/ci/github-actions/jobs.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index 2ea37c168dd3c..6bb7e2230d0fd 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -46,6 +46,10 @@ runners:
   - &job-aarch64-linux
     os: ubuntu-22.04-arm64-8core-32gb
 
+  - &job-linux-4c-codebuild
+    os: codebuild-ubuntu-24-4c-${{ github.run_id }}-${{ github.run_attempt }}
+    <<: *base-job
+
 envs:
   env-x86_64-apple-tests: &env-x86_64-apple-tests
     SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact
@@ -237,7 +241,7 @@ auto:
     <<: *job-linux-4c
 
   - image: x86_64-gnu
-    <<: *job-linux-4c
+    <<: *job-linux-4c-codebuild
 
   # This job ensures commits landing on nightly still pass the full
   # test suite on the stable channel. There are some UI tests that

From efb9f7135f7f2fbd731c8de2e41944c9ad434b2f Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 15:28:03 +0100
Subject: [PATCH 02/37] fix python script

---
 src/ci/github-actions/calculate-job-matrix.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
index 7de6d5fcd5f75..faa6da7243c01 100755
--- a/src/ci/github-actions/calculate-job-matrix.py
+++ b/src/ci/github-actions/calculate-job-matrix.py
@@ -176,6 +176,21 @@ def format_run_type(run_type: WorkflowRunType) -> str:
         raise AssertionError()
 
 
+# Add new function before main:
+def substitute_github_vars(jobs: list) -> list:
+    """Replace GitHub context variables with environment variables in job configs."""
+    for job in jobs:
+        if "os" in job:
+            job["os"] = job["os"].replace(
+                "${{ github.run_id }}",
+                os.environ["GITHUB_RUN_ID"]
+            ).replace(
+                "${{ github.run_attempt }}",
+                os.environ["GITHUB_RUN_ATTEMPT"]
+            )
+    return jobs
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
 
@@ -195,6 +210,7 @@ def format_run_type(run_type: WorkflowRunType) -> str:
         jobs = calculate_jobs(run_type, data)
     jobs = skip_jobs(jobs, channel)
 
+
     if not jobs:
         raise Exception("Scheduled job list is empty, this is an error")
 

From 9eaf00c848ccefd84e6a4853220e7fbe0a2fce5c Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 15:39:20 +0100
Subject: [PATCH 03/37] fix python

---
 src/ci/github-actions/calculate-job-matrix.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
index faa6da7243c01..969acd992dcc0 100755
--- a/src/ci/github-actions/calculate-job-matrix.py
+++ b/src/ci/github-actions/calculate-job-matrix.py
@@ -209,6 +209,7 @@ def substitute_github_vars(jobs: list) -> list:
     if run_type is not None:
         jobs = calculate_jobs(run_type, data)
     jobs = skip_jobs(jobs, channel)
+    jobs = substitute_github_vars(jobs)
 
 
     if not jobs:

From ee92da1480dd3973a5c193e6dac4bb2e77dde439 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 16:00:47 +0100
Subject: [PATCH 04/37] install docker

---
 .github/workflows/ci.yml           | 3 +++
 src/ci/scripts/ubuntu-codebuild.sh | 3 +++
 2 files changed, 6 insertions(+)
 create mode 100644 src/ci/scripts/ubuntu-codebuild.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f6df348b72140..66aa0e5a6236d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -145,6 +145,9 @@ jobs:
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
 
+      - if: contains(matrix.os, 'codebuild-ubuntu')
+        run: src/ci/scripts/ubuntu-codebuild.sh
+
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
 
diff --git a/src/ci/scripts/ubuntu-codebuild.sh b/src/ci/scripts/ubuntu-codebuild.sh
new file mode 100644
index 0000000000000..de5634886f023
--- /dev/null
+++ b/src/ci/scripts/ubuntu-codebuild.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+sudo apt install -y docker-buildx

From 48f9d4f4999c3138f88a656f9842141ec016b3f5 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 16:10:55 +0100
Subject: [PATCH 05/37] fix apt

---
 src/ci/scripts/ubuntu-codebuild.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ci/scripts/ubuntu-codebuild.sh b/src/ci/scripts/ubuntu-codebuild.sh
index de5634886f023..fefb28a58f249 100644
--- a/src/ci/scripts/ubuntu-codebuild.sh
+++ b/src/ci/scripts/ubuntu-codebuild.sh
@@ -1,3 +1,4 @@
 #!/bin/bash
 
-sudo apt install -y docker-buildx
+apt-get update
+apt-get install -y docker-buildx

From c1c40b5fab837746de3121d7bdad2786cf062e76 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 16:25:51 +0100
Subject: [PATCH 06/37] fix apt

---
 .github/workflows/ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 66aa0e5a6236d..e59eef89d86d9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -146,7 +146,8 @@ jobs:
         run: src/ci/scripts/install-awscli.sh
 
       - if: contains(matrix.os, 'codebuild-ubuntu')
-        run: src/ci/scripts/ubuntu-codebuild.sh
+        name: Install sotftware for codebuild
+        run: chmod +x src/ci/scripts/ubuntu-codebuild.sh && ./src/ci/scripts/ubuntu-codebuild.sh
 
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh

From 7ade7c4c05eb7bf048fe69359838ee7e9bb5d3e7 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 16:58:50 +0100
Subject: [PATCH 07/37] debug

---
 src/ci/scripts/enable-docker-ipv6.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ci/scripts/enable-docker-ipv6.sh b/src/ci/scripts/enable-docker-ipv6.sh
index 03d5a75e24e27..257ffe374f90d 100755
--- a/src/ci/scripts/enable-docker-ipv6.sh
+++ b/src/ci/scripts/enable-docker-ipv6.sh
@@ -7,6 +7,9 @@ IFS=$'\n\t'
 
 source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
 
+# Print docker version
+docker --version
+
 if isLinux; then
     sudo mkdir -p /etc/docker
     echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' \

From 05a77eed55688ebdb659270c71c17fd7d8860f59 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 17:40:11 +0100
Subject: [PATCH 08/37] debug info

---
 src/ci/scripts/enable-docker-ipv6.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ci/scripts/enable-docker-ipv6.sh b/src/ci/scripts/enable-docker-ipv6.sh
index 257ffe374f90d..fb09a399d9bec 100755
--- a/src/ci/scripts/enable-docker-ipv6.sh
+++ b/src/ci/scripts/enable-docker-ipv6.sh
@@ -9,6 +9,9 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
 
 # Print docker version
 docker --version
+lsb_release -a
+echo "is docker active"
+systemctl is-active docker
 
 if isLinux; then
     sudo mkdir -p /etc/docker

From f0fc184342cc4fd8a9b2e4a29595223365826107 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 5 Dec 2024 17:51:43 +0100
Subject: [PATCH 09/37] no ipv6

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e59eef89d86d9..d78d492127ad8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -178,6 +178,8 @@ jobs:
 
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
+        # Don't run on codebuild because systemctl is not available
+        if: ${{ !contains(matrix.os, 'codebuild-ubuntu') }}
 
       # Disable automatic line ending conversion (again). On Windows, when we're
       # installing dependencies, something switches the git configuration directory or

From b8188740ec208a7aebe0180adeb8ed0783f237e9 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 6 Dec 2024 16:15:22 +0100
Subject: [PATCH 10/37] update codebuild project

---
 src/ci/github-actions/jobs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index 6bb7e2230d0fd..7df149128b4e2 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -47,7 +47,7 @@ runners:
     os: ubuntu-22.04-arm64-8core-32gb
 
   - &job-linux-4c-codebuild
-    os: codebuild-ubuntu-24-4c-${{ github.run_id }}-${{ github.run_attempt }}
+    os: codebuild-ubuntu-22-4c-${{ github.run_id }}-${{ github.run_attempt }}
     <<: *base-job
 
 envs:

From ba9a0c6aca6930339e6dc4f6d7c258d20df750e6 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 11 Dec 2024 12:16:39 +0100
Subject: [PATCH 11/37] update to 8 core

---
 src/ci/github-actions/jobs.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index 7df149128b4e2..10756dd4951cd 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -46,8 +46,8 @@ runners:
   - &job-aarch64-linux
     os: ubuntu-22.04-arm64-8core-32gb
 
-  - &job-linux-4c-codebuild
-    os: codebuild-ubuntu-22-4c-${{ github.run_id }}-${{ github.run_attempt }}
+  - &job-linux-8c-codebuild
+    os: codebuild-ubuntu-22-8c-${{ github.run_id }}-${{ github.run_attempt }}
     <<: *base-job
 
 envs:
@@ -241,7 +241,7 @@ auto:
     <<: *job-linux-4c
 
   - image: x86_64-gnu
-    <<: *job-linux-4c-codebuild
+    <<: *job-linux-8c-codebuild
 
   # This job ensures commits landing on nightly still pass the full
   # test suite on the stable channel. There are some UI tests that

From 3f1e15646f09849a68715e6e242380b83ece974d Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 11 Dec 2024 15:08:31 +0100
Subject: [PATCH 12/37] exclude problematic test

---
 tests/rustdoc-ui/issues/issue-98690.rs     | 10 ----------
 tests/rustdoc-ui/issues/issue-98690.stderr |  1 -
 2 files changed, 11 deletions(-)
 delete mode 100644 tests/rustdoc-ui/issues/issue-98690.rs
 delete mode 100644 tests/rustdoc-ui/issues/issue-98690.stderr

diff --git a/tests/rustdoc-ui/issues/issue-98690.rs b/tests/rustdoc-ui/issues/issue-98690.rs
deleted file mode 100644
index 01708f3f64d3e..0000000000000
--- a/tests/rustdoc-ui/issues/issue-98690.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-//@ compile-flags: --test --persist-doctests /../../ -Z unstable-options
-//@ failure-status: 101
-//@ only-linux
-
-#![crate_name = "foo"]
-
-//! ```rust
-//! use foo::dummy;
-//! dummy();
-//! ```
diff --git a/tests/rustdoc-ui/issues/issue-98690.stderr b/tests/rustdoc-ui/issues/issue-98690.stderr
deleted file mode 100644
index 47d94f99a4563..0000000000000
--- a/tests/rustdoc-ui/issues/issue-98690.stderr
+++ /dev/null
@@ -1 +0,0 @@
-Couldn't create directory for doctest executables: Permission denied (os error 13)

From b009d5c2b0fbe77056ac68cabefe307191a5bfa7 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 12 Dec 2024 15:33:18 +0100
Subject: [PATCH 13/37] add buildspec for troubleshooting

---
 buildspec.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 buildspec.yml

diff --git a/buildspec.yml b/buildspec.yml
new file mode 100644
index 0000000000000..d541f8430f262
--- /dev/null
+++ b/buildspec.yml
@@ -0,0 +1,11 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+      - echo "Starting sleep for troubleshooting..."
+      - sleep 3600  # Sleeps for 1 hour
+
+  post_build:
+    commands:
+      - echo "Build completed"

From c3ed253040b4ea2dcb78cd1c1b9ea87eda2ba7d8 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:49:25 +0100
Subject: [PATCH 14/37] don't run container as root

---
 src/ci/docker/run.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index d554186df4cfe..710f014db7d4a 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -282,8 +282,12 @@ else
     # LOCAL_USER_ID will map to a different subuid range on the host.
     # The "keep-id" mode maps the current UID directly into the container.
     args="$args --env NO_CHANGE_USER=1 --userns=keep-id"
-  else
+  else if [[ "$id" != 0 ]]; then
     args="$args --env LOCAL_USER_ID=$id"
+  else
+    # If we're running as root, we don't want to run the container as root,
+    # so we set id `1001` instead of `0`.
+    args="$args --env LOCAL_USER_ID=1001"
   fi
 fi
 

From 3c704790b562e03b923253a00593c0c880e408a0 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:50:06 +0100
Subject: [PATCH 15/37] Revert "add buildspec for troubleshooting"

This reverts commit b009d5c2b0fbe77056ac68cabefe307191a5bfa7.
---
 buildspec.yml | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 buildspec.yml

diff --git a/buildspec.yml b/buildspec.yml
deleted file mode 100644
index d541f8430f262..0000000000000
--- a/buildspec.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-version: 0.2
-
-phases:
-  build:
-    commands:
-      - echo "Starting sleep for troubleshooting..."
-      - sleep 3600  # Sleeps for 1 hour
-
-  post_build:
-    commands:
-      - echo "Build completed"

From 6aab2241957d7e8375e8c0e756590cb8625b5455 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:50:20 +0100
Subject: [PATCH 16/37] Revert "exclude problematic test"

This reverts commit 3f1e15646f09849a68715e6e242380b83ece974d.
---
 tests/rustdoc-ui/issues/issue-98690.rs     | 10 ++++++++++
 tests/rustdoc-ui/issues/issue-98690.stderr |  1 +
 2 files changed, 11 insertions(+)
 create mode 100644 tests/rustdoc-ui/issues/issue-98690.rs
 create mode 100644 tests/rustdoc-ui/issues/issue-98690.stderr

diff --git a/tests/rustdoc-ui/issues/issue-98690.rs b/tests/rustdoc-ui/issues/issue-98690.rs
new file mode 100644
index 0000000000000..01708f3f64d3e
--- /dev/null
+++ b/tests/rustdoc-ui/issues/issue-98690.rs
@@ -0,0 +1,10 @@
+//@ compile-flags: --test --persist-doctests /../../ -Z unstable-options
+//@ failure-status: 101
+//@ only-linux
+
+#![crate_name = "foo"]
+
+//! ```rust
+//! use foo::dummy;
+//! dummy();
+//! ```
diff --git a/tests/rustdoc-ui/issues/issue-98690.stderr b/tests/rustdoc-ui/issues/issue-98690.stderr
new file mode 100644
index 0000000000000..47d94f99a4563
--- /dev/null
+++ b/tests/rustdoc-ui/issues/issue-98690.stderr
@@ -0,0 +1 @@
+Couldn't create directory for doctest executables: Permission denied (os error 13)

From 8bdc25e174be896eedaa9c4af060a9b6fa6d8f05 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:51:07 +0100
Subject: [PATCH 17/37] don't install docker

---
 src/ci/scripts/ubuntu-codebuild.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ci/scripts/ubuntu-codebuild.sh b/src/ci/scripts/ubuntu-codebuild.sh
index fefb28a58f249..da5371965de46 100644
--- a/src/ci/scripts/ubuntu-codebuild.sh
+++ b/src/ci/scripts/ubuntu-codebuild.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
-apt-get update
-apt-get install -y docker-buildx
+# apt-get update
+# apt-get install -y docker-buildx

From a59528536c8ed3ab6c1efd69d4d178cfbabd2cd3 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:58:24 +0100
Subject: [PATCH 18/37] fix syntax

---
 src/ci/docker/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 710f014db7d4a..82f17f82261b8 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -282,7 +282,7 @@ else
     # LOCAL_USER_ID will map to a different subuid range on the host.
     # The "keep-id" mode maps the current UID directly into the container.
     args="$args --env NO_CHANGE_USER=1 --userns=keep-id"
-  else if [[ "$id" != 0 ]]; then
+  elif [[ "$id" != 0 ]]; then
     args="$args --env LOCAL_USER_ID=$id"
   else
     # If we're running as root, we don't want to run the container as root,

From 2550cc35f0013905de76d60b5ac4aa3c312ea663 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:53:17 +0100
Subject: [PATCH 19/37] debug

---
 src/ci/scripts/run-build-from-ci.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh
index 55e75800d91c4..90d6894c7c234 100755
--- a/src/ci/scripts/run-build-from-ci.sh
+++ b/src/ci/scripts/run-build-from-ci.sh
@@ -17,7 +17,9 @@ echo "::add-matcher::src/ci/github-actions/problem_matchers.json"
 # the environment
 rustup self uninstall -y || true
 if [ -z "${IMAGE+x}" ]; then
+    echo "Running ci/run.sh"
     src/ci/run.sh
 else
+    echo "Running docker/run.sh with image ${IMAGE}"
     src/ci/docker/run.sh "${IMAGE}"
 fi

From b1e84339a23ae89bc6f7cd12306e5262466a2319 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 13 Dec 2024 14:19:59 +0100
Subject: [PATCH 20/37] debug

---
 src/ci/run.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index b874f71832d73..d79ae911a7b79 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -22,6 +22,7 @@ if [ "$NO_CHANGE_USER" = "" ]; then
     echo -e '[safe]\n\tdirectory = *' > /home/user/.gitconfig
 
     exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user
+    echo "whoami: $(whoami)"
   fi
 fi
 

From 8f92115668c97e48d41c920c771f71a802bd573d Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 13 Dec 2024 16:39:49 +0100
Subject: [PATCH 21/37] add debug info

---
 src/ci/run.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index d79ae911a7b79..99dc2e6e6a009 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -2,12 +2,15 @@
 
 set -e
 
+echo "Running inside src/ci/run.sh script"
+
 if [ -n "$CI_JOB_NAME" ]; then
   echo "[CI_JOB_NAME=$CI_JOB_NAME]"
 fi
 
 if [ "$NO_CHANGE_USER" = "" ]; then
   if [ "$LOCAL_USER_ID" != "" ]; then
+    echo "Starting with UID: $LOCAL_USER_ID"
     id -u user &>/dev/null || useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user
     export HOME=/home/user
     unset LOCAL_USER_ID
@@ -21,6 +24,7 @@ if [ "$NO_CHANGE_USER" = "" ]; then
     # For NO_CHANGE_USER done in the small number of Dockerfiles affected.
     echo -e '[safe]\n\tdirectory = *' > /home/user/.gitconfig
 
+    echo "Switching to user"
     exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user
     echo "whoami: $(whoami)"
   fi

From bd5ad7ac5a2a41b0d8d8a879cfbb4498cd190e6b Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Mon, 16 Dec 2024 14:42:05 +0100
Subject: [PATCH 22/37] debug

---
 src/ci/run.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 99dc2e6e6a009..e4421bf8ed61e 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -8,7 +8,9 @@ if [ -n "$CI_JOB_NAME" ]; then
   echo "[CI_JOB_NAME=$CI_JOB_NAME]"
 fi
 
+echo "NO_CHANGE_USER=$NO_CHANGE_USER. LOCAL_USER_ID=$LOCAL_USER_ID"
 if [ "$NO_CHANGE_USER" = "" ]; then
+  echo "can change user"
   if [ "$LOCAL_USER_ID" != "" ]; then
     echo "Starting with UID: $LOCAL_USER_ID"
     id -u user &>/dev/null || useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user

From 0c4f7c1c1c15d88e7331597707f1767aa400e79c Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Mon, 16 Dec 2024 19:35:31 +0100
Subject: [PATCH 23/37] debug

---
 src/ci/docker/run.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 82f17f82261b8..e92b7c6eb3317 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -281,10 +281,13 @@ else
     # Rootless podman creates a separate user namespace, where an inner
     # LOCAL_USER_ID will map to a different subuid range on the host.
     # The "keep-id" mode maps the current UID directly into the container.
+    echo "Running in rootless podman"
     args="$args --env NO_CHANGE_USER=1 --userns=keep-id"
   elif [[ "$id" != 0 ]]; then
+    echo "Running in docker as non-root"
     args="$args --env LOCAL_USER_ID=$id"
   else
+    echo "Running in docker as root. Using id 1001."
     # If we're running as root, we don't want to run the container as root,
     # so we set id `1001` instead of `0`.
     args="$args --env LOCAL_USER_ID=1001"

From 1559d9dc512433c9eec6def171a0c7f9999a1eac Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Tue, 17 Dec 2024 16:08:38 +0100
Subject: [PATCH 24/37] debug

---
 src/ci/docker/run.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index e92b7c6eb3317..fbf1354281608 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -39,6 +39,7 @@ root_dir="`dirname $src_dir`"
 source "$ci_dir/shared.sh"
 
 if isCI; then
+    echo "CI detected"
     objdir=$root_dir/obj
 else
     objdir=$root_dir/obj/$image
@@ -53,6 +54,7 @@ fi
 CACHE_DOMAIN="${CACHE_DOMAIN:-ci-caches.rust-lang.org}"
 
 if [ -f "$docker_dir/$image/Dockerfile" ]; then
+    echo "Dockerfile found for $image"
     hash_key=/tmp/.docker-hash-key.txt
     rm -f "${hash_key}"
     echo $image >> $hash_key
@@ -136,6 +138,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
           "$context"
     # On auto/try builds, we can also write to the cache.
     else
+        echo "Logging into the Docker registry"
         # Log into the Docker registry, so that we can read/write cache and the final image
         echo ${DOCKER_TOKEN} | docker login ${REGISTRY} \
             --username ${REGISTRY_USERNAME} \
@@ -144,6 +147,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
         # Enable a new Docker driver so that --cache-from/to works with a registry backend
         docker buildx create --use --driver docker-container
 
+        echo "Building Docker image with cache"
         # Build the image using registry caching backend
         retry docker \
           buildx \
@@ -156,11 +160,13 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
           --output=type=docker \
           "$context"
 
+        echo "Docker image built"
         # Print images for debugging purposes
         docker images
 
         # Tag the built image and push it to the registry
         docker tag rust-ci "${IMAGE_TAG}"
+        echo "Pushing Docker image to the registry"
         docker push "${IMAGE_TAG}"
 
         # Record the container registry tag/url for reuse, e.g. by rustup.rs builds
@@ -212,6 +218,7 @@ else
     exit 1
 fi
 
+echo "Creating directories"
 mkdir -p $HOME/.cargo
 mkdir -p $objdir/tmp
 mkdir -p $objdir/cores
@@ -267,6 +274,7 @@ args="$args --privileged"
 # `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all
 # read/written as the same user as the bare-metal user.
 if [ -f /.dockerenv ]; then
+  echo "Dockerenv detected"
   docker create -v /checkout --name checkout alpine:3.4 /bin/true
   docker cp . checkout:/checkout
   args="$args --volumes-from checkout"

From 1200473f051b30afa9c56b976190b4c54fb9a992 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 18 Dec 2024 09:58:46 +0100
Subject: [PATCH 25/37] change user in docker-in-docker

---
 src/ci/docker/run.sh | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index fbf1354281608..4bbb5506ccd7e 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -274,7 +274,7 @@ args="$args --privileged"
 # `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all
 # read/written as the same user as the bare-metal user.
 if [ -f /.dockerenv ]; then
-  echo "Dockerenv detected"
+  echo "Dockerenv detected. We are in docker-in-docker scenario."
   docker create -v /checkout --name checkout alpine:3.4 /bin/true
   docker cp . checkout:/checkout
   args="$args --volumes-from checkout"
@@ -283,23 +283,23 @@ else
   args="$args --volume $objdir:/checkout/obj"
   args="$args --volume $HOME/.cargo:/cargo"
   args="$args --volume /tmp/toolstate:/tmp/toolstate"
+fi
 
-  id=$(id -u)
-  if [[ "$id" != 0 && "$(docker version)" =~ Podman ]]; then
-    # Rootless podman creates a separate user namespace, where an inner
-    # LOCAL_USER_ID will map to a different subuid range on the host.
-    # The "keep-id" mode maps the current UID directly into the container.
-    echo "Running in rootless podman"
-    args="$args --env NO_CHANGE_USER=1 --userns=keep-id"
-  elif [[ "$id" != 0 ]]; then
-    echo "Running in docker as non-root"
-    args="$args --env LOCAL_USER_ID=$id"
-  else
-    echo "Running in docker as root. Using id 1001."
-    # If we're running as root, we don't want to run the container as root,
-    # so we set id `1001` instead of `0`.
-    args="$args --env LOCAL_USER_ID=1001"
-  fi
+id=$(id -u)
+if [[ "$id" != 0 && "$(docker version)" =~ Podman ]]; then
+  # Rootless podman creates a separate user namespace, where an inner
+  # LOCAL_USER_ID will map to a different subuid range on the host.
+  # The "keep-id" mode maps the current UID directly into the container.
+  echo "Running in rootless podman"
+  args="$args --env NO_CHANGE_USER=1 --userns=keep-id"
+elif [[ "$id" != 0 ]]; then
+  echo "Running in docker as non-root"
+  args="$args --env LOCAL_USER_ID=$id"
+else
+  echo "Running in docker as root. Using id 1001."
+  # If we're running as root, we don't want to run the container as root,
+  # so we set id `1001` instead of `0`.
+  args="$args --env LOCAL_USER_ID=1001"
 fi
 
 if [ "$dev" = "1" ]

From 6b9ed616c96e4204128f9ecc22292f54f19e2fb3 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 20 Dec 2024 16:58:44 +0100
Subject: [PATCH 26/37] run whoami before switching user

---
 src/ci/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index e4421bf8ed61e..31b78b39c3376 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -8,6 +8,7 @@ if [ -n "$CI_JOB_NAME" ]; then
   echo "[CI_JOB_NAME=$CI_JOB_NAME]"
 fi
 
+echo "whoami: $(whoami)"
 echo "NO_CHANGE_USER=$NO_CHANGE_USER. LOCAL_USER_ID=$LOCAL_USER_ID"
 if [ "$NO_CHANGE_USER" = "" ]; then
   echo "can change user"
@@ -28,7 +29,6 @@ if [ "$NO_CHANGE_USER" = "" ]; then
 
     echo "Switching to user"
     exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user
-    echo "whoami: $(whoami)"
   fi
 fi
 

From 8ac61b08ce8b0255e508b9b60fb21606dac9b99f Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 20 Dec 2024 17:02:08 +0100
Subject: [PATCH 27/37] print home

---
 src/ci/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 31b78b39c3376..251912ab7a75a 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -8,7 +8,7 @@ if [ -n "$CI_JOB_NAME" ]; then
   echo "[CI_JOB_NAME=$CI_JOB_NAME]"
 fi
 
-echo "whoami: $(whoami)"
+echo "whoami: $(whoami). Home: $HOME"
 echo "NO_CHANGE_USER=$NO_CHANGE_USER. LOCAL_USER_ID=$LOCAL_USER_ID"
 if [ "$NO_CHANGE_USER" = "" ]; then
   echo "can change user"

From a9b8ea4bc2547abbdf88c7bf9dd63d4ffe0460ab Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 20 Dec 2024 17:03:25 +0100
Subject: [PATCH 28/37] debug

---
 src/ci/run.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 251912ab7a75a..9f0efa15d3a77 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -9,6 +9,10 @@ if [ -n "$CI_JOB_NAME" ]; then
 fi
 
 echo "whoami: $(whoami). Home: $HOME"
+echo "--- current directory ---"
+pwd
+ls -l
+echo "-------------------------"
 echo "NO_CHANGE_USER=$NO_CHANGE_USER. LOCAL_USER_ID=$LOCAL_USER_ID"
 if [ "$NO_CHANGE_USER" = "" ]; then
   echo "can change user"

From 5e3e5b999a6ef5773921a9a29cd7a234e9f38e37 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 20 Dec 2024 17:27:15 +0100
Subject: [PATCH 29/37] fix

---
 src/ci/run.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 9f0efa15d3a77..2d12456ac8e90 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -22,6 +22,9 @@ if [ "$NO_CHANGE_USER" = "" ]; then
     export HOME=/home/user
     unset LOCAL_USER_ID
 
+    # Give ownership of the current directory to the user
+    chown -R user .
+
     # Ensure that runners are able to execute git commands in the worktree,
     # overriding the typical git protections. In our docker container we're running
     # as root, while the user owning the checkout is not root.

From 09542449492df1815f8fbdaaed51c80865a00102 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 20 Dec 2024 17:42:00 +0100
Subject: [PATCH 30/37] fix

---
 src/ci/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 2d12456ac8e90..61aad1f94c513 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -23,7 +23,7 @@ if [ "$NO_CHANGE_USER" = "" ]; then
     unset LOCAL_USER_ID
 
     # Give ownership of the current directory to the user
-    chown -R user .
+    chown -R $LOCAL_USER_ID:$LOCAL_USER_ID .
 
     # Ensure that runners are able to execute git commands in the worktree,
     # overriding the typical git protections. In our docker container we're running

From 0b2fb0a7b2bd54c09901b4fd2154656d02a91969 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Fri, 20 Dec 2024 17:56:17 +0100
Subject: [PATCH 31/37] try fix issue

---
 src/ci/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 61aad1f94c513..323e14d351193 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -23,7 +23,7 @@ if [ "$NO_CHANGE_USER" = "" ]; then
     unset LOCAL_USER_ID
 
     # Give ownership of the current directory to the user
-    chown -R $LOCAL_USER_ID:$LOCAL_USER_ID .
+    chown -R user:user .
 
     # Ensure that runners are able to execute git commands in the worktree,
     # overriding the typical git protections. In our docker container we're running

From dbee4e0ef0f5c7bb99b738663b32262c4fa2eb2c Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 8 Jan 2025 12:45:09 +0100
Subject: [PATCH 32/37] remove codebuild script

---
 .github/workflows/ci.yml           | 4 ----
 src/ci/scripts/ubuntu-codebuild.sh | 4 ----
 2 files changed, 8 deletions(-)
 delete mode 100644 src/ci/scripts/ubuntu-codebuild.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9d9865db74af0..fa2055ad262c2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -145,10 +145,6 @@ jobs:
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
 
-      - if: contains(matrix.os, 'codebuild-ubuntu')
-        name: Install sotftware for codebuild
-        run: chmod +x src/ci/scripts/ubuntu-codebuild.sh && ./src/ci/scripts/ubuntu-codebuild.sh
-
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
 
diff --git a/src/ci/scripts/ubuntu-codebuild.sh b/src/ci/scripts/ubuntu-codebuild.sh
deleted file mode 100644
index da5371965de46..0000000000000
--- a/src/ci/scripts/ubuntu-codebuild.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-# apt-get update
-# apt-get install -y docker-buildx

From 156ea59deb5a7706124683cc0bffc8ad40985f91 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 8 Jan 2025 12:55:43 +0100
Subject: [PATCH 33/37] debug prints

---
 src/bootstrap/bootstrap.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index 89415afbe3bc4..f7a8367edf14d 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -1002,6 +1002,9 @@ def build_bootstrap(self):
         env = os.environ.copy()
         if "GITHUB_ACTIONS" in env:
             print("::group::Building bootstrap")
+            print("Current home directory:", os.path.expanduser("~"))
+            print("Current working directory:", os.getcwd())
+            print("Current HOME:", env["HOME"])
         else:
             eprint("Building bootstrap")
 

From f249d32f965d2c8e13ca4292be339e2f84c69376 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 8 Jan 2025 14:08:58 +0100
Subject: [PATCH 34/37] debug

---
 src/bootstrap/bootstrap.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index f7a8367edf14d..999f77786c341 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -1009,6 +1009,7 @@ def build_bootstrap(self):
             eprint("Building bootstrap")
 
         args = self.build_bootstrap_cmd(env)
+        print("Running", args, "in", self.rust_root)
         # Run this from the source directory so cargo finds .cargo/config
         run(args, env=env, verbose=self.verbose, cwd=self.rust_root)
 
@@ -1018,6 +1019,7 @@ def build_bootstrap(self):
     def build_bootstrap_cmd(self, env):
         """For tests."""
         build_dir = os.path.join(self.build_dir, "bootstrap")
+        print("Building bootstrap in", build_dir)
         if self.clean and os.path.exists(build_dir):
             shutil.rmtree(build_dir)
         # `CARGO_BUILD_TARGET` breaks bootstrap build.

From 031c531ef3fc0ab30bbd90a7ce256c8c2d2ab275 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 8 Jan 2025 14:18:04 +0100
Subject: [PATCH 35/37] fmt

---
 src/ci/github-actions/calculate-job-matrix.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
index f3c9a9311c914..7bfd08827353d 100755
--- a/src/ci/github-actions/calculate-job-matrix.py
+++ b/src/ci/github-actions/calculate-job-matrix.py
@@ -186,12 +186,10 @@ def substitute_github_vars(jobs: list) -> list:
     """Replace GitHub context variables with environment variables in job configs."""
     for job in jobs:
         if "os" in job:
-            job["os"] = job["os"].replace(
-                "${{ github.run_id }}",
-                os.environ["GITHUB_RUN_ID"]
-            ).replace(
-                "${{ github.run_attempt }}",
-                os.environ["GITHUB_RUN_ATTEMPT"]
+            job["os"] = (
+                job["os"]
+                .replace("${{ github.run_id }}", os.environ["GITHUB_RUN_ID"])
+                .replace("${{ github.run_attempt }}", os.environ["GITHUB_RUN_ATTEMPT"])
             )
     return jobs
 
@@ -216,7 +214,6 @@ def substitute_github_vars(jobs: list) -> list:
     jobs = skip_jobs(jobs, channel)
     jobs = substitute_github_vars(jobs)
 
-
     if not jobs:
         raise Exception("Scheduled job list is empty, this is an error")
 

From ad17cb785d518dee124bef2516532709a0793c0c Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 8 Jan 2025 15:01:43 +0100
Subject: [PATCH 36/37] fix

---
 src/ci/run.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 323e14d351193..4260443139b1b 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -22,8 +22,9 @@ if [ "$NO_CHANGE_USER" = "" ]; then
     export HOME=/home/user
     unset LOCAL_USER_ID
 
-    # Give ownership of the current directory to the user
+    # Give ownership of necessary directories to the user
     chown -R user:user .
+    chown -R user:user /cargo
 
     # Ensure that runners are able to execute git commands in the worktree,
     # overriding the typical git protections. In our docker container we're running

From 30c160004933062ff25cb59d6e3709ce4ded67f4 Mon Sep 17 00:00:00 2001
From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com>
Date: Wed, 8 Jan 2025 15:47:14 +0100
Subject: [PATCH 37/37] fix

---
 src/ci/run.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 4260443139b1b..1ca05fddf15b6 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -24,6 +24,7 @@ if [ "$NO_CHANGE_USER" = "" ]; then
 
     # Give ownership of necessary directories to the user
     chown -R user:user .
+    mkdir /cargo
     chown -R user:user /cargo
 
     # Ensure that runners are able to execute git commands in the worktree,