-
Notifications
You must be signed in to change notification settings - Fork 323
213 lines (186 loc) · 8.56 KB
/
push-main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
name: Build from Main
on:
push:
branches:
- main
jobs:
create-runner:
uses: gitpod-io/gce-github-runner/.github/workflows/create-vm.yml@main
secrets:
runner_token: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_TOKEN }}
gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}
concurrency:
group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-create-runner
cancel-in-progress: false
# Build images using artifactory as image registry.
# To implement manual approvals, the workflow uses an Environment.
#
# From your GitHub repo click Settings. In the left menu, click Environments.
# Click New environment, set the name production, and click Configure environment.
# Check the "Required reviewers" box and enter at least one user or team name.
sync:
runs-on: ${{ needs.create-runner.outputs.label }}
needs: create-runner
concurrency:
group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-sync
cancel-in-progress: true
environment: "production"
permissions:
contents: "read"
id-token: "write"
env:
WORKLOAD_IDENTITY_POOL_ID: projects/665270063338/locations/global/workloadIdentityPools/workspace-images-github-actions/providers/workspace-images-gha-provider
GAR_IMAGE_REGISTRY: europe-docker.pkg.dev
DH_IMAGE_REGISTRY: registry.hub.docker.com
IAM_SERVICE_ACCOUNT: workspace-images-gha-sa@gitpod-artifacts.iam.gserviceaccount.com
DAZZLE_VERSION: 0.1.17
BUILDKIT_VERSION: 0.12.3
SKOPEO_VERSION: 1.15.2
steps:
- name: 📥 Checkout workspace-images
uses: actions/checkout@v4
with:
repository: gitpod-io/workspace-images
- name: 🔧 Setup tools
run: |
sudo apt-get install --yes python3-pip shellcheck
curl -sSL https://github.com/mvdan/sh/releases/download/v3.5.0/shfmt_v3.5.0_linux_amd64 -o shfmt
sudo mv shfmt /usr/local/bin/shfmt && sudo chmod +x /usr/local/bin/shfmt
sudo pip3 install pre-commit
- name: 🤓 Run pre-commit
run: |
pre-commit run --all-files
- name: 🔆 Install dazzle
env:
DAZZLE_VERSION: ${{env.DAZZLE_VERSION}}
run: |
curl -sSL "https://github.com/gitpod-io/dazzle/releases/download/v${DAZZLE_VERSION}/dazzle_${DAZZLE_VERSION}_Linux_x86_64.tar.gz" | sudo tar -xvz -C /usr/local/bin
- name: 🕰️ Create timestamp tag
id: create-timestamp-tag
run: |
echo "TIMESTAMP_TAG=$(date '+%Y-%m-%d-%H-%M-%S')" >> $GITHUB_ENV
- name: 🔆 Install skopeo
env:
SKOPEO_VERSION: ${{env.SKOPEO_VERSION}}
run: |
# Generate a temporal file to store skopeo auth
# Any step using skopeo needs SKOPEO_AUTH_DIR env var
SKOPEO_AUTH_DIR=$(mktemp -d)
# to test locally
# export GITHUB_ENV=$(mktemp)
echo "SKOPEO_AUTH_DIR=${SKOPEO_AUTH_DIR}" >> $GITHUB_ENV
# Any step using skopeo needs SKOPEO_SYNC_FILES env var
SKOPEO_SYNC_FILES=$(mktemp -d)
echo "SKOPEO_SYNC_FILES=${SKOPEO_SYNC_FILES}" >> $GITHUB_ENV
# limit what we push as time stamped images to images built in the current year and month only
current_year=$(date +%Y)
current_month=$(date +%m)
sed -i "s/TIMESTAMP_TAG/$TIMESTAMP_TAG/g" "${GITHUB_WORKSPACE}/.github/promote-images.yml"
cp "${GITHUB_WORKSPACE}/.github/promote-images.yml" "${SKOPEO_SYNC_FILES}"
# Build a fake skopeo script to run a container
cat <<EOF | sudo tee /usr/local/bin/skopeo > /dev/null
#/bin/bash
docker run --rm \
-v "${SKOPEO_AUTH_DIR}":/skopeo.auth \
-v "${SKOPEO_SYNC_FILES}":/.github \
-e REGISTRY_AUTH_FILE=/skopeo.auth/auth \
quay.io/skopeo/stable:v"${SKOPEO_VERSION}" "\$@"
EOF
sudo chmod +x /usr/local/bin/skopeo
# don't fail parsing the file while it contains empty creds the first time
echo "{}" > $SKOPEO_AUTH_DIR/auth
- name: 🏗️ Setup buildkit
env:
BUILDKIT_VERSION: ${{env.BUILDKIT_VERSION}}
run: |
curl -sSL "https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/buildkit-v${BUILDKIT_VERSION}.linux-amd64.tar.gz" | sudo tar xvz -C /usr
sudo buildkitd --oci-worker=true --oci-worker-net=host --debug --group docker &
sudo su -c "while ! test -S /run/buildkit/buildkitd.sock; do sleep 0.1; done"
sudo chmod +777 /run/buildkit/buildkitd.sock
- name: ☁️ Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2.1.2
with:
version: 393.0.0
- name: 🔐 Authenticate to Google Cloud
id: "auth"
uses: google-github-actions/auth@v2.1.7
with:
token_format: "access_token"
access_token_lifetime: "43200s"
workload_identity_provider: ${{env.WORKLOAD_IDENTITY_POOL_ID}}
service_account: ${{env.IAM_SERVICE_ACCOUNT}}
- name: ✍🏽 Login to GAR using skopeo
env:
SKOPEO_AUTH_DIR: ${{env.SKOPEO_AUTH_DIR}}
GAR_IMAGE_REGISTRY: ${{env.GAR_IMAGE_REGISTRY}}
SKOPEO_SYNC_FILES: ${env.SKOPEO_SYNC_FILES}
run: |
sudo -E skopeo login -u oauth2accesstoken --password=${{ steps.auth.outputs.access_token }} $GAR_IMAGE_REGISTRY
- name: ✍🏽 Login to GAR using docker cli
env:
GAR_IMAGE_REGISTRY: ${{env.GAR_IMAGE_REGISTRY}}
run: |
docker login -u oauth2accesstoken --password=${{ steps.auth.outputs.access_token }} $GAR_IMAGE_REGISTRY
- name: 🔨 Dazzle build
env:
GAR_IMAGE_REGISTRY: ${{env.GAR_IMAGE_REGISTRY}}
run: |
dazzle build "${GAR_IMAGE_REGISTRY}/gitpod-artifacts/docker-dev/workspace-base-images" --chunked-without-hash
dazzle build "${GAR_IMAGE_REGISTRY}/gitpod-artifacts/docker-dev/workspace-base-images"
- name: 🖇️ Dazzle combine
env:
GAR_IMAGE_REGISTRY: ${{env.GAR_IMAGE_REGISTRY}}
run: |
dazzle combine "${GAR_IMAGE_REGISTRY}/gitpod-artifacts/docker-dev/workspace-base-images" --all
- name: 🔧 Setup copy tools
run: |
# this installs yq 3
sudo pip3 install yq
- name: 📋 Copy images with tag in the Artifact Registry
env:
SKOPEO_AUTH_DIR: ${{env.SKOPEO_AUTH_DIR}}
GAR_IMAGE_REGISTRY: ${{env.GAR_IMAGE_REGISTRY}}
TIMESTAMP_TAG: ${{env.TIMESTAMP_TAG}}
SKOPEO_SYNC_FILES: ${{env.SKOPEO_SYNC_FILES}}
run: |
set -euo pipefail
IFS=$'\n' read -r -d '' -a IMAGE_TAGS_ARR < <(yq -r '.sync.images."workspace-base-images"[]' ./.github/sync-containers.yml && printf '\0')
UPLOADS_STRING=""
for IMAGE_TAG in "${IMAGE_TAGS_ARR[@]}"; do
UPLOADS_STRING+=$(printf "./.github/workflows/push-main/upload_image.sh %s%s" ${IMAGE_TAG}'\n')
done
# remove the trailing newline
UPLOADS_STRING="${UPLOADS_STRING::-2}"
echo -e "${UPLOADS_STRING}" | xargs -I CMD -P 10 -n 1 \
env SKOPEO_AUTH_DIR="${SKOPEO_AUTH_DIR}" GAR_IMAGE_REGISTRY="${GAR_IMAGE_REGISTRY}" TIMESTAMP_TAG="${TIMESTAMP_TAG}" SKOPEO_SYNC_FILES="${SKOPEO_SYNC_FILES}" \
bash -c CMD --
- name: ✍🏽 Login to Docker Hub using skopeo
env:
DOCKERHUB_USER_NAME: ${{secrets.DOCKERHUB_USER_NAME}}
DOCKERHUB_ACCESS_TOKEN: ${{secrets.DOCKERHUB_ACCESS_TOKEN}}
DH_IMAGE_REGISTRY: ${{env.DH_IMAGE_REGISTRY}}
SKOPEO_SYNC_FILES: ${{env.SKOPEO_SYNC_FILES}}
run: |
sudo -E skopeo login -u "${DOCKERHUB_USER_NAME}" --password="${DOCKERHUB_ACCESS_TOKEN}" "${DH_IMAGE_REGISTRY}"
- name: 🐳 Sync images with specific tags to Docker Hub
env:
DH_IMAGE_REGISTRY: ${{env.DH_IMAGE_REGISTRY}}
SKOPEO_SYNC_FILES: ${{env.SKOPEO_SYNC_FILES}}
run: |
sudo -E skopeo sync \
--src yaml \
--retry-times=2 \
--keep-going \
--dest docker \
/.github/promote-images.yml "${DH_IMAGE_REGISTRY}/gitpod"
delete-runner:
if: always()
needs:
- create-runner
- sync
uses: gitpod-io/gce-github-runner/.github/workflows/delete-vm.yml@main
secrets:
gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}
with:
runner-label: ${{ needs.create-runner.outputs.label }}
machine-zone: ${{ needs.create-runner.outputs.machine-zone }}