wkdev_sdk_bots: Add first Containerfile for a specific bot: wpe-linux…

…-bot-37

The goal is to have a single image, that upon startup, acts as a
buildbot, fully configured. See NOTES file for details.

images/buildbot_wpe_linux_bot_37

@@ -0,0 +1 @@
+wkdev_sdk_bots

images/wkdev_sdk_bots/Containerfile

@@ -1,163 +1,28 @@
 # Copyright 2024 Igalia S.L.
 # SPDX-License: MIT
 
-FROM ubuntu:24.04
+FROM ghcr.io/igalia/wkdev-sdk:latest
 
 LABEL maintainer="webkit-gtk@lists.webkit.org"
 LABEL version="1.0"
-LABEL org.opencontainers.image.title="WebKit SDK"
-LABEL org.opencontainers.image.description="Provides a complete WebKit Gtk/WPE development environment based on Ubuntu 24.04"
+LABEL org.opencontainers.image.title="WebKit SDK Bots"
+LABEL org.opencontainers.image.description="Provides a complete WebKit Gtk/WPE environment to run buildbot"
 LABEL org.opencontainers.image.source=https://github.com/Igalia/wkdev-sdk
 
-# Tweakable "make -j <x>" setting.
-ARG NUMBER_OF_PARALLEL_BUILDS=4
-ARG CONTAINER_LOCALE=en_US.UTF-8
+# Checkout buildbot EWS, deactivate 'reload_systemd' step of deploy.sh
+RUN cd /root && \
+    git clone https://${DEPLOY_WEBKIT_BUILDBOT_EWS_USER}:${DEPLOY_WEBKIT_BUILDBOT_EWS_PASSWORD}@gitlab.igalia.com/teams/webkit/deploy-webkit-buildbot-ews.git && \
+    cd deploy-webkit-buildbot-ews && \
+    sed -i -e s/^reload_systemd$// deploy.sh && \
+    ./deploy.sh --type ${BOT_TYPE} --port ${BOT_PORT}
 
-# No need to modify these.
-ARG APT_UPDATE="apt-get update"
-ARG APT_BUILDDEP="apt-get --assume-yes build-dep"
-ARG APT_UPGRADE="apt-get --assume-yes upgrade"
-ARG APT_INSTALL="apt-get --assume-yes install --no-install-recommends"
-ARG APT_AUTOREMOVE="apt-get --assume-yes autoremove"
+COPY ./config/${BOT_TYPE}.tac.template /home/${BOT_TYPE}/worker/${BOT_TYPE}.tac
 
-# Disable prompt during package configuration
-ENV DEBIAN_FRONTEND noninteractive
+RUN sed -i -e "s/#WORKERNAME#/${BOT_WORKERNAME}/" /home/${BOT_TYPE}/worker/${BOT_TYPE}.tac && \
+    sed -i -e "s/#PASSWORD#/${BOT_PASSWORD}/" /home/${BOT_TYPE}/worker/${BOT_TYPE}.tac
 
-# Debian package build settings
-ENV DEBEMAIL "webkit-gtk@lists.webkit.org"
-ENV DEBFULLNAME "Igalia"
+RUN ln -s /etc/systemd/system/${BOT_TYPE}-worker@.service /etc/systemd/system/multi-user.target.wants/${BOT_TYPE}-worker@${BOT_TYPE}.service && \
+    mkdir -p /var/lib/systemd/linger && \
+    touch /var/lib/systemd/linger/${BOT_TYPE}
 
-# Enable debugging in WebKit's sandbox.
-ENV WEBKIT_ENABLE_DEBUG_PERMISSIONS_IN_SANDBOX "1"
-
-# Used in webkitdirs.pm to prefer building against system libraries instead of the Flatpak SDK.
-ENV WEBKIT_BUILD_USE_SYSTEM_LIBRARIES "1"
-
-# Delete the default ubuntu user which has a UID of 1000.
-# Podman refuses to map a user from the host if the UID is already in /etc/passwd.
-RUN userdel ubuntu
-
-# NOTE: All RUN commands contain the (autoremove / clean / rm step to ensure that no intermediate layer
-#       ever contains unncessary stuff that never appears in the final image, only in deeper layers, and
-#       thus increases the whole image size no gain, except an "easier to read" Dockerfile.
-
-# Disable sandboxing (dropping privileges to _apt user during apt-get update/install/... fails when using
-# podman in podman if both are rootless; since it's no gain in security in the container anyhow, disable it.
-RUN echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/no-sandbox
-
-# Update package list, upgrade to latest version, install necessary packages for
-# early bootstrapping: .deb package configuration + locale generation.
-RUN ${APT_UPDATE} && \
-    ${APT_INSTALL} apt-utils dialog libterm-readline-gnu-perl locales unminimize && \
-    ${APT_UPGRADE} && ${APT_AUTOREMOVE}
-
-# Disable exclusion of locales / translations / documentation (default in Ubuntu images)
-RUN yes | /usr/bin/unminimize
-
-# Switch to fixed locale.
-RUN locale-gen ${CONTAINER_LOCALE}
-ENV LC_ALL ${CONTAINER_LOCALE}
-ENV LANG ${CONTAINER_LOCALE}
-RUN dpkg-reconfigure locales
-
-# Install all dependencies for WebKit/GStreamer/etc in one pass.
-WORKDIR /var/tmp/wkdev-packages
-COPY /required_system_packages/*.lst .
-RUN sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources && \
-    ${APT_UPDATE} && \
-    for list in *.lst; do \
-        ${APT_INSTALL} $(sed -e "s/.*#.*//; /^$/d" "${list}"); \
-    done; \
-    ${APT_BUILDDEP} gst-libav1.0 gst-plugins-bad1.0 gst-plugins-base1.0 \
-	                gst-plugins-good1.0 gst-plugins-ugly1.0 && \
-    git clone --filter=blob:none --no-checkout --depth=1 https://github.com/WebKit/WebKit.git && \
-    cd WebKit && \
-    git sparse-checkout set Tools/ && \
-    git checkout main && \
-    yes | ./Tools/gtk/install-dependencies && \
-    yes | ./Tools/wpe/install-dependencies && \
-    cd .. && \
-    rm -rf WebKit && \
-    ${APT_AUTOREMOVE}
-
-# Add Rust environment.
-ENV RUSTUP_HOME="/opt/rust" \
-    CARGO_HOME="/opt/rust" \
-    PATH="/opt/rust/bin:${PATH}"
-
-RUN rustup default stable && \
-    rustup component remove rust-docs && \
-    cargo install --root /usr/local --version 0.8.1 --locked sccache && \
-    cargo install --root /usr/local cargo-c
-
-# Copy jhbuild helper files and do the initial build & install
-COPY /jhbuild/jhbuildrc /etc/xdg/jhbuildrc
-COPY /jhbuild/webkit-sdk-deps.modules /jhbuild/webkit-sdk-deps.modules
-COPY /jhbuild/patches/ /jhbuild/patches/
-WORKDIR /jhbuild
-RUN git clone https://gitlab.gnome.org/GNOME/jhbuild.git && \
-    cd jhbuild && \
-    ./autogen.sh --prefix=/usr/local && \
-    make && \
-    make install && \
-    export JHBUILD_RUN_AS_ROOT=1 WKDEV_IN_IMAGE_BUILD=1 && \
-    jhbuild --no-interact build && \
-    rm -r /var/tmp/jhbuild/build
-
-# Register basic JHBuild environment
-# TODO: Instead of hardcoding here the values it would be better to
-# explore the possibility of generating it dynamically with "jhbuild shell"
-# when the user enters into the container (or similar), but that may cause
-# issues with the env not exported when someone enter into the
-# container via direct command exec rather than by login
-ENV LIB "/jhbuild/install/lib"
-ENV INCLUDE "/jhbuild/install/include"
-ENV LD_LIBRARY_PATH "/jhbuild/install/lib"
-ENV GST_PLUGIN_PATH_1_0 "/jhbuild/install/lib/gstreamer-1.0"
-ENV PKG_CONFIG_PATH "/jhbuild/install/lib/pkgconfig:/jhbuild/install/share/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/share/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig"
-ENV LDFLAGS "-L/jhbuild/install/lib"
-ENV C_INCLUDE_PATH "/jhbuild/install/include"
-ENV CPLUS_INCLUDE_PATH "/jhbuild/install/include"
-ENV GI_TYPELIB_PATH "/jhbuild/install/lib/girepository-1.0"
-ENV XDG_DATA_DIRS "/jhbuild/install/share:/usr/local/share:/usr/share"
-ENV PATH "/jhbuild/install/bin:$PATH"
-
-# Podman proxy, connecting to host instance
-COPY /rootfs/usr/bin/podman-host /usr/bin/podman-host
-
-COPY /rootfs/etc/ccache.conf /etc/ccache.conf
-
-# Convenience symlink for clang tools, the VSCode extension doesn't find these by default.
-RUN for command in clang clang++ clangd clang-format clang-tidy lld lldb lldb-server lldb-vscode; do \
-        ln -s "/usr/bin/${command}-18" "/usr/local/bin/${command}"; \
-    done && ln -s "/usr/bin/lld-18" "/usr/local/bin/ld.lld";
-
-# Fix Qt6 system packages - missing symlinks in the Ubuntu-provided packages.
-RUN export QT_VERSION=$(qmake6 -query QT_VERSION) && \
-    for directory in /usr/include/x86_64-linux-gnu/qt6/*; do \
-      ln -s ${directory} ${directory}/${QT_VERSION}  >/dev/null 2>&1 || true; \
-    done
-
-# Check GStreamer plugins are installed.
-RUN gst-inspect-1.0 audiornnoise && \
-    gst-inspect-1.0 cea608tott && \
-    gst-inspect-1.0 dav1ddec && \
-    gst-inspect-1.0 livesync && \
-    gst-inspect-1.0 rsrtp
-
-# Remove systemd services that would startup by default, when spawning
-# systemd as PID 1 within the container (usually, we don't spawn systemd
-# within the wkdev-sdk container, for interactive usage, but we do so
-# when deploying the wkdev-sdk image as bot).
-RUN systemctl stop apache2.service && \
-    systemctl stop man-db.service && \
-    systemctl stop atop.service atopacct.service atop-rotate.timer && \
-    systemctl stop cups.path cups.service cups.socket
-
-RUN systemctl disable apache2.service && \
-    systemctl disable man-db.service && \
-    systemctl disable atop.service atopacct.service atop-rotate.timer && \
-    systemctl disable cups.path cups.service cups.socket
-
-# Switch back to interactive prompt, when using apt.
-ENV DEBIAN_FRONTEND dialog
+CMD [ "/sbin/init" ]

images/wkdev_sdk_bots/NOTES

@@ -0,0 +1,27 @@
+#1) Build wkdev-sdk container
+---------------------------------------------
+
+$ ${WKDEV_SDK}/scripts/host-only/wkdev-sdk-bakery --mode build --verbose
+
+#2) Build wkdev-sdk-bots container
+---------------------------------------------
+
+$ ${WKDEV_SDK}/scripts/host-only/wkdev-sdk-bakery --mode build --verbose --name buildbot-wpe-linux-bot-37 \
+  -e DEPLOY_WEBKIT_BUILDBOT_EWS_USER=deploy+gitclone \
+  -e DEPLOY_WEBKIT_BUILDBOT_EWS_PASSWORD=XXX \
+  -e BOT_WORKERNAME=wpe-linux-bot-37 \
+  -e BOT_PASSWORD="YYY" \
+  -e BOT_TYPE=buildbot \
+  -e BOT_PORT=wpe
+
+#3) Run new container in background
+---------------------------------------------
+
+$ podman run --device /dev/fuse --detach --name buildbot-wpe-linux-bot-37 ghcr.io/igalia/buildbot-wpe-linux-bot-37:latest
+
+#4) Enter container, check status
+---------------------------------------------
+
+$ podman exec -it buildbot-wpe-linux-bot-37 /bin/bash
+$ journalctl -u buildbot-worker@buildbot.service
+$ systemctl status

images/wkdev_sdk_bots/config/buildbot.tac.template

@@ -0,0 +1,50 @@
+import os
+
+from buildbot_worker.bot import Worker
+from twisted.application import service
+
+# Set here a custom basedir if you want
+basedir = '.'
+
+# if this is a relocatable tac file, get the directory containing the TAC
+if basedir == '.':
+    import os.path
+    basedir = os.path.abspath(os.path.dirname(__file__))
+
+# note: this line is matched against to check that this is a worker
+# directory; do not edit it.
+application = service.Application('buildbot-worker')
+
+# Uncomment below if you preffer to log to a file instead to stdout (systemd)
+#rotateLength = 10000000
+#maxRotatedFiles = 10
+#from twisted.python.logfile import LogFile
+#from twisted.python.log import ILogObserver, FileLogObserver
+#logfile = LogFile.fromFullPath(
+#    os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+#    maxRotatedFiles=maxRotatedFiles)
+#application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+
+# This is used for reporting the number of CPUs available to the master.
+if os.getenv("NUMBER_OF_PROCESSORS"):
+    numcpus = os.getenv("NUMBER_OF_PROCESSORS")
+else:
+    import multiprocessing
+    numcpus = multiprocessing.cpu_count()
+
+buildmaster_host = 'build-bots.webkit.org'
+port = 17000
+keepalive = 120
+umask = None
+maxdelay = 300
+allow_shutdown = None
+maxretries = None
+workername = '#WORKERNAME#'
+passwd = '#PASSWORD#'
+
+s = Worker(buildmaster_host, port, workername, passwd, basedir,
+           keepalive, umask=umask, maxdelay=maxdelay,
+           numcpus=numcpus, allow_shutdown=allow_shutdown,
+           maxRetries=maxretries)
+s.setServiceParent(application)