merge master

Signed-off-by: Chris Martin <chris@cmartinit.co.uk>

.github/workflows/airflow-operator.yml

@@ -16,7 +16,6 @@ on:
       - 'internal/jobservice/*'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
-      - 'scripts/build-airflow-operator.sh'
       - 'scripts/build-python-client.sh'
       - 'third_party/airflow/**'
       - './magefiles/tests.go'
@@ -37,7 +36,6 @@ on:
       - 'internal/jobservice/*'
       - 'pkg/api/*.proto'
       - 'pkg/api/jobservice/*.proto'
-      - 'scripts/build-airflow-operator.sh'
       - 'scripts/build-python-client.sh'
       - 'third_party/airflow/**'
 

.goreleaser.yml

@@ -195,7 +195,7 @@ dockers:
       - --label=org.opencontainers.image.version={{ .Version }}
       - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }}
       - --label=org.opencontainers.image.revision={{ .FullCommit }}
-      - --label=org.opencontainers.image.base.name=alpine:3.18.3
+      - --label=org.opencontainers.image.base.name=alpine:3.20.1
       - --label=org.opencontainers.image.licenses=Apache-2.0
       - --label=org.opencontainers.image.vendor=G-Research
     ids:

.mergify.yml

@@ -9,6 +9,6 @@ pull_request_rules:
           - "#approved-reviews-by>=2"
           - and: 
             - "#approved-reviews-by>=1"
-            - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|JamesMurkin|msumner91|masipauskas|mijovicmia|MustafaI|zuqq|richscott|robertdavidsmith|samclark)"
+            - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|JamesMurkin|msumner91|masipauskas|mijovicmia|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx)"
         title:
           Two are checks required.

MAINTAINERS.md

@@ -16,6 +16,7 @@
 | Rich Scott           | [richscott](https://github.com/richscott)               | G-Research  |
 | Robert Smith         | [robertdavidsmith](https://github.com/robertdavidsmith) | G-Research  |
 | Sam Clark            | [samclark](https://github.com/samclark)                 | G-Research  |
+| Geoffrey Wilson      | [suprjinx](https://github.com/suprjinx)                 | G-Research  |
 
 ## Past
 
@@ -26,6 +27,5 @@
 | Carlo Camurri        | [carlocamurri](https://github.com/carlocamurri)         | G-Research  |
 | Clifton Houck        | [ClifHouck](https://github.com/ClifHouck)               | G-Research  |
 | Daniel Rastelli      | [theAntiYeti](https://github.com/theAntiYeti)           | G-Research  |
-| Geoffrey Wilson      | [suprjinx](https://github.com/suprjinx)                 | G-Research  |
 | Jamie Poole          | [jimbobby5](https://github.com/jimbobby5)               | G-Research  |
 | Kevin Hannon         | [kannon92](https://github.com/kannon92)                 | G-Research  |

build/airflow-operator/Dockerfile

@@ -1,5 +1,5 @@
 ARG PLATFORM=x86_64
-ARG BASE_IMAGE=python:3.8.18-bookworm
+ARG BASE_IMAGE=python:3.10.14-bookworm
 FROM --platform=$PLATFORM ${BASE_IMAGE}
 
 RUN mkdir /proto

build/armada-load-tester/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/armada/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/armadactl/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/binoculars/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/eventingester/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/executor/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/fakeexecutor/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/jobservice/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/lookoutingesterv2/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/lookoutv2/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/scheduler/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/scheduleringester/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build/testsuite/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.3
+FROM alpine:3.20.1
 
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 

build_goreleaser/armadactl/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 FROM ${BASE_IMAGE}
 
 LABEL org.opencontainers.image.title=armadactl

build_goreleaser/binoculars/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=binoculars
 LABEL org.opencontainers.image.description="binoculars"

build_goreleaser/bundles/armada/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=armada
 LABEL org.opencontainers.image.description="Armada Bundle"

build_goreleaser/bundles/full/Dockerfile

@@ -1,6 +1,6 @@
 ARG NODE_BUILD_IMAGE=node:16.14-buster
 ARG OPENAPI_BUILD_IMAGE=openapitools/openapi-generator-cli:v5.4.0
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${OPENAPI_BUILD_IMAGE} AS OPENAPI
 LABEL org.opencontainers.image.title=armada-full-bundle

build_goreleaser/bundles/lookout/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=armada-lookout-bundle

build_goreleaser/eventingester/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.5
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=eventingester

build_goreleaser/executor/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=executor

build_goreleaser/fakeexecutor/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=fakeexecutor
 LABEL org.opencontainers.image.description="Fake Executor"

build_goreleaser/jobservice/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=jobservice

build_goreleaser/loadtester/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=loadtester

build_goreleaser/lookoutingesterv2/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=lookoutingesterv2
@@ -13,4 +13,4 @@ COPY config/lookoutingesterv2/config.yaml /app/config/lookoutingesterv2/config.y
 
 WORKDIR /app
 
-ENTRYPOINT ["./lookoutingesterv2"]
+ENTRYPOINT ["./lookoutingesterv2"]

build_goreleaser/lookoutv2/Dockerfile

@@ -1,6 +1,6 @@
 ARG NODE_BUILD_IMAGE=node:16.14-buster
 ARG OPENAPI_BUILD_IMAGE=openapitools/openapi-generator-cli:v5.4.0
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${OPENAPI_BUILD_IMAGE} AS OPENAPI
 

build_goreleaser/scheduler/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=scheduler

build_goreleaser/scheduleringester/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=scheduleringester

build_goreleaser/server/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=armada-server

build_goreleaser/testsuite/Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=alpine:3.18.3
+ARG BASE_IMAGE=alpine:3.20.1
 FROM ${BASE_IMAGE}
 LABEL org.opencontainers.image.title=testsuite
 LABEL org.opencontainers.image.description="Test Suite"

cmd/armada-load-tester/cmd/loadtest.go

@@ -52,7 +52,7 @@ var loadtestCmd = &cobra.Command{
 			  containers:
 				- name: sleep
 				  imagePullPolicy: IfNotPresent
-				  image: alpine:3.18.3
+				  image: alpine:3.20.1
 				  command:
 					- sh
 				  args:

config/armada/config.yaml

@@ -75,6 +75,7 @@ pulsar:
   compressionLevel: faster
   eventsPrinter: false
   eventsPrinterSubscription: "EventsPrinter"
+  maxAllowedEventsPerMessage: 1000
   maxAllowedMessageSize: 4194304 # 4MB
   receiverQueueSize: 100
 postgres:

config/eventingester/config.yaml

@@ -12,9 +12,9 @@ pulsar:
   receiverQueueSize: 100
 subscriptionName: "events-ingester"
 minMessageCompressionSize: 1024
-batchSize: 1048576  #1MB
+maxOutputMessageSizeBytes: 1048576  #1MB
+batchSize: 10000
 batchDuration: 100ms
-batchMessages: 10000
 eventRetentionPolicy:
   retentionDuration: 336h
 metricsPort: 9001

config/scheduler/config.yaml

@@ -37,6 +37,7 @@ pulsar:
   maxConnectionsPerBroker: 1
   compressionType: zlib
   compressionLevel: faster
+  maxAllowedEventsPerMessage: 1000
   maxAllowedMessageSize: 4194304 #4Mi
 armadaApi:
   armadaUrl: "server:50051"
@@ -86,6 +87,7 @@ scheduling:
   disableScheduling: false
   enableAssertions: false
   protectedFractionOfFairShare: 1.0
+  useAdjustedFairShareProtection: true
   nodeIdLabel: "kubernetes.io/hostname"
   priorityClasses:
     armada-default:

developer/config/job.yaml

@@ -11,7 +11,7 @@ jobs:
       containers:
         - name: sleep
           imagePullPolicy: IfNotPresent
-          image: alpine:3.10
+          image: alpine:latest
           args:
             - "exit"
             - "1"
@@ -26,4 +26,4 @@ jobs:
 timeout: "100s"
 expectedEvents:
   - submitted:
-  - failed:
+  - failed:

developer/env/docker/server.env

@@ -1,3 +1,3 @@
 ARMADA_QUEUECACHEREFRESHPERIOD="1s"
 ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:10000,http://example.com:10000"
-
+ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST=postgres

docker-compose.yaml

@@ -61,6 +61,7 @@ services:
     depends_on:
       - lookoutv2-migration
       - eventingester
+      - lookoutingesterv2
     working_dir: /app
     env_file:
       - developer/env/docker/server.env