From 5796a536942296b4ce352028f723d8f435039fd2 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Wed, 3 Jul 2024 13:40:55 +0100 Subject: [PATCH 1/3] add queue priority to metrics (#3766) Signed-off-by: Chris Martin Co-authored-by: Chris Martin --- internal/common/metrics/domain.go | 1 + internal/common/metrics/scheduler_metrics.go | 106 ++++++++++--------- internal/scheduler/metrics.go | 8 ++ 3 files changed, 66 insertions(+), 49 deletions(-) diff --git a/internal/common/metrics/domain.go b/internal/common/metrics/domain.go index ba6eb2eaf34..599f1fdc8b6 100644 --- a/internal/common/metrics/domain.go +++ b/internal/common/metrics/domain.go @@ -10,6 +10,7 @@ import ( type QueueMetricProvider interface { GetQueuedJobMetrics(queueName string) []*QueueMetrics GetRunningJobMetrics(queueName string) []*QueueMetrics + GetQueuePriorites() map[string]float64 } type QueueMetrics struct { diff --git a/internal/common/metrics/scheduler_metrics.go b/internal/common/metrics/scheduler_metrics.go index 64bf0ab5ff1..a99af7e7509 100644 --- a/internal/common/metrics/scheduler_metrics.go +++ b/internal/common/metrics/scheduler_metrics.go @@ -11,154 +11,147 @@ const MetricPrefix = "armada_" var QueueSizeDesc = prometheus.NewDesc( MetricPrefix+"queue_size", "Number of jobs in a queue", - []string{"queueName"}, + []string{"queueName", "queue"}, nil, ) var QueueDistinctSchedulingKeysDesc = prometheus.NewDesc( MetricPrefix+"queue_distinct_scheduling_keys", "Number of distinct scheduling keys requested by a queue", - []string{"queueName"}, - nil, -) - -var QueuePriorityDesc = prometheus.NewDesc( - MetricPrefix+"queue_priority", - "Priority of a queue", - []string{"pool", "queueName"}, + []string{"queueName", "queue"}, nil, ) var QueueResourcesDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_queued", "Resource required by queued jobs", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var MinQueueResourcesDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_queued_min", "Min resource required by queued job", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var MaxQueueResourcesDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_queued_max", "Max resource required by queued job", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var MedianQueueResourcesDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_queued_median", "Median resource required by queued jobs", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var CountQueueResourcesDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_queued_count", "Count of queued jobs requiring resource", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var MinQueueDurationDesc = prometheus.NewDesc( MetricPrefix+"job_queued_seconds_min", "Min queue time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var MaxQueueDurationDesc = prometheus.NewDesc( MetricPrefix+"job_queued_seconds_max", "Max queue time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var MedianQueueDurationDesc = prometheus.NewDesc( MetricPrefix+"job_queued_seconds_median", "Median queue time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var QueueDurationDesc = prometheus.NewDesc( MetricPrefix+"job_queued_seconds", "Queued time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var MinJobRunDurationDesc = prometheus.NewDesc( MetricPrefix+"job_run_time_seconds_min", "Min run time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var MaxJobRunDurationDesc = prometheus.NewDesc( MetricPrefix+"job_run_time_seconds_max", "Max run time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var MedianJobRunDurationDesc = prometheus.NewDesc( MetricPrefix+"job_run_time_seconds_median", "Median run time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var JobRunDurationDesc = prometheus.NewDesc( MetricPrefix+"job_run_time_seconds", "Run time for Armada jobs", - []string{"pool", "priorityClass", "queueName"}, + []string{"pool", "priorityClass", "queueName", "queue"}, nil, ) var QueueAllocatedDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_allocated", "Resource allocated to running jobs of a queue", - []string{"cluster", "pool", "priorityClass", "queueName", "resourceType", "nodeType"}, + []string{"cluster", "pool", "priorityClass", "queueName", "queue", "resourceType", "nodeType"}, nil, ) var MinQueueAllocatedDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_allocated_min", "Min resource allocated by a running job", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var MaxQueueAllocatedDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_allocated_max", "Max resource allocated by a running job", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var MedianQueueAllocatedDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_allocated_median", "Median resource allocated by a running job", - []string{"pool", "priorityClass", "queueName", "resourceType"}, + []string{"pool", "priorityClass", "queueName", "queue", "resourceType"}, nil, ) var QueueUsedDesc = prometheus.NewDesc( MetricPrefix+"queue_resource_used", "Resource actually being used by running jobs of a queue", - []string{"cluster", "pool", "queueName", "resourceType", "nodeType"}, + []string{"cluster", "pool", "queueName", "queue", "resourceType", "nodeType"}, nil, ) var QueueLeasedPodCountDesc = prometheus.NewDesc( MetricPrefix+"queue_leased_pod_count", "Number of leased pods", - []string{"cluster", "pool", "queueName", "phase", "nodeType"}, + []string{"cluster", "pool", "queueName", "queue", "phase", "nodeType"}, nil, ) @@ -176,6 +169,13 @@ var ClusterAvailableCapacityDesc = prometheus.NewDesc( nil, ) +var QueuePriorityDesc = prometheus.NewDesc( + MetricPrefix+"queue_priority", + "Queue priority factor", + []string{"queueName", "queue"}, + nil, +) + var AllDescs = []*prometheus.Desc{ QueueSizeDesc, QueuePriorityDesc, @@ -201,6 +201,7 @@ var AllDescs = []*prometheus.Desc{ QueueLeasedPodCountDesc, ClusterCapacityDesc, ClusterAvailableCapacityDesc, + QueuePriorityDesc, } func Describe(out chan<- *prometheus.Desc) { @@ -264,83 +265,86 @@ func CollectQueueMetrics(queueCounts map[string]int, queueDistinctSchedulingKeyC } } } + for q, priority := range metricsProvider.GetQueuePriorites() { + metrics = append(metrics, NewQueuePriorityMetric(priority, q)) + } return metrics } func NewQueueSizeMetric(value int, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(QueueSizeDesc, prometheus.GaugeValue, float64(value), queue) + return prometheus.MustNewConstMetric(QueueSizeDesc, prometheus.GaugeValue, float64(value), queue, queue) } func NewQueueDistinctSchedulingKeyMetric(value int, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(QueueDistinctSchedulingKeysDesc, prometheus.GaugeValue, float64(value), queue) + return prometheus.MustNewConstMetric(QueueDistinctSchedulingKeysDesc, prometheus.GaugeValue, float64(value), queue, queue) } func NewQueueDuration(count uint64, sum float64, buckets map[float64]uint64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstHistogram(QueueDurationDesc, count, sum, buckets, pool, priorityClass, queue) + return prometheus.MustNewConstHistogram(QueueDurationDesc, count, sum, buckets, pool, priorityClass, queue, queue) } func NewQueueResources(value float64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(QueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(QueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue, resource) } func NewMaxQueueResources(value float64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(MaxQueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(MaxQueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue, resource) } func NewMinQueueResources(value float64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(MinQueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(MinQueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue, resource) } func NewMedianQueueResources(value float64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(MedianQueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(MedianQueueResourcesDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue, resource) } func NewCountQueueResources(value uint64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(CountQueueResourcesDesc, prometheus.GaugeValue, float64(value), pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(CountQueueResourcesDesc, prometheus.GaugeValue, float64(value), pool, priorityClass, queue, queue, resource) } func NewMinQueueDuration(value float64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(MinQueueDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue) + return prometheus.MustNewConstMetric(MinQueueDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue) } func NewMaxQueueDuration(value float64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(MaxQueueDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue) + return prometheus.MustNewConstMetric(MaxQueueDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue) } func NewMedianQueueDuration(value float64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(MedianQueueDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue) + return prometheus.MustNewConstMetric(MedianQueueDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue) } func NewMinJobRunDuration(value float64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(MinJobRunDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue) + return prometheus.MustNewConstMetric(MinJobRunDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue) } func NewMaxJobRunDuration(value float64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(MaxJobRunDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue) + return prometheus.MustNewConstMetric(MaxJobRunDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue) } func NewMedianJobRunDuration(value float64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstMetric(MedianJobRunDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue) + return prometheus.MustNewConstMetric(MedianJobRunDurationDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue) } func NewJobRunRunDuration(count uint64, sum float64, buckets map[float64]uint64, pool string, priorityClass string, queue string) prometheus.Metric { - return prometheus.MustNewConstHistogram(JobRunDurationDesc, count, sum, buckets, pool, priorityClass, queue) + return prometheus.MustNewConstHistogram(JobRunDurationDesc, count, sum, buckets, pool, priorityClass, queue, queue) } func NewMinQueueAllocated(value float64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(MinQueueAllocatedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(MinQueueAllocatedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue, resource) } func NewMaxQueueAllocated(value float64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(MaxQueueAllocatedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(MaxQueueAllocatedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue, resource) } func NewMedianQueueAllocated(value float64, pool string, priorityClass string, queue string, resource string) prometheus.Metric { - return prometheus.MustNewConstMetric(MedianQueueAllocatedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, resource) + return prometheus.MustNewConstMetric(MedianQueueAllocatedDesc, prometheus.GaugeValue, value, pool, priorityClass, queue, queue, resource) } func NewQueueLeasedPodCount(value float64, cluster string, pool string, queue string, phase string, nodeType string) prometheus.Metric { - return prometheus.MustNewConstMetric(QueueLeasedPodCountDesc, prometheus.GaugeValue, value, cluster, pool, queue, phase, nodeType) + return prometheus.MustNewConstMetric(QueueLeasedPodCountDesc, prometheus.GaugeValue, value, cluster, pool, queue, queue, phase, nodeType) } func NewClusterAvailableCapacity(value float64, cluster string, pool string, resource string, nodeType string) prometheus.Metric { @@ -352,9 +356,13 @@ func NewClusterTotalCapacity(value float64, cluster string, pool string, resourc } func NewQueueAllocated(value float64, queue string, cluster string, pool string, priorityClass string, resource string, nodeType string) prometheus.Metric { - return prometheus.MustNewConstMetric(QueueAllocatedDesc, prometheus.GaugeValue, value, cluster, pool, priorityClass, queue, resource, nodeType) + return prometheus.MustNewConstMetric(QueueAllocatedDesc, prometheus.GaugeValue, value, cluster, pool, priorityClass, queue, queue, resource, nodeType) } func NewQueueUsed(value float64, queue string, cluster string, pool string, resource string, nodeType string) prometheus.Metric { - return prometheus.MustNewConstMetric(QueueUsedDesc, prometheus.GaugeValue, value, cluster, pool, queue, resource, nodeType) + return prometheus.MustNewConstMetric(QueueUsedDesc, prometheus.GaugeValue, value, cluster, pool, queue, queue, resource, nodeType) +} + +func NewQueuePriorityMetric(value float64, queue string) prometheus.Metric { + return prometheus.MustNewConstMetric(QueuePriorityDesc, prometheus.GaugeValue, value, queue, queue) } diff --git a/internal/scheduler/metrics.go b/internal/scheduler/metrics.go index a5eeeddc9a1..d2ab9d34318 100644 --- a/internal/scheduler/metrics.go +++ b/internal/scheduler/metrics.go @@ -24,6 +24,7 @@ import ( type queueState struct { queuedJobRecorder *commonmetrics.JobMetricsRecorder runningJobRecorder *commonmetrics.JobMetricsRecorder + priority float64 } // metricProvider is a simple implementation of QueueMetricProvider @@ -31,6 +32,12 @@ type metricProvider struct { queueStates map[string]*queueState } +func (m metricProvider) GetQueuePriorites() map[string]float64 { + return armadamaps.MapValues(m.queueStates, func(v *queueState) float64 { + return v.priority + }) +} + func (m metricProvider) GetQueuedJobMetrics(queueName string) []*commonmetrics.QueueMetrics { state, ok := m.queueStates[queueName] if ok { @@ -143,6 +150,7 @@ func (c *MetricsCollector) updateQueueMetrics(ctx *armadacontext.Context) ([]pro provider.queueStates[queue.Name] = &queueState{ queuedJobRecorder: commonmetrics.NewJobMetricsRecorder(), runningJobRecorder: commonmetrics.NewJobMetricsRecorder(), + priority: queue.PriorityFactor, } queuedJobsCount[queue.Name] = 0 schedulingKeysByQueue[queue.Name] = map[schedulerobjects.SchedulingKey]bool{} From 01333d8475aa73b1fc5b2fe8549fb9cd6f7cbc86 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Wed, 3 Jul 2024 14:02:47 +0100 Subject: [PATCH 2/3] Remove Gogo Proto From Executor API (#3763) * initial timestamps Signed-off-by: Chris Martin * initial timestamps Signed-off-by: Chris Martin * initial timestamps Signed-off-by: Chris Martin * tests Signed-off-by: Chris Martin * regen proto Signed-off-by: Chris Martin * fix diff Signed-off-by: Chris Martin * submit.go Signed-off-by: Chris Martin * submit.go Signed-off-by: Chris Martin * more fixes Signed-off-by: Chris Martin * more fixes Signed-off-by: Chris Martin * lint Signed-off-by: Chris Martin * fix tests Signed-off-by: Chris Martin * fix proto gen Signed-off-by: Chris Martin * fix test Signed-off-by: Chris Martin * comment out test Signed-off-by: Chris Martin * fix Signed-off-by: Chris Martin * fix Signed-off-by: Chris Martin * fix Signed-off-by: Chris Martin * another test Signed-off-by: Chris Martin * another test Signed-off-by: Chris Martin * initial commit Signed-off-by: Chris Martin * fix test Signed-off-by: Chris Martin --------- Signed-off-by: Chris Martin Co-authored-by: Chris Martin --- internal/executor/service/job_requester.go | 2 +- .../executor/service/job_requester_test.go | 4 +- internal/executor/service/lease_requester.go | 6 +- .../executor/service/lease_requester_test.go | 6 +- internal/executor/util/uuid.go | 6 +- internal/executor/util/uuid_test.go | 23 +- .../utilisation/cluster_utilisation.go | 26 +- internal/scheduler/api.go | 8 +- internal/scheduler/api_test.go | 6 +- pkg/executorapi/executorapi.pb.go | 861 +++++++----------- pkg/executorapi/executorapi.proto | 26 +- pkg/executorapi/util.go | 54 +- 12 files changed, 406 insertions(+), 622 deletions(-) diff --git a/internal/executor/service/job_requester.go b/internal/executor/service/job_requester.go index 0e5a09be52b..7eb2167f978 100644 --- a/internal/executor/service/job_requester.go +++ b/internal/executor/service/job_requester.go @@ -103,7 +103,7 @@ func (r *JobRequester) createLeaseRequest() (*LeaseRequest, error) { } // Returns the RunIds of all managed pods that haven't been assigned to a node -func (r *JobRequester) getUnassignedRunIds(capacityReport *utilisation.ClusterAvailableCapacityReport) ([]armadaevents.Uuid, error) { +func (r *JobRequester) getUnassignedRunIds(capacityReport *utilisation.ClusterAvailableCapacityReport) ([]*armadaevents.Uuid, error) { allAssignedRunIds := []string{} allJobRunIds := []string{} diff --git a/internal/executor/service/job_requester_test.go b/internal/executor/service/job_requester_test.go index bcc2316f0d0..027598a365b 100644 --- a/internal/executor/service/job_requester_test.go +++ b/internal/executor/service/job_requester_test.go @@ -76,7 +76,7 @@ func TestRequestJobsRuns_ConstructsCorrectLeaseRequest(t *testing.T) { AvailableResource: *capacityReport.AvailableCapacity, Nodes: []*executorapi.NodeInfo{&capacityReport.Nodes[0]}, // Should add any ids in the state but not in the capacity report into unassigned job run ids - UnassignedJobRunIds: []armadaevents.Uuid{}, + UnassignedJobRunIds: []*armadaevents.Uuid{}, MaxJobsToLease: uint32(defaultMaxLeasedJobs), }, }, @@ -86,7 +86,7 @@ func TestRequestJobsRuns_ConstructsCorrectLeaseRequest(t *testing.T) { AvailableResource: *capacityReport.AvailableCapacity, Nodes: []*executorapi.NodeInfo{&capacityReport.Nodes[0]}, // Should add any ids in the state but not in the capacity report into unassigned job run ids - UnassignedJobRunIds: []armadaevents.Uuid{*armadaevents.ProtoUuidFromUuid(leasedRunId)}, + UnassignedJobRunIds: []*armadaevents.Uuid{armadaevents.ProtoUuidFromUuid(leasedRunId)}, MaxJobsToLease: 0, }, }, diff --git a/internal/executor/service/lease_requester.go b/internal/executor/service/lease_requester.go index a5dd67dffce..14ef77b1c45 100644 --- a/internal/executor/service/lease_requester.go +++ b/internal/executor/service/lease_requester.go @@ -20,7 +20,7 @@ import ( type LeaseRequest struct { AvailableResource armadaresource.ComputeResources Nodes []*executorapi.NodeInfo - UnassignedJobRunIds []armadaevents.Uuid + UnassignedJobRunIds []*armadaevents.Uuid MaxJobsToLease uint32 } @@ -60,8 +60,8 @@ func (requester *JobLeaseRequester) LeaseJobRuns(ctx *armadacontext.Context, req leaseRequest := &executorapi.LeaseRequest{ ExecutorId: requester.clusterIdentity.GetClusterId(), Pool: requester.clusterIdentity.GetClusterPool(), - MinimumJobSize: requester.minimumJobSize, - Resources: request.AvailableResource, + MinimumJobSize: requester.minimumJobSize.ToProtoMap(), + Resources: request.AvailableResource.ToProtoMap(), Nodes: request.Nodes, UnassignedJobRunIds: request.UnassignedJobRunIds, MaxJobsToLease: request.MaxJobsToLease, diff --git a/internal/executor/service/lease_requester_test.go b/internal/executor/service/lease_requester_test.go index fd625124e7b..d596e8bbc03 100644 --- a/internal/executor/service/lease_requester_test.go +++ b/internal/executor/service/lease_requester_test.go @@ -101,15 +101,15 @@ func TestLeaseJobRuns_Send(t *testing.T) { RunIdsByState: map[string]api.JobState{"id1": api.JobState_RUNNING}, }, }, - UnassignedJobRunIds: []armadaevents.Uuid{*id1}, + UnassignedJobRunIds: []*armadaevents.Uuid{id1}, MaxJobsToLease: uint32(5), } expectedRequest := &executorapi.LeaseRequest{ ExecutorId: defaultClusterIdentity.GetClusterId(), Pool: defaultClusterIdentity.GetClusterPool(), - Resources: leaseRequest.AvailableResource, - MinimumJobSize: defaultMinimumJobSize, + Resources: leaseRequest.AvailableResource.ToProtoMap(), + MinimumJobSize: defaultMinimumJobSize.ToProtoMap(), Nodes: leaseRequest.Nodes, UnassignedJobRunIds: leaseRequest.UnassignedJobRunIds, MaxJobsToLease: leaseRequest.MaxJobsToLease, diff --git a/internal/executor/util/uuid.go b/internal/executor/util/uuid.go index ac3eb48c16f..e2dd377588f 100644 --- a/internal/executor/util/uuid.go +++ b/internal/executor/util/uuid.go @@ -6,14 +6,14 @@ import ( "github.com/armadaproject/armada/pkg/armadaevents" ) -func StringUuidsToUuids(uuidStrings []string) ([]armadaevents.Uuid, error) { - result := make([]armadaevents.Uuid, 0, len(uuidStrings)) +func StringUuidsToUuids(uuidStrings []string) ([]*armadaevents.Uuid, error) { + result := make([]*armadaevents.Uuid, 0, len(uuidStrings)) for _, uuidString := range uuidStrings { uuid, err := armadaevents.ProtoUuidFromUuidString(uuidString) if err != nil { return nil, fmt.Errorf("failed to convert uuid string %s to uuid because %s", uuidString, err) } - result = append(result, *uuid) + result = append(result, uuid) } return result, nil } diff --git a/internal/executor/util/uuid_test.go b/internal/executor/util/uuid_test.go index bfb822b36f0..6b61154d03f 100644 --- a/internal/executor/util/uuid_test.go +++ b/internal/executor/util/uuid_test.go @@ -3,6 +3,9 @@ package util import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/armadaproject/armada/pkg/armadaevents" ) @@ -10,15 +13,15 @@ func TestStringUuidsToUuids(t *testing.T) { tests := []struct { name string uuidStrings []string - want []armadaevents.Uuid + want []*armadaevents.Uuid wantErr bool }{ - {"invalid uuid", []string{"1", "2", "3"}, []armadaevents.Uuid{}, true}, - {"valid uuid", []string{"52a3cfa6-8ce1-42b1-97cf-74f1b63f21b9"}, []armadaevents.Uuid{{5954831446549021361, 10939090601399755193}}, false}, + {"invalid uuid", []string{"1", "2", "3"}, []*armadaevents.Uuid{}, true}, + {"valid uuid", []string{"52a3cfa6-8ce1-42b1-97cf-74f1b63f21b9"}, []*armadaevents.Uuid{{5954831446549021361, 10939090601399755193}}, false}, { "valid uuid2", []string{"52a3cfa6-8ce1-42b1-97cf-74f1b63f21b9", "59567531-2a42-4b5b-9aba-b3d400c35b4c"}, - []armadaevents.Uuid{ + []*armadaevents.Uuid{ {5954831446549021361, 10939090601399755193}, {6437461571395537755, 11149421550636325708}, }, @@ -29,15 +32,13 @@ func TestStringUuidsToUuids(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := StringUuidsToUuids(tt.uuidStrings) - if (err != nil) != tt.wantErr { - t.Errorf("StringUuidsToUuids() error = %v, wantErr %v", err, tt.wantErr) - return + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) } - for i, v := range got { - if v != tt.want[i] { - t.Errorf("StringUuidsToUuids() = %v, want %v", v, tt.want[i]) - } + assert.Equal(t, tt.want[i], v) } }) } diff --git a/internal/executor/utilisation/cluster_utilisation.go b/internal/executor/utilisation/cluster_utilisation.go index ca2b57ea666..0f2866003b2 100644 --- a/internal/executor/utilisation/cluster_utilisation.go +++ b/internal/executor/utilisation/cluster_utilisation.go @@ -3,6 +3,8 @@ package utilisation import ( "fmt" + armadaslices "github.com/armadaproject/armada/internal/common/slices" + "github.com/armadaproject/armada/pkg/executorapi" "github.com/pkg/errors" @@ -112,24 +114,26 @@ func (cls *ClusterUtilisationService) GetAvailableClusterCapacity() (*ClusterAva usageByQueue := cls.getPodUtilisationByQueue(runningNodePodsArmada) resourceUsageByQueue := make(map[string]*executorapi.ComputeResource) for queueName, resourceUsage := range usageByQueue { - resourceUsageByQueue[queueName] = &executorapi.ComputeResource{Resources: resourceUsage} + resourceUsageByQueue[queueName] = executorapi.ComputeResourceFromProtoResources(resourceUsage) } - nodeAllocatedResources := make(map[int32]executorapi.ComputeResource) + nodeAllocatedResources := make(map[int32]*executorapi.ComputeResource) for p, rl := range allocatedByPriority { - nodeAllocatedResources[p] = executorapi.ComputeResource{Resources: rl.Resources} + nodeAllocatedResources[p] = executorapi.ComputeResourceFromProtoResources(rl.Resources) } - nodeNonArmadaAllocatedResources := make(map[int32]executorapi.ComputeResource) + nodeNonArmadaAllocatedResources := make(map[int32]*executorapi.ComputeResource) for p, rl := range allocatedByPriorityNonArmada { - nodeNonArmadaAllocatedResources[p] = executorapi.ComputeResource{Resources: rl.Resources} + nodeNonArmadaAllocatedResources[p] = executorapi.ComputeResourceFromProtoResources(rl.Resources) } nodes = append(nodes, executorapi.NodeInfo{ - Name: node.Name, - Labels: cls.filterTrackedLabels(node.Labels), - Taints: node.Spec.Taints, - AllocatableResources: allocatable, - AvailableResources: available, - TotalResources: allocatable, + Name: node.Name, + Labels: cls.filterTrackedLabels(node.Labels), + Taints: armadaslices.Map(node.Spec.Taints, func(t v1.Taint) *v1.Taint { + return &t + }), + AllocatableResources: allocatable.ToProtoMap(), + AvailableResources: available.ToProtoMap(), + TotalResources: allocatable.ToProtoMap(), AllocatedResources: nodeAllocatedResources, RunIdsByState: runIdsByNode[node.Name], NonArmadaAllocatedResources: nodeNonArmadaAllocatedResources, diff --git a/internal/scheduler/api.go b/internal/scheduler/api.go index f455150d274..b493873908b 100644 --- a/internal/scheduler/api.go +++ b/internal/scheduler/api.go @@ -335,10 +335,10 @@ func (srv *ExecutorApi) executorFromLeaseRequest(ctx *armadacontext.Context, req Id: req.ExecutorId, Pool: req.Pool, Nodes: nodes, - MinimumJobSize: schedulerobjects.ResourceList{Resources: req.MinimumJobSize}, + MinimumJobSize: executorapi.ResourceListFromProtoResources(req.MinimumJobSize), LastUpdateTime: now, - UnassignedJobRuns: slices.Map(req.UnassignedJobRunIds, func(jobId armadaevents.Uuid) string { - return strings.ToLower(armadaevents.UuidFromProtoUuid(&jobId).String()) + UnassignedJobRuns: slices.Map(req.UnassignedJobRunIds, func(jobId *armadaevents.Uuid) string { + return strings.ToLower(armadaevents.UuidFromProtoUuid(jobId).String()) }), } } @@ -356,7 +356,7 @@ func runIdsFromLeaseRequest(req *executorapi.LeaseRequest) ([]uuid.UUID, error) } } for _, runId := range req.UnassignedJobRunIds { - runIds = append(runIds, armadaevents.UuidFromProtoUuid(&runId)) + runIds = append(runIds, armadaevents.UuidFromProtoUuid(runId)) } return runIds, nil } diff --git a/internal/scheduler/api_test.go b/internal/scheduler/api_test.go index 022aae6a2c4..4b080235a52 100644 --- a/internal/scheduler/api_test.go +++ b/internal/scheduler/api_test.go @@ -60,7 +60,7 @@ func TestExecutorApi_LeaseJobRuns(t *testing.T) { NodeType: "node-type-1", }, }, - UnassignedJobRunIds: []armadaevents.Uuid{*armadaevents.ProtoUuidFromUuid(runId3)}, + UnassignedJobRunIds: []*armadaevents.Uuid{armadaevents.ProtoUuidFromUuid(runId3)}, MaxJobsToLease: uint32(maxJobsPerCall), } defaultExpectedExecutor := &schedulerobjects.Executor{ @@ -71,7 +71,7 @@ func TestExecutorApi_LeaseJobRuns(t *testing.T) { Id: "test-executor-test-node", Name: "test-node", Executor: "test-executor", - TotalResources: schedulerobjects.ResourceList{}, + TotalResources: schedulerobjects.NewResourceList(0), StateByJobRunId: map[string]schedulerobjects.JobRunState{runId1.String(): schedulerobjects.JobRunState_RUNNING, runId2.String(): schedulerobjects.JobRunState_RUNNING}, NonArmadaAllocatedResources: map[int32]schedulerobjects.ResourceList{}, AllocatableByPriorityAndResource: map[int32]schedulerobjects.ResourceList{ @@ -87,7 +87,7 @@ func TestExecutorApi_LeaseJobRuns(t *testing.T) { ReportingNodeType: "node-type-1", }, }, - MinimumJobSize: schedulerobjects.ResourceList{}, + MinimumJobSize: schedulerobjects.NewResourceList(0), LastUpdateTime: testClock.Now().UTC(), UnassignedJobRuns: []string{runId3.String()}, } diff --git a/pkg/executorapi/executorapi.pb.go b/pkg/executorapi/executorapi.pb.go index 1f2e18e36a2..9470141afb2 100644 --- a/pkg/executorapi/executorapi.pb.go +++ b/pkg/executorapi/executorapi.pb.go @@ -9,12 +9,8 @@ import ( io "io" math "math" math_bits "math/bits" - reflect "reflect" - strings "strings" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" - github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" types "github.com/gogo/protobuf/types" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -40,28 +36,28 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package // Used by the scheduler when allocating jobs to executors. type NodeInfo struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Taints []v1.Taint `protobuf:"bytes,2,rep,name=taints,proto3" json:"taints"` + Taints []*v1.Taint `protobuf:"bytes,2,rep,name=taints,proto3" json:"taints,omitempty"` Labels map[string]string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // To be deprecated in favour of total_resources + allocated_resources. - AllocatableResources map[string]resource.Quantity `protobuf:"bytes,4,rep,name=allocatable_resources,json=allocatableResources,proto3" json:"allocatableResources" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + AllocatableResources map[string]*resource.Quantity `protobuf:"bytes,4,rep,name=allocatable_resources,json=allocatableResources,proto3" json:"allocatableResources,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // To be deprecated in favour of total_resources + allocated_resources. - AvailableResources map[string]resource.Quantity `protobuf:"bytes,5,rep,name=available_resources,json=availableResources,proto3" json:"availableResources" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + AvailableResources map[string]*resource.Quantity `protobuf:"bytes,5,rep,name=available_resources,json=availableResources,proto3" json:"availableResources,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Total node resources. // Resources available for allocation is given by the difference between this and allocated_resources. - TotalResources map[string]resource.Quantity `protobuf:"bytes,6,rep,name=total_resources,json=totalResources,proto3" json:"totalResources" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + TotalResources map[string]*resource.Quantity `protobuf:"bytes,6,rep,name=total_resources,json=totalResources,proto3" json:"totalResources,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Each pod is created with a priority class. Each priority class has an integer priority associated with it. // This is a map from priority to the total amount of resources allocated to pods with that priority. // It is used by the scheduler to decide whether more jobs should be sent to an executor. // In particular, jobs may be sent to an executor even if all resources are allocated // if the sent jobs are of higher priority. - AllocatedResources map[int32]ComputeResource `protobuf:"bytes,7,rep,name=allocated_resources,json=allocatedResources,proto3" json:"allocatedResources" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + AllocatedResources map[int32]*ComputeResource `protobuf:"bytes,7,rep,name=allocated_resources,json=allocatedResources,proto3" json:"allocatedResources,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // All run ids of jobs on the node, mapped to their current state // this should be of type armadaevents.uuid, but this creates a circular loop // once the old scheduler has gone, we can correct this - RunIdsByState map[string]api.JobState `protobuf:"bytes,8,rep,name=run_ids_by_state,json=runIdsByState,proto3" json:"runIdsByState" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=api.JobState"` + RunIdsByState map[string]api.JobState `protobuf:"bytes,8,rep,name=run_ids_by_state,json=runIdsByState,proto3" json:"runIdsByState,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=api.JobState"` // The amount of resource allocated to non-armada pods by priority - NonArmadaAllocatedResources map[int32]ComputeResource `protobuf:"bytes,9,rep,name=non_armada_allocated_resources,json=nonArmadaAllocatedResources,proto3" json:"nonArmadaAllocatedResources" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - Unschedulable bool `protobuf:"varint,10,opt,name=unschedulable,proto3" json:"unschedulable,omitempty"` + NonArmadaAllocatedResources map[int32]*ComputeResource `protobuf:"bytes,9,rep,name=non_armada_allocated_resources,json=nonArmadaAllocatedResources,proto3" json:"nonArmadaAllocatedResources,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Unschedulable bool `protobuf:"varint,10,opt,name=unschedulable,proto3" json:"unschedulable,omitempty"` // This should only be used for metrics // An aggregated real usage of jobs by queue ResourceUsageByQueue map[string]*ComputeResource `protobuf:"bytes,11,rep,name=resource_usage_by_queue,json=resourceUsageByQueue,proto3" json:"resourceUsageByQueue,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -70,8 +66,9 @@ type NodeInfo struct { NodeType string `protobuf:"bytes,12,opt,name=node_type,json=nodeType,proto3" json:"nodeType,omitempty"` } -func (m *NodeInfo) Reset() { *m = NodeInfo{} } -func (*NodeInfo) ProtoMessage() {} +func (m *NodeInfo) Reset() { *m = NodeInfo{} } +func (m *NodeInfo) String() string { return proto.CompactTextString(m) } +func (*NodeInfo) ProtoMessage() {} func (*NodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{0} } @@ -109,7 +106,7 @@ func (m *NodeInfo) GetName() string { return "" } -func (m *NodeInfo) GetTaints() []v1.Taint { +func (m *NodeInfo) GetTaints() []*v1.Taint { if m != nil { return m.Taints } @@ -123,28 +120,28 @@ func (m *NodeInfo) GetLabels() map[string]string { return nil } -func (m *NodeInfo) GetAllocatableResources() map[string]resource.Quantity { +func (m *NodeInfo) GetAllocatableResources() map[string]*resource.Quantity { if m != nil { return m.AllocatableResources } return nil } -func (m *NodeInfo) GetAvailableResources() map[string]resource.Quantity { +func (m *NodeInfo) GetAvailableResources() map[string]*resource.Quantity { if m != nil { return m.AvailableResources } return nil } -func (m *NodeInfo) GetTotalResources() map[string]resource.Quantity { +func (m *NodeInfo) GetTotalResources() map[string]*resource.Quantity { if m != nil { return m.TotalResources } return nil } -func (m *NodeInfo) GetAllocatedResources() map[int32]ComputeResource { +func (m *NodeInfo) GetAllocatedResources() map[int32]*ComputeResource { if m != nil { return m.AllocatedResources } @@ -158,7 +155,7 @@ func (m *NodeInfo) GetRunIdsByState() map[string]api.JobState { return nil } -func (m *NodeInfo) GetNonArmadaAllocatedResources() map[int32]ComputeResource { +func (m *NodeInfo) GetNonArmadaAllocatedResources() map[int32]*ComputeResource { if m != nil { return m.NonArmadaAllocatedResources } @@ -187,11 +184,12 @@ func (m *NodeInfo) GetNodeType() string { } type ComputeResource struct { - Resources map[string]resource.Quantity `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Resources map[string]*resource.Quantity `protobuf:"bytes,1,rep,name=resources,proto3" json:"resources,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } -func (m *ComputeResource) Reset() { *m = ComputeResource{} } -func (*ComputeResource) ProtoMessage() {} +func (m *ComputeResource) Reset() { *m = ComputeResource{} } +func (m *ComputeResource) String() string { return proto.CompactTextString(m) } +func (*ComputeResource) ProtoMessage() {} func (*ComputeResource) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{1} } @@ -222,7 +220,7 @@ func (m *ComputeResource) XXX_DiscardUnknown() { var xxx_messageInfo_ComputeResource proto.InternalMessageInfo -func (m *ComputeResource) GetResources() map[string]resource.Quantity { +func (m *ComputeResource) GetResources() map[string]*resource.Quantity { if m != nil { return m.Resources } @@ -233,8 +231,9 @@ type EventList struct { Events []*armadaevents.EventSequence `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` } -func (m *EventList) Reset() { *m = EventList{} } -func (*EventList) ProtoMessage() {} +func (m *EventList) Reset() { *m = EventList{} } +func (m *EventList) String() string { return proto.CompactTextString(m) } +func (*EventList) ProtoMessage() {} func (*EventList) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{2} } @@ -278,22 +277,23 @@ type LeaseRequest struct { // Nodes are split into pools. This field indicates for which pool jobs are leased. Pool string `protobuf:"bytes,2,opt,name=pool,proto3" json:"pool,omitempty"` // Total resources available for scheduling across all nodes. - Resources map[string]resource.Quantity `protobuf:"bytes,3,rep,name=resources,proto3" json:"resources" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Resources map[string]*resource.Quantity `protobuf:"bytes,3,rep,name=resources,proto3" json:"resources,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Jobs submitted to this executor must require at least this amount of resources. - MinimumJobSize map[string]resource.Quantity `protobuf:"bytes,4,rep,name=minimum_job_size,json=minimumJobSize,proto3" json:"minimumJobSize" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + MinimumJobSize map[string]*resource.Quantity `protobuf:"bytes,4,rep,name=minimum_job_size,json=minimumJobSize,proto3" json:"minimumJobSize,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // For each node in the cluster: // - The total allocatable resources on that node. // - The job runs running on those nodes, // - Any taints and labels on the node. Nodes []*NodeInfo `protobuf:"bytes,5,rep,name=nodes,proto3" json:"nodes,omitempty"` // Run Ids of jobs owned by the executor but not currently assigned to a node. - UnassignedJobRunIds []armadaevents.Uuid `protobuf:"bytes,6,rep,name=unassigned_job_run_ids,json=unassignedJobRunIds,proto3" json:"unassignedJobRunIds"` + UnassignedJobRunIds []*armadaevents.Uuid `protobuf:"bytes,6,rep,name=unassigned_job_run_ids,json=unassignedJobRunIds,proto3" json:"unassignedJobRunIds,omitempty"` // Max number of jobs this request should return MaxJobsToLease uint32 `protobuf:"varint,7,opt,name=max_jobs_to_lease,json=maxJobsToLease,proto3" json:"maxJobsToLease,omitempty"` } -func (m *LeaseRequest) Reset() { *m = LeaseRequest{} } -func (*LeaseRequest) ProtoMessage() {} +func (m *LeaseRequest) Reset() { *m = LeaseRequest{} } +func (m *LeaseRequest) String() string { return proto.CompactTextString(m) } +func (*LeaseRequest) ProtoMessage() {} func (*LeaseRequest) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{3} } @@ -338,14 +338,14 @@ func (m *LeaseRequest) GetPool() string { return "" } -func (m *LeaseRequest) GetResources() map[string]resource.Quantity { +func (m *LeaseRequest) GetResources() map[string]*resource.Quantity { if m != nil { return m.Resources } return nil } -func (m *LeaseRequest) GetMinimumJobSize() map[string]resource.Quantity { +func (m *LeaseRequest) GetMinimumJobSize() map[string]*resource.Quantity { if m != nil { return m.MinimumJobSize } @@ -359,7 +359,7 @@ func (m *LeaseRequest) GetNodes() []*NodeInfo { return nil } -func (m *LeaseRequest) GetUnassignedJobRunIds() []armadaevents.Uuid { +func (m *LeaseRequest) GetUnassignedJobRunIds() []*armadaevents.Uuid { if m != nil { return m.UnassignedJobRunIds } @@ -383,8 +383,9 @@ type JobRunLease struct { Job *armadaevents.SubmitJob `protobuf:"bytes,6,opt,name=job,proto3" json:"job,omitempty"` } -func (m *JobRunLease) Reset() { *m = JobRunLease{} } -func (*JobRunLease) ProtoMessage() {} +func (m *JobRunLease) Reset() { *m = JobRunLease{} } +func (m *JobRunLease) String() string { return proto.CompactTextString(m) } +func (*JobRunLease) ProtoMessage() {} func (*JobRunLease) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{4} } @@ -462,8 +463,9 @@ type CancelRuns struct { JobRunIdsToCancel []*armadaevents.Uuid `protobuf:"bytes,1,rep,name=job_run_ids_to_cancel,json=jobRunIdsToCancel,proto3" json:"jobRunIdsToCancel,omitempty"` } -func (m *CancelRuns) Reset() { *m = CancelRuns{} } -func (*CancelRuns) ProtoMessage() {} +func (m *CancelRuns) Reset() { *m = CancelRuns{} } +func (m *CancelRuns) String() string { return proto.CompactTextString(m) } +func (*CancelRuns) ProtoMessage() {} func (*CancelRuns) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{5} } @@ -506,8 +508,9 @@ type PreemptRuns struct { JobRunIdsToPreempt []*armadaevents.Uuid `protobuf:"bytes,1,rep,name=job_run_ids_to_preempt,json=jobRunIdsToPreempt,proto3" json:"jobRunIdsToPreempt,omitempty"` } -func (m *PreemptRuns) Reset() { *m = PreemptRuns{} } -func (*PreemptRuns) ProtoMessage() {} +func (m *PreemptRuns) Reset() { *m = PreemptRuns{} } +func (m *PreemptRuns) String() string { return proto.CompactTextString(m) } +func (*PreemptRuns) ProtoMessage() {} func (*PreemptRuns) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{6} } @@ -549,8 +552,9 @@ func (m *PreemptRuns) GetJobRunIdsToPreempt() []*armadaevents.Uuid { type EndMarker struct { } -func (m *EndMarker) Reset() { *m = EndMarker{} } -func (*EndMarker) ProtoMessage() {} +func (m *EndMarker) Reset() { *m = EndMarker{} } +func (m *EndMarker) String() string { return proto.CompactTextString(m) } +func (*EndMarker) ProtoMessage() {} func (*EndMarker) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{7} } @@ -590,8 +594,9 @@ type LeaseStreamMessage struct { Event isLeaseStreamMessage_Event `protobuf_oneof:"event"` } -func (m *LeaseStreamMessage) Reset() { *m = LeaseStreamMessage{} } -func (*LeaseStreamMessage) ProtoMessage() {} +func (m *LeaseStreamMessage) Reset() { *m = LeaseStreamMessage{} } +func (m *LeaseStreamMessage) String() string { return proto.CompactTextString(m) } +func (*LeaseStreamMessage) ProtoMessage() {} func (*LeaseStreamMessage) Descriptor() ([]byte, []int) { return fileDescriptor_57e0d9d0e484e459, []int{8} } @@ -693,20 +698,20 @@ func (*LeaseStreamMessage) XXX_OneofWrappers() []interface{} { func init() { proto.RegisterType((*NodeInfo)(nil), "executorapi.NodeInfo") - proto.RegisterMapType((map[string]resource.Quantity)(nil), "executorapi.NodeInfo.AllocatableResourcesEntry") - proto.RegisterMapType((map[int32]ComputeResource)(nil), "executorapi.NodeInfo.AllocatedResourcesEntry") - proto.RegisterMapType((map[string]resource.Quantity)(nil), "executorapi.NodeInfo.AvailableResourcesEntry") + proto.RegisterMapType((map[string]*resource.Quantity)(nil), "executorapi.NodeInfo.AllocatableResourcesEntry") + proto.RegisterMapType((map[int32]*ComputeResource)(nil), "executorapi.NodeInfo.AllocatedResourcesEntry") + proto.RegisterMapType((map[string]*resource.Quantity)(nil), "executorapi.NodeInfo.AvailableResourcesEntry") proto.RegisterMapType((map[string]string)(nil), "executorapi.NodeInfo.LabelsEntry") - proto.RegisterMapType((map[int32]ComputeResource)(nil), "executorapi.NodeInfo.NonArmadaAllocatedResourcesEntry") + proto.RegisterMapType((map[int32]*ComputeResource)(nil), "executorapi.NodeInfo.NonArmadaAllocatedResourcesEntry") proto.RegisterMapType((map[string]*ComputeResource)(nil), "executorapi.NodeInfo.ResourceUsageByQueueEntry") proto.RegisterMapType((map[string]api.JobState)(nil), "executorapi.NodeInfo.RunIdsByStateEntry") - proto.RegisterMapType((map[string]resource.Quantity)(nil), "executorapi.NodeInfo.TotalResourcesEntry") + proto.RegisterMapType((map[string]*resource.Quantity)(nil), "executorapi.NodeInfo.TotalResourcesEntry") proto.RegisterType((*ComputeResource)(nil), "executorapi.ComputeResource") - proto.RegisterMapType((map[string]resource.Quantity)(nil), "executorapi.ComputeResource.ResourcesEntry") + proto.RegisterMapType((map[string]*resource.Quantity)(nil), "executorapi.ComputeResource.ResourcesEntry") proto.RegisterType((*EventList)(nil), "executorapi.EventList") proto.RegisterType((*LeaseRequest)(nil), "executorapi.LeaseRequest") - proto.RegisterMapType((map[string]resource.Quantity)(nil), "executorapi.LeaseRequest.MinimumJobSizeEntry") - proto.RegisterMapType((map[string]resource.Quantity)(nil), "executorapi.LeaseRequest.ResourcesEntry") + proto.RegisterMapType((map[string]*resource.Quantity)(nil), "executorapi.LeaseRequest.MinimumJobSizeEntry") + proto.RegisterMapType((map[string]*resource.Quantity)(nil), "executorapi.LeaseRequest.ResourcesEntry") proto.RegisterType((*JobRunLease)(nil), "executorapi.JobRunLease") proto.RegisterType((*CancelRuns)(nil), "executorapi.CancelRuns") proto.RegisterType((*PreemptRuns)(nil), "executorapi.PreemptRuns") @@ -717,103 +722,102 @@ func init() { func init() { proto.RegisterFile("pkg/executorapi/executorapi.proto", fileDescriptor_57e0d9d0e484e459) } var fileDescriptor_57e0d9d0e484e459 = []byte{ - // 1533 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0x4d, 0x6f, 0x1b, 0xc5, - 0x1b, 0xcf, 0xe6, 0xc5, 0x4d, 0xc6, 0x49, 0xda, 0x4c, 0xde, 0x36, 0x4e, 0xfe, 0x76, 0xea, 0x4a, - 0x7f, 0xa5, 0xa2, 0x5d, 0xd3, 0x14, 0xa1, 0x82, 0x00, 0x11, 0x57, 0x51, 0x9b, 0xa8, 0x2d, 0x74, - 0x93, 0x22, 0x8a, 0x90, 0xac, 0x59, 0xef, 0xd4, 0x59, 0xc7, 0xbb, 0xb3, 0xdd, 0x99, 0x4d, 0xeb, - 0x9e, 0x10, 0x07, 0x0e, 0x20, 0x24, 0x0e, 0x5c, 0x38, 0x54, 0xf0, 0x11, 0xf8, 0x10, 0x1c, 0x7a, - 0xac, 0xc4, 0xa5, 0x27, 0x8b, 0xa6, 0x37, 0x7f, 0x0a, 0x34, 0x2f, 0x1b, 0xcf, 0x3a, 0xeb, 0xa4, - 0x48, 0x48, 0xe4, 0x14, 0xcf, 0xf3, 0xf6, 0x7b, 0x5e, 0xe6, 0x79, 0xe6, 0xc9, 0x82, 0x8b, 0xe1, - 0x7e, 0xa3, 0x82, 0x9f, 0xe2, 0x7a, 0xcc, 0x48, 0x84, 0x42, 0x4f, 0xff, 0x6d, 0x85, 0x11, 0x61, - 0x04, 0xe6, 0x35, 0x52, 0xe1, 0x7f, 0x5c, 0x1e, 0x45, 0x3e, 0x72, 0x11, 0x3e, 0xc0, 0x01, 0xa3, - 0x15, 0xf9, 0x47, 0xca, 0x16, 0xe6, 0x04, 0x3b, 0xf4, 0x2a, 0x34, 0x76, 0x7c, 0x8f, 0x29, 0xea, - 0x72, 0x83, 0x90, 0x46, 0x0b, 0x57, 0xc4, 0xc9, 0x89, 0x1f, 0x55, 0xb0, 0x1f, 0xb2, 0xb6, 0x62, - 0x5e, 0x6d, 0x78, 0x6c, 0x2f, 0x76, 0xac, 0x3a, 0xf1, 0x2b, 0x0d, 0xd2, 0x20, 0x3d, 0x29, 0x7e, - 0x12, 0x07, 0xf1, 0x4b, 0x89, 0x97, 0xf7, 0x6f, 0x50, 0xcb, 0x23, 0x02, 0xa4, 0x4e, 0x22, 0x5c, - 0x39, 0xb8, 0x56, 0x69, 0xe0, 0x00, 0x47, 0x88, 0x61, 0x57, 0xc9, 0xbc, 0xd7, 0x93, 0xf1, 0x51, - 0x7d, 0xcf, 0x0b, 0x70, 0xd4, 0xae, 0x24, 0x9e, 0x45, 0x98, 0x92, 0x38, 0xaa, 0xe3, 0x7e, 0xad, - 0xf2, 0xef, 0x10, 0x8c, 0xdf, 0x23, 0x2e, 0xde, 0x0a, 0x1e, 0x11, 0xf8, 0x7f, 0x30, 0x1a, 0x20, - 0x1f, 0x9b, 0xc6, 0xaa, 0xb1, 0x36, 0x51, 0x85, 0xdd, 0x4e, 0x69, 0x9a, 0x9f, 0xaf, 0x10, 0xdf, - 0x63, 0xc2, 0x7b, 0x5b, 0xf0, 0xe1, 0x06, 0xc8, 0x31, 0xe4, 0x05, 0x8c, 0x9a, 0xc3, 0xab, 0x23, - 0x6b, 0xf9, 0xf5, 0x25, 0x4b, 0x62, 0x5b, 0x3c, 0x7f, 0xdc, 0x3f, 0xeb, 0xe0, 0x9a, 0xb5, 0xcb, - 0x25, 0xaa, 0xd3, 0x2f, 0x3a, 0xa5, 0xa1, 0x6e, 0xa7, 0xa4, 0x14, 0x6c, 0xf5, 0x17, 0x7e, 0x06, - 0x72, 0x2d, 0xe4, 0xe0, 0x16, 0x35, 0x47, 0x84, 0x89, 0x8b, 0x96, 0x5e, 0x83, 0xc4, 0x23, 0xeb, - 0x8e, 0x90, 0xd9, 0x0c, 0x58, 0xd4, 0xae, 0xce, 0x75, 0x3b, 0xa5, 0x0b, 0x52, 0x49, 0xf3, 0x48, - 0x99, 0x81, 0xdf, 0x1a, 0x60, 0x1e, 0xb5, 0x5a, 0xa4, 0x8e, 0x18, 0x72, 0x5a, 0xb8, 0x96, 0x44, - 0x4c, 0xcd, 0x51, 0x01, 0x50, 0xc9, 0x06, 0xd8, 0xe8, 0xa9, 0xd8, 0x89, 0x86, 0x84, 0x5b, 0x51, - 0x9e, 0xcf, 0xa1, 0x0c, 0x11, 0x3b, 0x93, 0x0a, 0x9f, 0x81, 0x59, 0x74, 0x80, 0xbc, 0x56, 0x9f, - 0x07, 0x63, 0xc2, 0x83, 0xab, 0x03, 0x3c, 0x48, 0x14, 0xfa, 0xf0, 0x0b, 0x0a, 0x1f, 0xa2, 0x63, - 0x02, 0x76, 0x06, 0x0d, 0x36, 0xc1, 0x79, 0x46, 0x18, 0x6a, 0x69, 0xb8, 0x39, 0x81, 0x7b, 0x39, - 0x1b, 0x77, 0x97, 0x0b, 0xf7, 0x61, 0x2e, 0x28, 0xcc, 0x69, 0x96, 0x62, 0xda, 0x7d, 0x67, 0x11, - 0xa7, 0x8c, 0x1f, 0xbb, 0x1a, 0xde, 0xb9, 0x13, 0xe3, 0x4c, 0x14, 0x06, 0xc6, 0x79, 0x4c, 0xc0, - 0xce, 0xa0, 0xc1, 0x3d, 0x70, 0x21, 0x8a, 0x83, 0x9a, 0xe7, 0xd2, 0x9a, 0xd3, 0xae, 0x51, 0x86, - 0x18, 0x36, 0xc7, 0x05, 0xf0, 0x5a, 0x36, 0xb0, 0x1d, 0x07, 0x5b, 0x2e, 0xad, 0xb6, 0x77, 0xb8, - 0xa8, 0xc4, 0x9c, 0x57, 0x98, 0x53, 0x91, 0xce, 0xb3, 0xd3, 0x47, 0xf8, 0x9b, 0x01, 0x8a, 0x01, - 0x09, 0x6a, 0xb2, 0xf3, 0x6b, 0x59, 0x11, 0x4f, 0x08, 0xe0, 0xf7, 0xb3, 0x81, 0xef, 0x91, 0x60, - 0x43, 0xa8, 0x0e, 0x0a, 0xfd, 0x92, 0x72, 0x63, 0x39, 0x18, 0x2c, 0x69, 0x9f, 0xc4, 0x84, 0x1b, - 0x60, 0x2a, 0x0e, 0x68, 0x7d, 0x0f, 0xbb, 0xb1, 0xb8, 0x0e, 0x26, 0x58, 0x35, 0xd6, 0xc6, 0xab, - 0xcb, 0xdd, 0x4e, 0x69, 0x31, 0xc5, 0xd0, 0x3a, 0x26, 0xad, 0x01, 0xbf, 0x37, 0xc0, 0x62, 0x12, - 0x50, 0x2d, 0xa6, 0xa8, 0x81, 0x79, 0x5e, 0x1f, 0xc7, 0x38, 0xc6, 0x66, 0xfe, 0xa4, 0xd6, 0x49, - 0xbc, 0x78, 0xc0, 0x75, 0xaa, 0xed, 0xfb, 0x5c, 0x43, 0xc6, 0x55, 0xee, 0x76, 0x4a, 0xc5, 0x28, - 0x83, 0xad, 0x79, 0x31, 0x97, 0xc5, 0x87, 0xd7, 0xc1, 0x44, 0x40, 0x5c, 0x5c, 0x63, 0xed, 0x10, - 0x9b, 0x93, 0x62, 0x0c, 0x2d, 0xf0, 0xbb, 0xc1, 0x89, 0xbb, 0xed, 0x50, 0x37, 0x30, 0x9e, 0xd0, - 0x0a, 0x08, 0xe4, 0xb5, 0x39, 0x01, 0x2f, 0x81, 0x91, 0x7d, 0xdc, 0x56, 0x43, 0x6c, 0x86, 0x57, - 0x79, 0x1f, 0xb7, 0x35, 0x45, 0xce, 0x85, 0x97, 0xc1, 0xd8, 0x01, 0x6a, 0xc5, 0xd8, 0x1c, 0x16, - 0x62, 0xb3, 0xdd, 0x4e, 0xe9, 0xbc, 0x20, 0x68, 0x82, 0x52, 0xe2, 0xc3, 0xe1, 0x1b, 0x46, 0xe1, - 0x57, 0x03, 0x2c, 0x0d, 0x1c, 0x15, 0x6f, 0x87, 0xf8, 0x50, 0x47, 0xcc, 0xaf, 0x5b, 0xda, 0xcc, - 0x3c, 0x9a, 0xd7, 0x56, 0xb8, 0xdf, 0x10, 0x43, 0x34, 0x49, 0x8e, 0x75, 0x3f, 0x46, 0x01, 0xf3, - 0x58, 0xfb, 0x54, 0x0f, 0x9f, 0x1b, 0x60, 0x71, 0xc0, 0x28, 0x39, 0x13, 0xfe, 0xfd, 0x62, 0x80, - 0xd9, 0x8c, 0x91, 0x73, 0x26, 0x7c, 0xfb, 0x81, 0xe7, 0x2e, 0xbb, 0x47, 0x75, 0xff, 0xc6, 0x06, - 0xfa, 0x77, 0x2b, 0xed, 0xdf, 0x4a, 0xaa, 0x61, 0x6e, 0x12, 0x3f, 0x8c, 0xd9, 0x51, 0x4d, 0x4e, - 0xf5, 0xe6, 0x09, 0x80, 0xc7, 0x47, 0xd6, 0xdb, 0xe5, 0xe9, 0x86, 0xee, 0xc7, 0xf4, 0xfa, 0x94, - 0xc8, 0xc7, 0x36, 0x71, 0x84, 0x9d, 0x53, 0x81, 0x7f, 0x36, 0xc0, 0xea, 0x69, 0x33, 0xeb, 0x3f, - 0xc8, 0xc7, 0x8f, 0x06, 0x58, 0x1a, 0x38, 0x6b, 0xde, 0x2e, 0x2f, 0xff, 0xa6, 0x3f, 0xe5, 0xef, - 0x86, 0xc1, 0xf9, 0x3e, 0x1d, 0xf8, 0x35, 0x98, 0xe8, 0x3d, 0x0a, 0x86, 0x98, 0x9a, 0xef, 0x9c, - 0x04, 0x62, 0xf5, 0xbd, 0x04, 0x33, 0xea, 0x25, 0xe8, 0x59, 0xb1, 0x7b, 0x3f, 0x79, 0x61, 0xa6, - 0xcf, 0x5e, 0xdb, 0x94, 0xbf, 0x00, 0x13, 0x9b, 0x7c, 0x0f, 0xbe, 0xe3, 0x51, 0x06, 0xb7, 0x40, - 0x4e, 0x2e, 0xc5, 0x2a, 0xfc, 0x65, 0x4b, 0x5f, 0x98, 0x2d, 0x21, 0xb8, 0x83, 0x1f, 0xc7, 0x38, - 0xa8, 0x63, 0xb9, 0xca, 0x49, 0x8e, 0xbe, 0xca, 0x49, 0x4a, 0xf9, 0x75, 0x0e, 0x4c, 0xde, 0xc1, - 0x88, 0x62, 0x9b, 0xcb, 0x53, 0x06, 0x3f, 0x00, 0x47, 0xeb, 0x78, 0xcd, 0x73, 0x55, 0xd0, 0x26, - 0xdf, 0xcd, 0x12, 0xf2, 0x96, 0xab, 0xd9, 0x01, 0x3d, 0x2a, 0x5f, 0x69, 0x43, 0x42, 0x5a, 0x6a, - 0xcc, 0x8b, 0x95, 0x96, 0x9f, 0xf5, 0x95, 0x96, 0x9f, 0xe1, 0x43, 0xbd, 0x80, 0x23, 0x19, 0xeb, - 0x84, 0xee, 0xd0, 0x3f, 0xaa, 0x1e, 0x24, 0xe0, 0x82, 0xef, 0x05, 0x9e, 0x1f, 0xfb, 0xb5, 0x26, - 0x71, 0x6a, 0xd4, 0x7b, 0x86, 0xd5, 0x4e, 0x7a, 0x75, 0x30, 0xc2, 0x5d, 0xa9, 0xc1, 0x7b, 0xd7, - 0x7b, 0x86, 0xfb, 0xb6, 0x33, 0x3f, 0xc5, 0xb4, 0xfb, 0xce, 0xf0, 0x53, 0x30, 0xc6, 0xdf, 0xc6, - 0x64, 0xef, 0x9c, 0xcf, 0x7c, 0xbe, 0x65, 0x75, 0x85, 0x9c, 0x5e, 0x5d, 0x41, 0x80, 0x2e, 0x58, - 0x88, 0x03, 0x44, 0xa9, 0xd7, 0x08, 0xb0, 0x2b, 0xbc, 0x56, 0x2b, 0x97, 0x5a, 0x29, 0x61, 0xba, - 0xb8, 0x0f, 0x62, 0xcf, 0xad, 0x2e, 0x2b, 0xef, 0x66, 0x7b, 0x9a, 0xdb, 0xc4, 0x91, 0x93, 0xcc, - 0xce, 0x22, 0xc2, 0x5b, 0x60, 0xc6, 0x47, 0x4f, 0xb9, 0x79, 0x5a, 0x63, 0xa4, 0xd6, 0xe2, 0xf1, - 0x9b, 0xe7, 0x56, 0x8d, 0xb5, 0xa9, 0xea, 0x4a, 0xb7, 0x53, 0x32, 0x7d, 0xf4, 0x74, 0x9b, 0x38, - 0x74, 0x97, 0x88, 0xcc, 0x68, 0x5e, 0x4e, 0xa7, 0x39, 0x67, 0xb4, 0x3f, 0xc4, 0x93, 0x97, 0x51, - 0xc7, 0x33, 0xd1, 0xbb, 0x7f, 0x0c, 0x83, 0xbc, 0xac, 0x84, 0x48, 0x21, 0xbc, 0x0d, 0x40, 0xaf, - 0xcc, 0xc2, 0xb5, 0xec, 0x2a, 0x8b, 0x6d, 0xac, 0xa9, 0x4a, 0xa8, 0x6f, 0x63, 0x09, 0x8d, 0x6f, - 0x56, 0x72, 0x79, 0xd4, 0x36, 0xab, 0xc7, 0x7d, 0xcb, 0x9f, 0x94, 0x80, 0x57, 0x40, 0x8e, 0x17, - 0x1f, 0x33, 0x73, 0x44, 0xc8, 0x8a, 0xb1, 0x20, 0x29, 0xfa, 0x58, 0x90, 0x14, 0xde, 0xca, 0x31, - 0xc5, 0x91, 0x39, 0xda, 0x6b, 0x65, 0x7e, 0xd6, 0x5b, 0x99, 0x9f, 0xb9, 0xd5, 0x46, 0x44, 0xe2, - 0x50, 0xde, 0x7f, 0x65, 0x55, 0x52, 0x74, 0xab, 0x92, 0x02, 0x3f, 0x02, 0x23, 0x4d, 0xe2, 0x98, - 0x39, 0x11, 0xf1, 0x62, 0x3a, 0xe2, 0x1d, 0xf1, 0xff, 0xfc, 0x36, 0x71, 0x64, 0x95, 0x9a, 0xc4, - 0xd1, 0xab, 0xd4, 0x24, 0x4e, 0x99, 0x02, 0x70, 0x13, 0x05, 0x75, 0xdc, 0xb2, 0xe3, 0x80, 0x42, - 0x0c, 0xe6, 0xb5, 0x5e, 0xe1, 0x77, 0xba, 0x2e, 0x98, 0x6a, 0x24, 0x66, 0xe5, 0xb3, 0xc4, 0xd7, - 0xff, 0x24, 0x77, 0x74, 0x97, 0x48, 0x6b, 0x1a, 0xcc, 0xcc, 0x31, 0x66, 0xf9, 0x09, 0xc8, 0x7f, - 0x1e, 0x61, 0xce, 0x16, 0xa8, 0x7b, 0x60, 0xa1, 0x0f, 0x35, 0x94, 0xdc, 0x13, 0x60, 0x57, 0xbb, - 0x9d, 0xd2, 0x8a, 0x66, 0x59, 0xd9, 0xd3, 0x70, 0xe1, 0x71, 0x6e, 0x39, 0x0f, 0x26, 0x36, 0x03, - 0xf7, 0x2e, 0x8a, 0xf6, 0x71, 0x54, 0xfe, 0x73, 0x18, 0x40, 0x71, 0x77, 0x76, 0x58, 0x84, 0x91, - 0x7f, 0x17, 0x53, 0xfe, 0x36, 0xc3, 0x4d, 0x30, 0x26, 0x1b, 0x59, 0xde, 0x21, 0x33, 0x35, 0x7c, - 0xb4, 0x1b, 0x27, 0x2f, 0x46, 0x2b, 0xdd, 0xd9, 0xb7, 0x87, 0x6c, 0xa9, 0x0d, 0x77, 0x41, 0x5e, - 0xe6, 0x8e, 0xc7, 0x45, 0x55, 0x13, 0x2c, 0xa6, 0x9f, 0xd4, 0xa3, 0xc4, 0xcb, 0xb7, 0xa0, 0x7e, - 0x74, 0x4e, 0x19, 0x04, 0x3d, 0x3a, 0xfc, 0x18, 0x8c, 0xe0, 0xc0, 0x15, 0xb7, 0x2d, 0xbf, 0xbe, - 0x90, 0xb2, 0x76, 0x14, 0x98, 0xac, 0x35, 0x0e, 0xdc, 0x94, 0x15, 0xae, 0x07, 0xbf, 0x04, 0x93, - 0x2a, 0xb5, 0xd2, 0xab, 0xd1, 0x8c, 0x10, 0xb5, 0xca, 0x54, 0x97, 0xba, 0x9d, 0xd2, 0x7c, 0xd8, - 0x23, 0xa4, 0x2c, 0xe6, 0x35, 0x46, 0xf5, 0x1c, 0x18, 0x13, 0xe5, 0x59, 0x7f, 0x6e, 0x80, 0xfc, - 0xa6, 0x32, 0xb7, 0x11, 0x7a, 0xf0, 0x9e, 0x7a, 0x0a, 0x65, 0xe6, 0x28, 0x5c, 0x1a, 0xf8, 0x64, - 0x14, 0x4a, 0xc7, 0x59, 0xa9, 0xd2, 0xac, 0x19, 0xef, 0x1a, 0xf0, 0x13, 0x30, 0x69, 0xe3, 0x90, - 0x44, 0x4c, 0x3c, 0xc8, 0x14, 0xf6, 0x25, 0x21, 0x79, 0xce, 0x0b, 0x0b, 0x96, 0xfc, 0x7c, 0x65, - 0x25, 0x1f, 0xa6, 0xac, 0x4d, 0xee, 0x77, 0xf5, 0xfe, 0xab, 0xd7, 0xc5, 0xa1, 0x6f, 0x0e, 0x8b, - 0xc6, 0x8b, 0xc3, 0xa2, 0xf1, 0xf2, 0xb0, 0x68, 0xfc, 0x75, 0x58, 0x34, 0x7e, 0x7a, 0x53, 0x1c, - 0x7a, 0xf9, 0xa6, 0x38, 0xf4, 0xea, 0x4d, 0x71, 0xe8, 0xab, 0x8a, 0xf6, 0x69, 0x4b, 0x5e, 0xbc, - 0x30, 0x22, 0x4d, 0x5c, 0x67, 0xea, 0x54, 0xe9, 0xfb, 0xf8, 0xe6, 0xe4, 0x04, 0xc4, 0xf5, 0xbf, - 0x03, 0x00, 0x00, 0xff, 0xff, 0x5e, 0x37, 0xd8, 0xe3, 0x96, 0x13, 0x00, 0x00, + // 1514 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x58, 0xcd, 0x6f, 0xdb, 0x46, + 0x16, 0x37, 0x2d, 0x5b, 0xb1, 0x47, 0xb6, 0x93, 0x8c, 0xbf, 0x68, 0x3b, 0x11, 0x1d, 0x2d, 0x76, + 0xe1, 0x60, 0x13, 0x6a, 0xe3, 0x2c, 0x8a, 0xb4, 0x68, 0x8b, 0x5a, 0x81, 0x91, 0xd8, 0x48, 0xd2, + 0x46, 0x76, 0x8a, 0xb6, 0x17, 0x62, 0x28, 0x4e, 0x64, 0xca, 0x22, 0x87, 0xe1, 0x0c, 0x9d, 0x28, + 0xa7, 0x1e, 0x8b, 0x7e, 0x00, 0x3d, 0xf4, 0xd2, 0x43, 0xd0, 0x5b, 0xff, 0x92, 0x1e, 0x7a, 0x0c, + 0xd0, 0x4b, 0x0f, 0x05, 0x51, 0x24, 0x37, 0x5e, 0xfb, 0x0f, 0x14, 0x33, 0x43, 0x5a, 0x43, 0x89, + 0xb2, 0x83, 0xa2, 0x40, 0x7d, 0xb2, 0xe7, 0xfd, 0xde, 0xf7, 0x9b, 0xf7, 0xe6, 0x89, 0xe0, 0x4a, + 0x70, 0xd8, 0xae, 0xe3, 0x67, 0xb8, 0x15, 0x31, 0x12, 0xa2, 0xc0, 0x55, 0xff, 0x37, 0x83, 0x90, + 0x30, 0x02, 0x2b, 0x0a, 0x69, 0xf5, 0x32, 0xe7, 0x47, 0xa1, 0x87, 0x1c, 0x84, 0x8f, 0xb0, 0xcf, + 0x68, 0x5d, 0xfe, 0x91, 0xbc, 0xab, 0x0b, 0x02, 0x0e, 0xdc, 0x3a, 0x8d, 0x6c, 0xcf, 0x65, 0x29, + 0x75, 0xad, 0x4d, 0x48, 0xbb, 0x8b, 0xeb, 0xe2, 0x64, 0x47, 0x8f, 0xeb, 0xd8, 0x0b, 0x58, 0x2f, + 0x05, 0x6b, 0x87, 0xb7, 0xa8, 0xe9, 0x12, 0x21, 0xd5, 0x22, 0x21, 0xae, 0x1f, 0xdd, 0xa8, 0xb7, + 0xb1, 0x8f, 0x43, 0xc4, 0xb0, 0x93, 0xf2, 0xfc, 0xbf, 0xcf, 0xe3, 0xa1, 0xd6, 0x81, 0xeb, 0xe3, + 0xb0, 0x57, 0xcf, 0x4c, 0x85, 0x98, 0x92, 0x28, 0x6c, 0xe1, 0x41, 0xa9, 0xda, 0x6f, 0x10, 0x4c, + 0x3d, 0x20, 0x0e, 0xde, 0xf1, 0x1f, 0x13, 0xf8, 0x1f, 0x30, 0xe1, 0x23, 0x0f, 0xeb, 0xda, 0xba, + 0xb6, 0x31, 0xdd, 0x80, 0x49, 0x6c, 0xcc, 0xf1, 0xf3, 0x35, 0xe2, 0xb9, 0x4c, 0xb8, 0xd3, 0x14, + 0x38, 0xbc, 0x03, 0xca, 0x0c, 0xb9, 0x3e, 0xa3, 0xfa, 0xf8, 0x7a, 0x69, 0xa3, 0xb2, 0xb9, 0x62, + 0x4a, 0xdb, 0x26, 0x4f, 0x08, 0xf7, 0xcf, 0x3c, 0xba, 0x61, 0xee, 0x73, 0x8e, 0xc6, 0x42, 0x12, + 0x1b, 0x17, 0x24, 0xb3, 0xa2, 0x26, 0x15, 0x87, 0x1f, 0x82, 0x72, 0x17, 0xd9, 0xb8, 0x4b, 0xf5, + 0x92, 0x50, 0x74, 0xc5, 0x54, 0x53, 0x9b, 0xf9, 0x65, 0xde, 0x13, 0x3c, 0xdb, 0x3e, 0x0b, 0x7b, + 0x52, 0xa1, 0x14, 0x52, 0x15, 0x4a, 0x0a, 0xfc, 0x42, 0x03, 0x8b, 0xa8, 0xdb, 0x25, 0x2d, 0xc4, + 0x90, 0xdd, 0xc5, 0x56, 0x16, 0x37, 0xd5, 0x27, 0x84, 0x81, 0x7a, 0xb1, 0x81, 0xad, 0xbe, 0x48, + 0x33, 0x93, 0x90, 0xe6, 0x6a, 0x49, 0x6c, 0x54, 0x51, 0x01, 0xac, 0x18, 0x5f, 0x28, 0xc2, 0xe1, + 0xe7, 0x1a, 0x98, 0x47, 0x47, 0xc8, 0xed, 0x0e, 0x38, 0x32, 0x29, 0x1c, 0xb9, 0x3e, 0xc2, 0x91, + 0x4c, 0x60, 0xc0, 0x8d, 0xf5, 0x24, 0x36, 0x2e, 0xa1, 0x21, 0x50, 0x71, 0x02, 0x0e, 0xa3, 0x30, + 0x00, 0xe7, 0x19, 0x61, 0xa8, 0xab, 0x58, 0x2f, 0x0b, 0xeb, 0x57, 0x8b, 0xad, 0xef, 0x73, 0xe6, + 0x01, 0xcb, 0x97, 0x92, 0xd8, 0xd0, 0x59, 0x0e, 0x50, 0xac, 0xce, 0xe5, 0x11, 0x19, 0xb4, 0xcc, + 0x06, 0x76, 0x14, 0xb3, 0xe7, 0x4e, 0x0c, 0x3a, 0x13, 0x28, 0x0c, 0x7a, 0x08, 0xcc, 0x05, 0x3d, + 0x84, 0x42, 0x1f, 0x5c, 0x08, 0x23, 0xdf, 0x72, 0x1d, 0x6a, 0xd9, 0x3d, 0x8b, 0x32, 0xc4, 0xb0, + 0x3e, 0x25, 0xcc, 0x6f, 0x14, 0x9b, 0x6f, 0x46, 0xfe, 0x8e, 0x43, 0x1b, 0xbd, 0x3d, 0xce, 0x2a, + 0x2d, 0xaf, 0x25, 0xb1, 0xb1, 0x1c, 0xaa, 0x74, 0xc5, 0xe8, 0x6c, 0x0e, 0x80, 0x3f, 0x6a, 0xa0, + 0xea, 0x13, 0xdf, 0x92, 0x0d, 0x6f, 0x15, 0x45, 0x3f, 0x2d, 0xcc, 0xbf, 0x55, 0x6c, 0xfe, 0x01, + 0xf1, 0xb7, 0x84, 0xe8, 0xa8, 0x34, 0x5c, 0x4d, 0x62, 0xe3, 0xdf, 0xfe, 0x68, 0x2e, 0xc5, 0xb5, + 0xb5, 0x13, 0xd8, 0xe0, 0x16, 0x98, 0x8d, 0x7c, 0xda, 0x3a, 0xc0, 0x4e, 0x24, 0xee, 0x89, 0x0e, + 0xd6, 0xb5, 0x8d, 0x29, 0x19, 0x6b, 0x0e, 0x50, 0x63, 0xcd, 0x01, 0xf0, 0x4b, 0x0d, 0x2c, 0x67, + 0x61, 0x59, 0x11, 0x45, 0x6d, 0xcc, 0x73, 0xfc, 0x24, 0xc2, 0x11, 0xd6, 0x2b, 0x27, 0x35, 0x58, + 0xe6, 0xc5, 0x23, 0x2e, 0xd3, 0xe8, 0x3d, 0xe4, 0x12, 0x4a, 0x83, 0x85, 0x05, 0xb0, 0xda, 0x60, + 0x45, 0x38, 0xbc, 0x09, 0xa6, 0x7d, 0xe2, 0x60, 0x8b, 0xf5, 0x02, 0xac, 0xcf, 0x88, 0x91, 0xb5, + 0x94, 0xc4, 0x06, 0xe4, 0xc4, 0xfd, 0x5e, 0xa0, 0x2a, 0x98, 0xca, 0x68, 0xab, 0x08, 0x54, 0x94, + 0x69, 0x02, 0xff, 0x05, 0x4a, 0x87, 0xb8, 0x97, 0x0e, 0xbc, 0x8b, 0x49, 0x6c, 0xcc, 0x1e, 0xe2, + 0x9e, 0x22, 0xc8, 0x51, 0x78, 0x15, 0x4c, 0x1e, 0xa1, 0x6e, 0x84, 0xf5, 0x71, 0xc1, 0x36, 0x9f, + 0xc4, 0xc6, 0x79, 0x41, 0x50, 0x18, 0x25, 0xc7, 0x3b, 0xe3, 0xb7, 0xb4, 0xd5, 0x1f, 0x34, 0xb0, + 0x32, 0x72, 0xa0, 0xbc, 0x99, 0xc5, 0x4f, 0x55, 0x8b, 0x95, 0x4d, 0x53, 0x99, 0xaf, 0xc7, 0xb3, + 0xdd, 0x0c, 0x0e, 0xdb, 0x62, 0xe0, 0x66, 0xc9, 0x31, 0x1f, 0x46, 0xc8, 0x67, 0x2e, 0xeb, 0x9d, + 0xea, 0xe1, 0x0b, 0x0d, 0x2c, 0x8f, 0x98, 0x34, 0x67, 0xc2, 0xbf, 0xef, 0x35, 0x30, 0x5f, 0x30, + 0x8b, 0xce, 0x84, 0x6f, 0x5f, 0xf1, 0xdc, 0x15, 0x77, 0xaa, 0xea, 0xdf, 0xe4, 0x48, 0xff, 0xee, + 0xe4, 0xfd, 0xbb, 0x94, 0x6b, 0x98, 0xdb, 0xc4, 0x0b, 0x22, 0x76, 0x5c, 0x93, 0x53, 0xbd, 0x79, + 0x0a, 0xe0, 0xf0, 0xf8, 0x7a, 0xb3, 0x3c, 0xdd, 0x52, 0xfd, 0x98, 0xdb, 0x9c, 0x15, 0xf9, 0xd8, + 0x25, 0xb6, 0xd0, 0x73, 0xaa, 0xe1, 0xef, 0x34, 0xb0, 0x7e, 0xda, 0xe4, 0xfa, 0x07, 0xf2, 0xf1, + 0x8d, 0x06, 0x56, 0x46, 0xce, 0x9a, 0x37, 0xcb, 0xcb, 0xdf, 0xe9, 0x4f, 0xed, 0xeb, 0x71, 0x70, + 0x7e, 0x40, 0x06, 0xda, 0x60, 0xba, 0xff, 0x34, 0x68, 0x62, 0x6a, 0xfe, 0xf7, 0x24, 0x23, 0xe6, + 0xc0, 0x7b, 0xb0, 0x9c, 0xc4, 0xc6, 0x7c, 0x58, 0x30, 0xfd, 0xfb, 0x6a, 0x79, 0x79, 0xe6, 0xce, + 0x5e, 0xf3, 0xd4, 0x3e, 0x06, 0xd3, 0xdb, 0x7c, 0x15, 0xbe, 0xe7, 0x52, 0x06, 0x77, 0x40, 0x59, + 0xee, 0xc5, 0x69, 0x12, 0xd6, 0x4c, 0x75, 0x67, 0x36, 0x05, 0xe3, 0x1e, 0x7e, 0x12, 0x61, 0xbf, + 0x85, 0xe5, 0xda, 0x27, 0x11, 0x75, 0xed, 0x93, 0x94, 0xda, 0x1f, 0x65, 0x30, 0x73, 0x0f, 0x23, + 0x8a, 0x9b, 0x9c, 0x9f, 0x32, 0xf8, 0x36, 0x38, 0xde, 0xc8, 0x2d, 0xd7, 0x49, 0x83, 0xd6, 0x93, + 0xd8, 0x58, 0xc8, 0xc8, 0x3b, 0x8e, 0xa2, 0x07, 0xf4, 0xa9, 0x7c, 0x09, 0x0e, 0x08, 0xe9, 0xa6, + 0xc3, 0x5e, 0x2c, 0xc1, 0xfc, 0xac, 0x2e, 0xc1, 0xfc, 0x0c, 0x2d, 0xb5, 0x8c, 0xa5, 0x82, 0x05, + 0x43, 0x75, 0xe8, 0x2f, 0xd4, 0x10, 0x46, 0xe0, 0x82, 0xe7, 0xfa, 0xae, 0x17, 0x79, 0x56, 0x87, + 0xd8, 0x16, 0x75, 0x9f, 0xe3, 0x74, 0x8b, 0xbd, 0x3e, 0xda, 0xce, 0x7d, 0x29, 0xc1, 0xfb, 0xd8, + 0x7d, 0x8e, 0x95, 0x15, 0xce, 0xcb, 0x01, 0xea, 0x0a, 0x97, 0x47, 0xe0, 0x07, 0x60, 0x92, 0xbf, + 0x96, 0xd9, 0xa2, 0xba, 0x58, 0xf8, 0xa0, 0xcb, 0x4a, 0x0b, 0x3e, 0xb5, 0xd2, 0x82, 0x00, 0x3b, + 0x60, 0x29, 0xf2, 0x11, 0xa5, 0x6e, 0xdb, 0xc7, 0x8e, 0xf0, 0x3d, 0x5d, 0xc8, 0xd2, 0xed, 0x13, + 0xe6, 0x0b, 0xfd, 0x28, 0x72, 0x9d, 0xc6, 0x95, 0x24, 0x36, 0x2e, 0xf7, 0xa5, 0x76, 0x89, 0x2d, + 0xe7, 0x9a, 0xa2, 0x7d, 0xbe, 0x00, 0x86, 0x77, 0xc0, 0x45, 0x0f, 0x3d, 0xe3, 0x46, 0xa8, 0xc5, + 0x88, 0xd5, 0xe5, 0xb9, 0xd0, 0xcf, 0xad, 0x6b, 0x1b, 0xb3, 0x69, 0xd8, 0xe8, 0xd9, 0x2e, 0xb1, + 0xe9, 0x3e, 0x11, 0x59, 0xca, 0x85, 0x9d, 0x43, 0xce, 0x68, 0xc7, 0x88, 0xa7, 0xb0, 0xa0, 0xa6, + 0x67, 0xa2, 0x9b, 0x7f, 0x1a, 0x07, 0x15, 0x59, 0x09, 0x91, 0x42, 0x78, 0x17, 0x80, 0x7e, 0xb1, + 0x85, 0x6b, 0xc5, 0xb5, 0x16, 0x5b, 0x5a, 0x27, 0x2d, 0xa1, 0xba, 0xa5, 0x65, 0x34, 0xbe, 0x71, + 0xc9, 0xa5, 0x52, 0xd9, 0xb8, 0x9e, 0x0c, 0x2c, 0x85, 0x92, 0x03, 0x5e, 0x03, 0x65, 0x5e, 0x7c, + 0xcc, 0xf4, 0x92, 0xe0, 0x15, 0x83, 0x42, 0x52, 0xd4, 0x41, 0x21, 0x29, 0xbc, 0xb9, 0x23, 0x8a, + 0x43, 0x7d, 0xa2, 0xdf, 0xdc, 0xfc, 0xac, 0x36, 0x37, 0x3f, 0x73, 0xad, 0xed, 0x90, 0x44, 0x81, + 0xec, 0x82, 0x54, 0xab, 0xa4, 0xa8, 0x5a, 0x25, 0x05, 0xbe, 0x0b, 0x4a, 0x1d, 0x62, 0xeb, 0x65, + 0x11, 0xf1, 0x72, 0x3e, 0xe2, 0x3d, 0xf1, 0x23, 0x7f, 0x97, 0xd8, 0xb2, 0x4a, 0x1d, 0x62, 0xab, + 0x55, 0xea, 0x10, 0xbb, 0x46, 0x01, 0xb8, 0x8d, 0xfc, 0x16, 0xee, 0x36, 0x23, 0x9f, 0x42, 0x0c, + 0x16, 0x95, 0x8e, 0xe1, 0x77, 0xba, 0x25, 0xc0, 0x74, 0x48, 0x16, 0xe5, 0xd3, 0x48, 0x62, 0x63, + 0x2d, 0xcb, 0x1d, 0xdd, 0x27, 0x52, 0x9b, 0x62, 0xe6, 0xe2, 0x10, 0x58, 0x7b, 0x0a, 0x2a, 0x1f, + 0x85, 0x98, 0xc3, 0xc2, 0xea, 0x01, 0x58, 0x1a, 0xb0, 0x1a, 0x48, 0xf4, 0x04, 0xb3, 0xe2, 0xe7, + 0x99, 0xa2, 0x39, 0xd5, 0xa7, 0xfe, 0x3c, 0x1b, 0x46, 0x6b, 0x15, 0x30, 0xbd, 0xed, 0x3b, 0xf7, + 0x51, 0x78, 0x88, 0xc3, 0xda, 0x2f, 0xe3, 0x00, 0x8a, 0xbb, 0xb3, 0xc7, 0x42, 0x8c, 0xbc, 0xfb, + 0x98, 0xf2, 0x37, 0x1b, 0x6e, 0x83, 0x49, 0xd9, 0xc8, 0xf2, 0x0e, 0xe9, 0xb9, 0x11, 0xa4, 0xdc, + 0x38, 0x79, 0x31, 0xba, 0xf9, 0xce, 0xbe, 0x3b, 0xd6, 0x94, 0xd2, 0x70, 0x1f, 0x54, 0x64, 0xee, + 0x78, 0x5c, 0x34, 0x6d, 0x82, 0xe5, 0xfc, 0x53, 0x7b, 0x9c, 0x78, 0xf9, 0x3a, 0xb4, 0x8e, 0xcf, + 0x39, 0x85, 0xa0, 0x4f, 0x87, 0xef, 0x81, 0x12, 0xf6, 0x1d, 0x71, 0xdb, 0x2a, 0x9b, 0x4b, 0x39, + 0x6d, 0xc7, 0x81, 0xc9, 0x5a, 0x63, 0xdf, 0xc9, 0x69, 0xe1, 0x72, 0xf0, 0x13, 0x30, 0x93, 0xa6, + 0x56, 0x7a, 0x35, 0x51, 0x10, 0xa2, 0x52, 0x99, 0xc6, 0x4a, 0x12, 0x1b, 0x8b, 0x41, 0x9f, 0x90, + 0xd3, 0x58, 0x51, 0x80, 0xc6, 0x39, 0x30, 0x29, 0xca, 0xb3, 0xf9, 0x42, 0x03, 0x95, 0xed, 0x54, + 0xdd, 0x56, 0xe0, 0xc2, 0x07, 0xe9, 0xe3, 0x28, 0x33, 0x47, 0xe1, 0xca, 0xc8, 0xe7, 0x63, 0xd5, + 0x18, 0x86, 0x72, 0xa5, 0xd9, 0xd0, 0xfe, 0xa7, 0xc1, 0xf7, 0xc1, 0x4c, 0x13, 0x07, 0x24, 0x64, + 0xe2, 0x89, 0xa6, 0x70, 0x20, 0x09, 0xd9, 0x03, 0xbf, 0xba, 0x64, 0xca, 0x6f, 0x5a, 0x66, 0xf6, + 0x4d, 0xcb, 0xdc, 0xe6, 0x7e, 0x37, 0x76, 0x7e, 0x7e, 0x55, 0xd5, 0x5e, 0xbe, 0xaa, 0x6a, 0xbf, + 0xbf, 0xaa, 0x6a, 0xdf, 0xbe, 0xae, 0x8e, 0xbd, 0x7c, 0x5d, 0x1d, 0xfb, 0xf5, 0x75, 0x75, 0xec, + 0xb3, 0x7a, 0xdb, 0x65, 0x07, 0x91, 0x6d, 0xb6, 0x88, 0x97, 0x7e, 0x40, 0x0b, 0x42, 0xd2, 0xc1, + 0x2d, 0x96, 0x9e, 0xea, 0x03, 0x5f, 0xe2, 0xec, 0xb2, 0x50, 0x7d, 0xf3, 0xcf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x3e, 0x88, 0x10, 0xeb, 0xa3, 0x13, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1046,16 +1050,18 @@ func (m *NodeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.NonArmadaAllocatedResources { v := m.NonArmadaAllocatedResources[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i = encodeVarintExecutorapi(dAtA, i, uint64(k)) i-- dAtA[i] = 0x8 @@ -1085,16 +1091,18 @@ func (m *NodeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.AllocatedResources { v := m.AllocatedResources[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i = encodeVarintExecutorapi(dAtA, i, uint64(k)) i-- dAtA[i] = 0x8 @@ -1107,16 +1115,18 @@ func (m *NodeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.TotalResources { v := m.TotalResources[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i -= len(k) copy(dAtA[i:], k) i = encodeVarintExecutorapi(dAtA, i, uint64(len(k))) @@ -1131,16 +1141,18 @@ func (m *NodeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.AvailableResources { v := m.AvailableResources[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i -= len(k) copy(dAtA[i:], k) i = encodeVarintExecutorapi(dAtA, i, uint64(len(k))) @@ -1155,16 +1167,18 @@ func (m *NodeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.AllocatableResources { v := m.AllocatableResources[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i -= len(k) copy(dAtA[i:], k) i = encodeVarintExecutorapi(dAtA, i, uint64(len(k))) @@ -1242,16 +1256,18 @@ func (m *ComputeResource) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.Resources { v := m.Resources[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i -= len(k) copy(dAtA[i:], k) i = encodeVarintExecutorapi(dAtA, i, uint64(len(k))) @@ -1359,16 +1375,18 @@ func (m *LeaseRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.MinimumJobSize { v := m.MinimumJobSize[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i -= len(k) copy(dAtA[i:], k) i = encodeVarintExecutorapi(dAtA, i, uint64(len(k))) @@ -1383,16 +1401,18 @@ func (m *LeaseRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { for k := range m.Resources { v := m.Resources[k] baseI := i - { - size, err := (&v).MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintExecutorapi(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintExecutorapi(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x12 i -= len(k) copy(dAtA[i:], k) i = encodeVarintExecutorapi(dAtA, i, uint64(len(k))) @@ -1749,8 +1769,12 @@ func (m *NodeInfo) Size() (n int) { for k, v := range m.AllocatableResources { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -1758,8 +1782,12 @@ func (m *NodeInfo) Size() (n int) { for k, v := range m.AvailableResources { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -1767,8 +1795,12 @@ func (m *NodeInfo) Size() (n int) { for k, v := range m.TotalResources { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -1776,8 +1808,12 @@ func (m *NodeInfo) Size() (n int) { for k, v := range m.AllocatedResources { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + sovExecutorapi(uint64(k)) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + sovExecutorapi(uint64(k)) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -1793,8 +1829,12 @@ func (m *NodeInfo) Size() (n int) { for k, v := range m.NonArmadaAllocatedResources { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + sovExecutorapi(uint64(k)) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + sovExecutorapi(uint64(k)) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -1831,8 +1871,12 @@ func (m *ComputeResource) Size() (n int) { for k, v := range m.Resources { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -1872,8 +1916,12 @@ func (m *LeaseRequest) Size() (n int) { for k, v := range m.Resources { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -1881,8 +1929,12 @@ func (m *LeaseRequest) Size() (n int) { for k, v := range m.MinimumJobSize { _ = k _ = v - l = v.Size() - mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + 1 + l + sovExecutorapi(uint64(l)) + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovExecutorapi(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovExecutorapi(uint64(len(k))) + l n += mapEntrySize + 1 + sovExecutorapi(uint64(mapEntrySize)) } } @@ -2045,305 +2097,6 @@ func sovExecutorapi(x uint64) (n int) { func sozExecutorapi(x uint64) (n int) { return sovExecutorapi(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (this *NodeInfo) String() string { - if this == nil { - return "nil" - } - repeatedStringForTaints := "[]Taint{" - for _, f := range this.Taints { - repeatedStringForTaints += fmt.Sprintf("%v", f) + "," - } - repeatedStringForTaints += "}" - keysForLabels := make([]string, 0, len(this.Labels)) - for k, _ := range this.Labels { - keysForLabels = append(keysForLabels, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForLabels) - mapStringForLabels := "map[string]string{" - for _, k := range keysForLabels { - mapStringForLabels += fmt.Sprintf("%v: %v,", k, this.Labels[k]) - } - mapStringForLabels += "}" - keysForAllocatableResources := make([]string, 0, len(this.AllocatableResources)) - for k, _ := range this.AllocatableResources { - keysForAllocatableResources = append(keysForAllocatableResources, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForAllocatableResources) - mapStringForAllocatableResources := "map[string]resource.Quantity{" - for _, k := range keysForAllocatableResources { - mapStringForAllocatableResources += fmt.Sprintf("%v: %v,", k, this.AllocatableResources[k]) - } - mapStringForAllocatableResources += "}" - keysForAvailableResources := make([]string, 0, len(this.AvailableResources)) - for k, _ := range this.AvailableResources { - keysForAvailableResources = append(keysForAvailableResources, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForAvailableResources) - mapStringForAvailableResources := "map[string]resource.Quantity{" - for _, k := range keysForAvailableResources { - mapStringForAvailableResources += fmt.Sprintf("%v: %v,", k, this.AvailableResources[k]) - } - mapStringForAvailableResources += "}" - keysForTotalResources := make([]string, 0, len(this.TotalResources)) - for k, _ := range this.TotalResources { - keysForTotalResources = append(keysForTotalResources, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForTotalResources) - mapStringForTotalResources := "map[string]resource.Quantity{" - for _, k := range keysForTotalResources { - mapStringForTotalResources += fmt.Sprintf("%v: %v,", k, this.TotalResources[k]) - } - mapStringForTotalResources += "}" - keysForAllocatedResources := make([]int32, 0, len(this.AllocatedResources)) - for k, _ := range this.AllocatedResources { - keysForAllocatedResources = append(keysForAllocatedResources, k) - } - github_com_gogo_protobuf_sortkeys.Int32s(keysForAllocatedResources) - mapStringForAllocatedResources := "map[int32]ComputeResource{" - for _, k := range keysForAllocatedResources { - mapStringForAllocatedResources += fmt.Sprintf("%v: %v,", k, this.AllocatedResources[k]) - } - mapStringForAllocatedResources += "}" - keysForRunIdsByState := make([]string, 0, len(this.RunIdsByState)) - for k, _ := range this.RunIdsByState { - keysForRunIdsByState = append(keysForRunIdsByState, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForRunIdsByState) - mapStringForRunIdsByState := "map[string]api.JobState{" - for _, k := range keysForRunIdsByState { - mapStringForRunIdsByState += fmt.Sprintf("%v: %v,", k, this.RunIdsByState[k]) - } - mapStringForRunIdsByState += "}" - keysForNonArmadaAllocatedResources := make([]int32, 0, len(this.NonArmadaAllocatedResources)) - for k, _ := range this.NonArmadaAllocatedResources { - keysForNonArmadaAllocatedResources = append(keysForNonArmadaAllocatedResources, k) - } - github_com_gogo_protobuf_sortkeys.Int32s(keysForNonArmadaAllocatedResources) - mapStringForNonArmadaAllocatedResources := "map[int32]ComputeResource{" - for _, k := range keysForNonArmadaAllocatedResources { - mapStringForNonArmadaAllocatedResources += fmt.Sprintf("%v: %v,", k, this.NonArmadaAllocatedResources[k]) - } - mapStringForNonArmadaAllocatedResources += "}" - keysForResourceUsageByQueue := make([]string, 0, len(this.ResourceUsageByQueue)) - for k, _ := range this.ResourceUsageByQueue { - keysForResourceUsageByQueue = append(keysForResourceUsageByQueue, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForResourceUsageByQueue) - mapStringForResourceUsageByQueue := "map[string]*ComputeResource{" - for _, k := range keysForResourceUsageByQueue { - mapStringForResourceUsageByQueue += fmt.Sprintf("%v: %v,", k, this.ResourceUsageByQueue[k]) - } - mapStringForResourceUsageByQueue += "}" - s := strings.Join([]string{`&NodeInfo{`, - `Name:` + fmt.Sprintf("%v", this.Name) + `,`, - `Taints:` + repeatedStringForTaints + `,`, - `Labels:` + mapStringForLabels + `,`, - `AllocatableResources:` + mapStringForAllocatableResources + `,`, - `AvailableResources:` + mapStringForAvailableResources + `,`, - `TotalResources:` + mapStringForTotalResources + `,`, - `AllocatedResources:` + mapStringForAllocatedResources + `,`, - `RunIdsByState:` + mapStringForRunIdsByState + `,`, - `NonArmadaAllocatedResources:` + mapStringForNonArmadaAllocatedResources + `,`, - `Unschedulable:` + fmt.Sprintf("%v", this.Unschedulable) + `,`, - `ResourceUsageByQueue:` + mapStringForResourceUsageByQueue + `,`, - `NodeType:` + fmt.Sprintf("%v", this.NodeType) + `,`, - `}`, - }, "") - return s -} -func (this *ComputeResource) String() string { - if this == nil { - return "nil" - } - keysForResources := make([]string, 0, len(this.Resources)) - for k, _ := range this.Resources { - keysForResources = append(keysForResources, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForResources) - mapStringForResources := "map[string]resource.Quantity{" - for _, k := range keysForResources { - mapStringForResources += fmt.Sprintf("%v: %v,", k, this.Resources[k]) - } - mapStringForResources += "}" - s := strings.Join([]string{`&ComputeResource{`, - `Resources:` + mapStringForResources + `,`, - `}`, - }, "") - return s -} -func (this *EventList) String() string { - if this == nil { - return "nil" - } - repeatedStringForEvents := "[]*EventSequence{" - for _, f := range this.Events { - repeatedStringForEvents += strings.Replace(fmt.Sprintf("%v", f), "EventSequence", "armadaevents.EventSequence", 1) + "," - } - repeatedStringForEvents += "}" - s := strings.Join([]string{`&EventList{`, - `Events:` + repeatedStringForEvents + `,`, - `}`, - }, "") - return s -} -func (this *LeaseRequest) String() string { - if this == nil { - return "nil" - } - repeatedStringForNodes := "[]*NodeInfo{" - for _, f := range this.Nodes { - repeatedStringForNodes += strings.Replace(f.String(), "NodeInfo", "NodeInfo", 1) + "," - } - repeatedStringForNodes += "}" - repeatedStringForUnassignedJobRunIds := "[]Uuid{" - for _, f := range this.UnassignedJobRunIds { - repeatedStringForUnassignedJobRunIds += fmt.Sprintf("%v", f) + "," - } - repeatedStringForUnassignedJobRunIds += "}" - keysForResources := make([]string, 0, len(this.Resources)) - for k, _ := range this.Resources { - keysForResources = append(keysForResources, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForResources) - mapStringForResources := "map[string]resource.Quantity{" - for _, k := range keysForResources { - mapStringForResources += fmt.Sprintf("%v: %v,", k, this.Resources[k]) - } - mapStringForResources += "}" - keysForMinimumJobSize := make([]string, 0, len(this.MinimumJobSize)) - for k, _ := range this.MinimumJobSize { - keysForMinimumJobSize = append(keysForMinimumJobSize, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForMinimumJobSize) - mapStringForMinimumJobSize := "map[string]resource.Quantity{" - for _, k := range keysForMinimumJobSize { - mapStringForMinimumJobSize += fmt.Sprintf("%v: %v,", k, this.MinimumJobSize[k]) - } - mapStringForMinimumJobSize += "}" - s := strings.Join([]string{`&LeaseRequest{`, - `ExecutorId:` + fmt.Sprintf("%v", this.ExecutorId) + `,`, - `Pool:` + fmt.Sprintf("%v", this.Pool) + `,`, - `Resources:` + mapStringForResources + `,`, - `MinimumJobSize:` + mapStringForMinimumJobSize + `,`, - `Nodes:` + repeatedStringForNodes + `,`, - `UnassignedJobRunIds:` + repeatedStringForUnassignedJobRunIds + `,`, - `MaxJobsToLease:` + fmt.Sprintf("%v", this.MaxJobsToLease) + `,`, - `}`, - }, "") - return s -} -func (this *JobRunLease) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&JobRunLease{`, - `JobRunId:` + strings.Replace(fmt.Sprintf("%v", this.JobRunId), "Uuid", "armadaevents.Uuid", 1) + `,`, - `Queue:` + fmt.Sprintf("%v", this.Queue) + `,`, - `Jobset:` + fmt.Sprintf("%v", this.Jobset) + `,`, - `User:` + fmt.Sprintf("%v", this.User) + `,`, - `Groups:` + fmt.Sprintf("%v", this.Groups) + `,`, - `Job:` + strings.Replace(fmt.Sprintf("%v", this.Job), "SubmitJob", "armadaevents.SubmitJob", 1) + `,`, - `}`, - }, "") - return s -} -func (this *CancelRuns) String() string { - if this == nil { - return "nil" - } - repeatedStringForJobRunIdsToCancel := "[]*Uuid{" - for _, f := range this.JobRunIdsToCancel { - repeatedStringForJobRunIdsToCancel += strings.Replace(fmt.Sprintf("%v", f), "Uuid", "armadaevents.Uuid", 1) + "," - } - repeatedStringForJobRunIdsToCancel += "}" - s := strings.Join([]string{`&CancelRuns{`, - `JobRunIdsToCancel:` + repeatedStringForJobRunIdsToCancel + `,`, - `}`, - }, "") - return s -} -func (this *PreemptRuns) String() string { - if this == nil { - return "nil" - } - repeatedStringForJobRunIdsToPreempt := "[]*Uuid{" - for _, f := range this.JobRunIdsToPreempt { - repeatedStringForJobRunIdsToPreempt += strings.Replace(fmt.Sprintf("%v", f), "Uuid", "armadaevents.Uuid", 1) + "," - } - repeatedStringForJobRunIdsToPreempt += "}" - s := strings.Join([]string{`&PreemptRuns{`, - `JobRunIdsToPreempt:` + repeatedStringForJobRunIdsToPreempt + `,`, - `}`, - }, "") - return s -} -func (this *EndMarker) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&EndMarker{`, - `}`, - }, "") - return s -} -func (this *LeaseStreamMessage) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&LeaseStreamMessage{`, - `Event:` + fmt.Sprintf("%v", this.Event) + `,`, - `}`, - }, "") - return s -} -func (this *LeaseStreamMessage_Lease) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&LeaseStreamMessage_Lease{`, - `Lease:` + strings.Replace(fmt.Sprintf("%v", this.Lease), "JobRunLease", "JobRunLease", 1) + `,`, - `}`, - }, "") - return s -} -func (this *LeaseStreamMessage_CancelRuns) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&LeaseStreamMessage_CancelRuns{`, - `CancelRuns:` + strings.Replace(fmt.Sprintf("%v", this.CancelRuns), "CancelRuns", "CancelRuns", 1) + `,`, - `}`, - }, "") - return s -} -func (this *LeaseStreamMessage_End) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&LeaseStreamMessage_End{`, - `End:` + strings.Replace(fmt.Sprintf("%v", this.End), "EndMarker", "EndMarker", 1) + `,`, - `}`, - }, "") - return s -} -func (this *LeaseStreamMessage_PreemptRuns) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&LeaseStreamMessage_PreemptRuns{`, - `PreemptRuns:` + strings.Replace(fmt.Sprintf("%v", this.PreemptRuns), "PreemptRuns", "PreemptRuns", 1) + `,`, - `}`, - }, "") - return s -} -func valueToStringExecutorapi(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} func (m *NodeInfo) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2434,7 +2187,7 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Taints = append(m.Taints, v1.Taint{}) + m.Taints = append(m.Taints, &v1.Taint{}) if err := m.Taints[len(m.Taints)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -2596,10 +2349,10 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.AllocatableResources == nil { - m.AllocatableResources = make(map[string]resource.Quantity) + m.AllocatableResources = make(map[string]*resource.Quantity) } var mapkey string - mapvalue := &resource.Quantity{} + var mapvalue *resource.Quantity for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -2693,7 +2446,7 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.AllocatableResources[mapkey] = *mapvalue + m.AllocatableResources[mapkey] = mapvalue iNdEx = postIndex case 5: if wireType != 2 { @@ -2725,10 +2478,10 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.AvailableResources == nil { - m.AvailableResources = make(map[string]resource.Quantity) + m.AvailableResources = make(map[string]*resource.Quantity) } var mapkey string - mapvalue := &resource.Quantity{} + var mapvalue *resource.Quantity for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -2822,7 +2575,7 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.AvailableResources[mapkey] = *mapvalue + m.AvailableResources[mapkey] = mapvalue iNdEx = postIndex case 6: if wireType != 2 { @@ -2854,10 +2607,10 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.TotalResources == nil { - m.TotalResources = make(map[string]resource.Quantity) + m.TotalResources = make(map[string]*resource.Quantity) } var mapkey string - mapvalue := &resource.Quantity{} + var mapvalue *resource.Quantity for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -2951,7 +2704,7 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.TotalResources[mapkey] = *mapvalue + m.TotalResources[mapkey] = mapvalue iNdEx = postIndex case 7: if wireType != 2 { @@ -2983,10 +2736,10 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.AllocatedResources == nil { - m.AllocatedResources = make(map[int32]ComputeResource) + m.AllocatedResources = make(map[int32]*ComputeResource) } var mapkey int32 - mapvalue := &ComputeResource{} + var mapvalue *ComputeResource for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -3066,7 +2819,7 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.AllocatedResources[mapkey] = *mapvalue + m.AllocatedResources[mapkey] = mapvalue iNdEx = postIndex case 8: if wireType != 2 { @@ -3211,10 +2964,10 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.NonArmadaAllocatedResources == nil { - m.NonArmadaAllocatedResources = make(map[int32]ComputeResource) + m.NonArmadaAllocatedResources = make(map[int32]*ComputeResource) } var mapkey int32 - mapvalue := &ComputeResource{} + var mapvalue *ComputeResource for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -3294,7 +3047,7 @@ func (m *NodeInfo) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.NonArmadaAllocatedResources[mapkey] = *mapvalue + m.NonArmadaAllocatedResources[mapkey] = mapvalue iNdEx = postIndex case 10: if wireType != 0 { @@ -3557,10 +3310,10 @@ func (m *ComputeResource) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Resources == nil { - m.Resources = make(map[string]resource.Quantity) + m.Resources = make(map[string]*resource.Quantity) } var mapkey string - mapvalue := &resource.Quantity{} + var mapvalue *resource.Quantity for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -3654,7 +3407,7 @@ func (m *ComputeResource) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.Resources[mapkey] = *mapvalue + m.Resources[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -3884,10 +3637,10 @@ func (m *LeaseRequest) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Resources == nil { - m.Resources = make(map[string]resource.Quantity) + m.Resources = make(map[string]*resource.Quantity) } var mapkey string - mapvalue := &resource.Quantity{} + var mapvalue *resource.Quantity for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -3981,7 +3734,7 @@ func (m *LeaseRequest) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.Resources[mapkey] = *mapvalue + m.Resources[mapkey] = mapvalue iNdEx = postIndex case 4: if wireType != 2 { @@ -4013,10 +3766,10 @@ func (m *LeaseRequest) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.MinimumJobSize == nil { - m.MinimumJobSize = make(map[string]resource.Quantity) + m.MinimumJobSize = make(map[string]*resource.Quantity) } var mapkey string - mapvalue := &resource.Quantity{} + var mapvalue *resource.Quantity for iNdEx < postIndex { entryPreIndex := iNdEx var wire uint64 @@ -4110,7 +3863,7 @@ func (m *LeaseRequest) Unmarshal(dAtA []byte) error { iNdEx += skippy } } - m.MinimumJobSize[mapkey] = *mapvalue + m.MinimumJobSize[mapkey] = mapvalue iNdEx = postIndex case 5: if wireType != 2 { @@ -4175,7 +3928,7 @@ func (m *LeaseRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.UnassignedJobRunIds = append(m.UnassignedJobRunIds, armadaevents.Uuid{}) + m.UnassignedJobRunIds = append(m.UnassignedJobRunIds, &armadaevents.Uuid{}) if err := m.UnassignedJobRunIds[len(m.UnassignedJobRunIds)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } diff --git a/pkg/executorapi/executorapi.proto b/pkg/executorapi/executorapi.proto index f86d815a6c7..21d8d98c138 100644 --- a/pkg/executorapi/executorapi.proto +++ b/pkg/executorapi/executorapi.proto @@ -6,38 +6,34 @@ option go_package = "github.com/armadaproject/armada/pkg/executorapi"; import "pkg/armadaevents/events.proto"; import "pkg/api/submit.proto"; import "google/protobuf/empty.proto"; -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; import "k8s.io/api/core/v1/generated.proto"; import "k8s.io/apimachinery/pkg/api/resource/generated.proto"; -option (gogoproto.goproto_stringer_all) = false; -option (gogoproto.stringer_all) = true; - // Used by the scheduler when allocating jobs to executors. message NodeInfo { string name = 1; - repeated k8s.io.api.core.v1.Taint taints = 2 [(gogoproto.nullable) = false]; + repeated k8s.io.api.core.v1.Taint taints = 2; map labels = 3; // To be deprecated in favour of total_resources + allocated_resources. - map allocatable_resources = 4 [(gogoproto.nullable) = false]; + map allocatable_resources = 4; // To be deprecated in favour of total_resources + allocated_resources. - map available_resources = 5 [(gogoproto.nullable) = false]; + map available_resources = 5; // Total node resources. // Resources available for allocation is given by the difference between this and allocated_resources. - map total_resources = 6 [(gogoproto.nullable) = false]; + map total_resources = 6; // Each pod is created with a priority class. Each priority class has an integer priority associated with it. // This is a map from priority to the total amount of resources allocated to pods with that priority. // It is used by the scheduler to decide whether more jobs should be sent to an executor. // In particular, jobs may be sent to an executor even if all resources are allocated // if the sent jobs are of higher priority. - map allocated_resources = 7 [(gogoproto.nullable) = false]; + map allocated_resources = 7; // All run ids of jobs on the node, mapped to their current state // this should be of type armadaevents.uuid, but this creates a circular loop // once the old scheduler has gone, we can correct this - map run_ids_by_state = 8 [(gogoproto.nullable) = false]; + map run_ids_by_state = 8; // The amount of resource allocated to non-armada pods by priority - map non_armada_allocated_resources = 9 [(gogoproto.nullable) = false]; + map non_armada_allocated_resources = 9; bool unschedulable = 10; // This should only be used for metrics // An aggregated real usage of jobs by queue @@ -48,7 +44,7 @@ message NodeInfo { } message ComputeResource { - map resources = 1 [(gogoproto.nullable) = false]; + map resources = 1; } message EventList { @@ -61,16 +57,16 @@ message LeaseRequest{ // Nodes are split into pools. This field indicates for which pool jobs are leased. string pool = 2; // Total resources available for scheduling across all nodes. - map resources = 3 [(gogoproto.nullable) = false]; + map resources = 3; // Jobs submitted to this executor must require at least this amount of resources. - map minimum_job_size = 4 [(gogoproto.nullable) = false]; + map minimum_job_size = 4; // For each node in the cluster: // - The total allocatable resources on that node. // - The job runs running on those nodes, // - Any taints and labels on the node. repeated NodeInfo nodes = 5; // Run Ids of jobs owned by the executor but not currently assigned to a node. - repeated armadaevents.Uuid unassigned_job_run_ids = 6 [(gogoproto.nullable) = false]; + repeated armadaevents.Uuid unassigned_job_run_ids = 6; // Max number of jobs this request should return uint32 max_jobs_to_lease = 7; } diff --git a/pkg/executorapi/util.go b/pkg/executorapi/util.go index 9158088dbf1..751ec123932 100644 --- a/pkg/executorapi/util.go +++ b/pkg/executorapi/util.go @@ -3,6 +3,12 @@ package executorapi import ( "time" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + armadamaps "github.com/armadaproject/armada/internal/common/maps" + armadaslices "github.com/armadaproject/armada/internal/common/slices" + "github.com/pkg/errors" "github.com/armadaproject/armada/internal/common/armadaerrors" @@ -28,20 +34,19 @@ func NewNodeFromNodeInfo(nodeInfo *NodeInfo, executor string, allowedPriorities allocatableByPriorityAndResource := schedulerobjects.NewAllocatableByPriorityAndResourceType( allowedPriorities, - schedulerobjects.ResourceList{ - Resources: nodeInfo.TotalResources, - }, + ResourceListFromProtoResources(nodeInfo.TotalResources), ) for p, rl := range nodeInfo.NonArmadaAllocatedResources { - allocatableByPriorityAndResource.MarkAllocated(p, schedulerobjects.ResourceList{Resources: rl.Resources}) + allocatableByPriorityAndResource.MarkAllocated(p, ResourceListFromProtoResources(rl.Resources)) } nonArmadaAllocatedResources := make(map[int32]schedulerobjects.ResourceList) for p, rl := range nodeInfo.NonArmadaAllocatedResources { - nonArmadaAllocatedResources[p] = schedulerobjects.ResourceList{Resources: rl.Resources} + nonArmadaAllocatedResources[p] = ResourceListFromProtoResources(rl.Resources) } resourceUsageByQueue := make(map[string]*schedulerobjects.ResourceList) for queueName, resourceUsage := range nodeInfo.ResourceUsageByQueue { - resourceUsageByQueue[queueName] = &schedulerobjects.ResourceList{Resources: resourceUsage.Resources} + rl := ResourceListFromProtoResources(resourceUsage.Resources) + resourceUsageByQueue[queueName] = &rl } jobRunsByState := make(map[string]schedulerobjects.JobRunState) @@ -49,13 +54,18 @@ func NewNodeFromNodeInfo(nodeInfo *NodeInfo, executor string, allowedPriorities jobRunsByState[jobId] = api.JobRunStateFromApiJobState(state) } return &schedulerobjects.Node{ - Id: api.NodeIdFromExecutorAndNodeName(executor, nodeInfo.Name), - Name: nodeInfo.Name, - Executor: executor, - LastSeen: lastSeen, - Taints: nodeInfo.GetTaints(), + Id: api.NodeIdFromExecutorAndNodeName(executor, nodeInfo.Name), + Name: nodeInfo.Name, + Executor: executor, + LastSeen: lastSeen, + Taints: armadaslices.Map(nodeInfo.GetTaints(), func(v *v1.Taint) v1.Taint { + if v != nil { + return *v + } + return v1.Taint{} + }), Labels: nodeInfo.GetLabels(), - TotalResources: schedulerobjects.ResourceList{Resources: nodeInfo.TotalResources}, + TotalResources: ResourceListFromProtoResources(nodeInfo.TotalResources), AllocatableByPriorityAndResource: allocatableByPriorityAndResource, NonArmadaAllocatedResources: nonArmadaAllocatedResources, StateByJobRunId: jobRunsByState, @@ -64,3 +74,23 @@ func NewNodeFromNodeInfo(nodeInfo *NodeInfo, executor string, allowedPriorities ReportingNodeType: nodeInfo.NodeType, }, nil } + +func ResourceListFromProtoResources(r map[string]*resource.Quantity) schedulerobjects.ResourceList { + return schedulerobjects.ResourceList{ + Resources: armadamaps.MapValues(r, func(v *resource.Quantity) resource.Quantity { + if v != nil { + return *v + } + return resource.Quantity{} + }), + } +} + +func ComputeResourceFromProtoResources(r map[string]resource.Quantity) *ComputeResource { + resources := make(map[string]*resource.Quantity, len(r)) + for k, v := range r { + r := v.DeepCopy() + resources[k] = &r + } + return &ComputeResource{Resources: resources} +} From 91a346f05f2827aedfb56f44ff0e939ca7e17786 Mon Sep 17 00:00:00 2001 From: robertdavidsmith <34475852+robertdavidsmith@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:23:56 +0100 Subject: [PATCH 3/3] Scheduler: Add floating resource support (#3767) --- internal/common/maps/maps.go | 28 ++++ internal/common/maps/maps_test.go | 42 ++++++ internal/scheduler/adapters/adapters.go | 12 ++ internal/scheduler/adapters/adapters_test.go | 17 +++ internal/scheduler/api.go | 33 ++++- internal/scheduler/api_test.go | 9 ++ .../scheduler/configuration/configuration.go | 20 +++ internal/scheduler/context/context.go | 42 ++++-- .../floating_resource_types.go | 118 ++++++++++++++++ .../floating_resource_types_test.go | 130 ++++++++++++++++++ internal/scheduler/gang_scheduler.go | 23 +++- internal/scheduler/gang_scheduler_test.go | 31 ++++- .../internaltypes/resource_list_factory.go | 8 +- .../resource_list_factory_test.go | 6 +- internal/scheduler/jobdb/comparison_test.go | 4 +- internal/scheduler/jobdb/job.go | 9 +- internal/scheduler/jobdb/job_run_test.go | 7 + internal/scheduler/jobdb/jobdb.go | 28 +++- internal/scheduler/jobdb/jobdb_test.go | 3 + internal/scheduler/metrics.go | 43 ++++-- internal/scheduler/metrics_test.go | 23 +++- internal/scheduler/nodedb/nodedb_test.go | 4 +- .../scheduler/nodedb/nodeiteration_test.go | 4 +- .../scheduler/nodedb/nodematching_test.go | 4 +- .../scheduler/preempting_queue_scheduler.go | 7 +- .../preempting_queue_scheduler_test.go | 9 +- internal/scheduler/queue_scheduler.go | 4 +- internal/scheduler/queue_scheduler_test.go | 2 +- internal/scheduler/schedulerapp.go | 12 ++ internal/scheduler/scheduling_algo.go | 17 ++- internal/scheduler/scheduling_algo_test.go | 2 + internal/scheduler/simulator/simulator.go | 14 ++ .../scheduler/testfixtures/testfixtures.go | 26 +++- 33 files changed, 664 insertions(+), 77 deletions(-) create mode 100644 internal/scheduler/floatingresources/floating_resource_types.go create mode 100644 internal/scheduler/floatingresources/floating_resource_types_test.go diff --git a/internal/common/maps/maps.go b/internal/common/maps/maps.go index ce9f7ec170b..28ba6e3540b 100644 --- a/internal/common/maps/maps.go +++ b/internal/common/maps/maps.go @@ -2,6 +2,15 @@ package maps import "github.com/armadaproject/armada/internal/common/interfaces" +// FromSlice maps element e of slice into map entry keyFunc(e): valueFunc(e) +func FromSlice[S []E, E any, K comparable, V any](slice S, keyFunc func(E) K, valueFunc func(E) V) map[K]V { + rv := make(map[K]V, len(slice)) + for _, elem := range slice { + rv[keyFunc(elem)] = valueFunc(elem) + } + return rv +} + // MapValues maps the values of m into valueFunc(v). func MapValues[M ~map[K]VA, K comparable, VA any, VB any](m M, valueFunc func(VA) VB) map[K]VB { rv := make(map[K]VB, len(m)) @@ -75,3 +84,22 @@ func Filter[M ~map[K]V, K comparable, V any](m M, predicate func(K, V) bool) M { } return rv } + +// RemoveInPlace removes elements that match keySelector +func RemoveInPlace[K comparable, V any](m map[K]V, keySelector func(K) bool) { + for k := range m { + if keySelector(k) { + delete(m, k) + } + } +} + +func Keys[K comparable, V any](m map[K]V) []K { + i := 0 + result := make([]K, len(m)) + for k := range m { + result[i] = k + i++ + } + return result +} diff --git a/internal/common/maps/maps_test.go b/internal/common/maps/maps_test.go index e829d88bb2c..f7d6838dae5 100644 --- a/internal/common/maps/maps_test.go +++ b/internal/common/maps/maps_test.go @@ -1,12 +1,26 @@ package maps import ( + "strconv" "testing" "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" ) +func TestFromSlice(t *testing.T) { + actual := FromSlice( + []int{1, 2}, + func(elem int) string { return strconv.Itoa(elem) }, + func(elem int) float64 { return float64(elem) * 0.1 }, + ) + expected := map[string]float64{ + "1": 0.1, + "2": 0.2, + } + assert.Equal(t, expected, actual) +} + func TestMapKeys(t *testing.T) { m := map[string][]int{ "foo": {1, 2, 3}, @@ -176,3 +190,31 @@ func TestFilterKeys(t *testing.T) { } assert.Equal(t, expected, actual) } + +func TestRemoveInPlace(t *testing.T) { + m := map[int]string{ + 1: "one", + 2: "two", + 3: "three", + 4: "four", + } + RemoveInPlace(m, func(i int) bool { + return i > 2 + }) + expected := map[int]string{ + 1: "one", + 2: "two", + } + assert.Equal(t, expected, m) +} + +func TestKeys(t *testing.T) { + m := map[int]string{ + 1: "one", + 2: "two", + } + result := Keys(m) + slices.Sort(result) + expected := []int{1, 2} + assert.Equal(t, expected, result) +} diff --git a/internal/scheduler/adapters/adapters.go b/internal/scheduler/adapters/adapters.go index 1115d88869d..19b314c1413 100644 --- a/internal/scheduler/adapters/adapters.go +++ b/internal/scheduler/adapters/adapters.go @@ -4,6 +4,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + k8sResource "k8s.io/apimachinery/pkg/api/resource" "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/types" @@ -62,3 +63,14 @@ func PriorityFromPodSpec(podSpec *v1.PodSpec, priorityClasses map[string]types.P // Couldn't find anything return 0, false } + +func K8sResourceListToMap(resources v1.ResourceList) map[string]k8sResource.Quantity { + if resources == nil { + return nil + } + result := make(map[string]k8sResource.Quantity, len(resources)) + for k, v := range resources { + result[string(k)] = v + } + return result +} diff --git a/internal/scheduler/adapters/adapters_test.go b/internal/scheduler/adapters/adapters_test.go index 6ab18152df8..520892a50d2 100644 --- a/internal/scheduler/adapters/adapters_test.go +++ b/internal/scheduler/adapters/adapters_test.go @@ -205,3 +205,20 @@ func TestPriorityFromPodSpec(t *testing.T) { }) } } + +func TestK8sResourceListToMap(t *testing.T) { + result := K8sResourceListToMap(v1.ResourceList{ + "one": resource.MustParse("1"), + "two": resource.MustParse("2"), + }) + expected := map[string]resource.Quantity{ + "one": resource.MustParse("1"), + "two": resource.MustParse("2"), + } + + assert.Equal(t, expected, result) +} + +func TestK8sResourceListToMap_PreservesNil(t *testing.T) { + assert.Nil(t, K8sResourceListToMap(nil)) +} diff --git a/internal/scheduler/api.go b/internal/scheduler/api.go index b493873908b..8094c6a5ce3 100644 --- a/internal/scheduler/api.go +++ b/internal/scheduler/api.go @@ -17,6 +17,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/maps" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/slices" priorityTypes "github.com/armadaproject/armada/internal/common/types" @@ -40,6 +41,9 @@ type ExecutorApi struct { allowedPriorities []int32 // Known priority classes priorityClasses map[string]priorityTypes.PriorityClass + // Allowed resource names - resource requests/limits not on this list are dropped. + // This is needed to ensure floating resources are not passed to k8s. + allowedResources map[string]bool // Max number of events in published Pulsar messages maxEventsPerPulsarMessage int // Max size of Pulsar messages produced. @@ -55,6 +59,7 @@ func NewExecutorApi(producer pulsar.Producer, jobRepository database.JobRepository, executorRepository database.ExecutorRepository, allowedPriorities []int32, + allowedResources []string, nodeIdLabel string, priorityClassNameOverride *string, priorityClasses map[string]priorityTypes.PriorityClass, @@ -69,6 +74,7 @@ func NewExecutorApi(producer pulsar.Producer, jobRepository: jobRepository, executorRepository: executorRepository, allowedPriorities: allowedPriorities, + allowedResources: maps.FromSlice(allowedResources, func(name string) string { return name }, func(name string) bool { return true }), maxEventsPerPulsarMessage: maxEventsPerPulsarMessage, maxPulsarMessageSizeBytes: maxPulsarMessageSizeBytes, nodeIdLabel: nodeIdLabel, @@ -109,7 +115,7 @@ func (srv *ExecutorApi) LeaseJobRuns(stream executorapi.ExecutorApi_LeaseJobRuns return err } ctx.Infof( - "executor currently has %d job runs; sending %d cancellations and %d new runs", + "Executor currently has %d job runs; sending %d cancellations and %d new runs", len(requestRuns), len(runsToCancel), len(newRuns), ) @@ -149,6 +155,8 @@ func (srv *ExecutorApi) LeaseJobRuns(stream executorapi.ExecutorApi_LeaseJobRuns srv.addPreemptibleLabel(submitMsg) + srv.dropDisallowedResources(submitMsg.MainObject.GetPodSpec().PodSpec) + // This must happen after anything that relies on the priorityClassName if srv.priorityClassNameOverride != nil { srv.setPriorityClassName(submitMsg, *srv.priorityClassNameOverride) @@ -216,6 +224,29 @@ func (srv *ExecutorApi) addPreemptibleLabel(job *armadaevents.SubmitJob) { addLabels(job, labels) } +// Drop non-supported resources. This is needed to ensure floating resources +// are not passed to k8s. +func (srv *ExecutorApi) dropDisallowedResources(pod *v1.PodSpec) { + if pod == nil { + return + } + srv.dropDisallowedResourcesFromContainers(pod.InitContainers) + srv.dropDisallowedResourcesFromContainers(pod.Containers) +} + +func (srv *ExecutorApi) dropDisallowedResourcesFromContainers(containers []v1.Container) { + for _, container := range containers { + removeDisallowedKeys(container.Resources.Limits, srv.allowedResources) + removeDisallowedKeys(container.Resources.Requests, srv.allowedResources) + } +} + +func removeDisallowedKeys(rl v1.ResourceList, allowedKeys map[string]bool) { + maps.RemoveInPlace(rl, func(name v1.ResourceName) bool { + return !allowedKeys[string(name)] + }) +} + func (srv *ExecutorApi) isPreemptible(job *armadaevents.SubmitJob) bool { priorityClassName := "" diff --git a/internal/scheduler/api_test.go b/internal/scheduler/api_test.go index 4b080235a52..27f48c0d925 100644 --- a/internal/scheduler/api_test.go +++ b/internal/scheduler/api_test.go @@ -19,10 +19,13 @@ import ( "github.com/armadaproject/armada/internal/common/mocks" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/common/pulsarutils" + "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/common/types" + schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" "github.com/armadaproject/armada/internal/scheduler/database" schedulermocks "github.com/armadaproject/armada/internal/scheduler/mocks" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" + "github.com/armadaproject/armada/internal/scheduler/testfixtures" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/armadaevents" "github.com/armadaproject/armada/pkg/executorapi" @@ -321,6 +324,7 @@ func TestExecutorApi_LeaseJobRuns(t *testing.T) { mockJobRepository, mockExecutorRepository, []int32{1000, 2000}, + testResourceNames(), "kubernetes.io/hostname", nil, priorityClasses, @@ -448,6 +452,7 @@ func TestExecutorApi_Publish(t *testing.T) { mockJobRepository, mockExecutorRepository, []int32{1000, 2000}, + testResourceNames(), "kubernetes.io/hostname", nil, priorityClasses, @@ -495,3 +500,7 @@ func groups(t *testing.T) ([]string, []byte) { require.NoError(t, err) return groups, compressed } + +func testResourceNames() []string { + return slices.Map(testfixtures.GetTestSupportedResourceTypes(), func(rt schedulerconfig.ResourceType) string { return rt.Name }) +} diff --git a/internal/scheduler/configuration/configuration.go b/internal/scheduler/configuration/configuration.go index 54800d0e7d9..8ec05a725fb 100644 --- a/internal/scheduler/configuration/configuration.go +++ b/internal/scheduler/configuration/configuration.go @@ -112,6 +112,19 @@ type LeaderConfig struct { LeaderConnection client.ApiConnectionDetails } +type FloatingResourceConfig struct { + // Resource name, e.g. "s3-connections" + Name string + // Per-pool config. + Pools []FloatingResourcePoolConfig +} + +type FloatingResourcePoolConfig struct { + // Name of the pool. + Name string + // Amount of this resource that can be allocated across all jobs in this pool. + Quantity resource.Quantity +} type HttpConfig struct { Port int `validate:"required"` } @@ -230,6 +243,13 @@ type SchedulingConfig struct { // // If not set, all taints are indexed. IndexedTaints []string + // Experimental - subject to change + // Resources that are outside of k8s, and not tied to a given k8s node or cluster. + // For example connections to an S3 server that sits outside of k8s could be rationed to limit load on the server. + // These can be requested like a normal k8s resource. Note there is no mechanism in armada + // to enforce actual usage, it relies on honesty. For example, there is nothing to stop a badly-behaved job + // requesting 2 S3 server connections and then opening 10. + ExperimentalFloatingResources []FloatingResourceConfig // WellKnownNodeTypes defines a set of well-known node types used to define "home" and "away" nodes for a given priority class. WellKnownNodeTypes []WellKnownNodeType `validate:"dive"` // Executor that haven't heartbeated in this time period are considered stale. diff --git a/internal/scheduler/context/context.go b/internal/scheduler/context/context.go index 9cf10b02fdf..b4d4f88603b 100644 --- a/internal/scheduler/context/context.go +++ b/internal/scheduler/context/context.go @@ -51,8 +51,10 @@ type SchedulingContext struct { WeightSum float64 // Per-queue scheduling contexts. QueueSchedulingContexts map[string]*QueueSchedulingContext - // Total resources across all clusters available at the start of the scheduling cycle. + // Total resources across all clusters in this pool available at the start of the scheduling cycle. TotalResources schedulerobjects.ResourceList + // Allocated resources across all clusters in this pool + Allocated schedulerobjects.ResourceList // Resources assigned across all queues during this scheduling cycle. ScheduledResources schedulerobjects.ResourceList ScheduledResourcesByPriorityClass schedulerobjects.QuantityByTAndResourceType[string] @@ -130,6 +132,8 @@ func (sctx *SchedulingContext) AddQueueSchedulingContext( allocated.Add(rl) } sctx.WeightSum += weight + sctx.Allocated.Add(allocated) + qctx := &QueueSchedulingContext{ SchedulingContext: sctx, Created: time.Now(), @@ -307,7 +311,7 @@ func (sctx *SchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingContex if !ok { return false, errors.Errorf("failed adding job %s to scheduling context: no context for queue %s", jctx.JobId, jctx.Job.Queue()) } - evictedInThisRound, err := qctx.AddJobSchedulingContext(jctx) + evictedInThisRound, err := qctx.addJobSchedulingContext(jctx) if err != nil { return false, err } @@ -321,6 +325,7 @@ func (sctx *SchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingContex sctx.ScheduledResourcesByPriorityClass.AddV1ResourceList(jctx.Job.PriorityClassName(), jctx.PodRequirements.ResourceRequirements.Requests) sctx.NumScheduledJobs++ } + sctx.Allocated.AddV1ResourceList(jctx.PodRequirements.ResourceRequirements.Requests) } return evictedInThisRound, nil } @@ -345,7 +350,7 @@ func (sctx *SchedulingContext) EvictJob(job *jobdb.Job) (bool, error) { if !ok { return false, errors.Errorf("failed evicting job %s from scheduling context: no context for queue %s", job.Id(), job.Queue()) } - scheduledInThisRound, err := qctx.EvictJob(job) + scheduledInThisRound, err := qctx.evictJob(job) if err != nil { return false, err } @@ -359,6 +364,7 @@ func (sctx *SchedulingContext) EvictJob(job *jobdb.Job) (bool, error) { sctx.EvictedResourcesByPriorityClass.AddV1ResourceList(job.PriorityClassName(), rl) sctx.NumEvictedJobs++ } + sctx.Allocated.SubV1ResourceList(rl) return scheduledInThisRound, nil } @@ -526,9 +532,9 @@ func (qctx *QueueSchedulingContext) ReportString(verbosity int32) string { return sb.String() } -// AddJobSchedulingContext adds a job scheduling context. +// addJobSchedulingContext adds a job scheduling context. // Automatically updates scheduled resources. -func (qctx *QueueSchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingContext) (bool, error) { +func (qctx *QueueSchedulingContext) addJobSchedulingContext(jctx *JobSchedulingContext) (bool, error) { if _, ok := qctx.SuccessfulJobSchedulingContexts[jctx.JobId]; ok { return false, errors.Errorf("failed adding job %s to queue: job already marked successful", jctx.JobId) } @@ -561,7 +567,7 @@ func (qctx *QueueSchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingC return evictedInThisRound, nil } -func (qctx *QueueSchedulingContext) EvictJob(job *jobdb.Job) (bool, error) { +func (qctx *QueueSchedulingContext) evictJob(job *jobdb.Job) (bool, error) { jobId := job.Id() if _, ok := qctx.UnsuccessfulJobSchedulingContexts[jobId]; ok { return false, errors.Errorf("failed evicting job %s from queue: job already marked unsuccessful", jobId) @@ -597,28 +603,34 @@ type GangSchedulingContext struct { Created time.Time Queue string GangInfo - JobSchedulingContexts []*JobSchedulingContext - TotalResourceRequests schedulerobjects.ResourceList - AllJobsEvicted bool + JobSchedulingContexts []*JobSchedulingContext + TotalResourceRequests schedulerobjects.ResourceList + AllJobsEvicted bool + RequestsFloatingResources bool } func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingContext { allJobsEvicted := true totalResourceRequests := schedulerobjects.NewResourceList(4) + requestsFloatingResources := false for _, jctx := range jctxs { allJobsEvicted = allJobsEvicted && jctx.IsEvicted totalResourceRequests.AddV1ResourceList(jctx.PodRequirements.ResourceRequirements.Requests) + if jctx.Job.RequestsFloatingResources() { + requestsFloatingResources = true + } } // Uniformity of the values that we pick off the first job in the gang was // checked when the jobs were submitted (e.g., in ValidateApiJobs). representative := jctxs[0] return &GangSchedulingContext{ - Created: time.Now(), - Queue: representative.Job.Queue(), - GangInfo: representative.GangInfo, - JobSchedulingContexts: jctxs, - TotalResourceRequests: totalResourceRequests, - AllJobsEvicted: allJobsEvicted, + Created: time.Now(), + Queue: representative.Job.Queue(), + GangInfo: representative.GangInfo, + JobSchedulingContexts: jctxs, + TotalResourceRequests: totalResourceRequests, + AllJobsEvicted: allJobsEvicted, + RequestsFloatingResources: requestsFloatingResources, } } diff --git a/internal/scheduler/floatingresources/floating_resource_types.go b/internal/scheduler/floatingresources/floating_resource_types.go new file mode 100644 index 00000000000..a2d15e00e01 --- /dev/null +++ b/internal/scheduler/floatingresources/floating_resource_types.go @@ -0,0 +1,118 @@ +package floatingresources + +import ( + "fmt" + "slices" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/armadaproject/armada/internal/common/maps" + "github.com/armadaproject/armada/internal/scheduler/configuration" + "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" +) + +type FloatingResourceTypes struct { + zeroFloatingResources schedulerobjects.ResourceList + pools map[string]*floatingResourcePool +} + +type floatingResourcePool struct { + totalResources schedulerobjects.ResourceList +} + +func NewFloatingResourceTypes(config []configuration.FloatingResourceConfig) (*FloatingResourceTypes, error) { + zeroFloatingResources := schedulerobjects.ResourceList{Resources: make(map[string]resource.Quantity, len(config))} + for _, c := range config { + if _, exists := zeroFloatingResources.Resources[c.Name]; exists { + return nil, fmt.Errorf("duplicate floating resource %s", c.Name) + } + zeroFloatingResources.Resources[c.Name] = resource.Quantity{} + } + + pools := map[string]*floatingResourcePool{} + for _, fr := range config { + for _, poolConfig := range fr.Pools { + pool, exists := pools[poolConfig.Name] + if !exists { + pool = &floatingResourcePool{ + totalResources: zeroFloatingResources.DeepCopy(), + } + pools[poolConfig.Name] = pool + } + existing := pool.totalResources.Resources[fr.Name] + if existing.Cmp(resource.Quantity{}) != 0 { + return nil, fmt.Errorf("duplicate floating resource %s for pool %s", fr.Name, poolConfig.Name) + } + pool.totalResources.Resources[fr.Name] = poolConfig.Quantity.DeepCopy() + } + } + + return &FloatingResourceTypes{ + zeroFloatingResources: zeroFloatingResources, + pools: pools, + }, nil +} + +func (frt *FloatingResourceTypes) WithinLimits(poolName string, allocated schedulerobjects.ResourceList) (bool, string) { + pool, exists := frt.pools[poolName] + if !exists { + return false, fmt.Sprintf("floating resources not connfigured for pool %s", poolName) + } + rl := pool.totalResources.DeepCopy() + rl.Sub(allocated) + for resourceName, quantity := range rl.Resources { + if !frt.isFloatingResource(resourceName) { + continue + } + if quantity.Cmp(resource.Quantity{}) == -1 { + return false, fmt.Sprintf("not enough floating resource %s in pool %s", resourceName, poolName) + } + } + return true, "" +} + +func (frt *FloatingResourceTypes) RemoveFloatingResources(allResources map[string]resource.Quantity) map[string]resource.Quantity { + result := make(map[string]resource.Quantity) + for k, v := range allResources { + if !frt.isFloatingResource(k) { + result[k] = v + } + } + return result +} + +func (frt *FloatingResourceTypes) HasFloatingResources(resources map[string]resource.Quantity) bool { + for resourceName, quantity := range resources { + if frt.isFloatingResource(resourceName) && quantity.Cmp(resource.Quantity{}) == 1 { + return true + } + } + return false +} + +func (frt *FloatingResourceTypes) AllPools() []string { + result := maps.Keys(frt.pools) + slices.Sort(result) + return result +} + +func (frt *FloatingResourceTypes) GetTotalAvailableForPool(poolName string) schedulerobjects.ResourceList { + pool, exists := frt.pools[poolName] + if !exists { + return frt.zeroFloatingResources.DeepCopy() + } + return pool.totalResources.DeepCopy() +} + +func (frt *FloatingResourceTypes) SummaryString() string { + if len(frt.zeroFloatingResources.Resources) == 0 { + return "none" + } + return strings.Join(maps.Keys(frt.zeroFloatingResources.Resources), " ") +} + +func (frt *FloatingResourceTypes) isFloatingResource(resourceName string) bool { + _, exists := frt.zeroFloatingResources.Resources[resourceName] + return exists +} diff --git a/internal/scheduler/floatingresources/floating_resource_types_test.go b/internal/scheduler/floatingresources/floating_resource_types_test.go new file mode 100644 index 00000000000..938d4daf775 --- /dev/null +++ b/internal/scheduler/floatingresources/floating_resource_types_test.go @@ -0,0 +1,130 @@ +package floatingresources + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/armadaproject/armada/internal/common/maps" + "github.com/armadaproject/armada/internal/scheduler/configuration" + "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" +) + +func TestHasFloatingResources(t *testing.T) { + sut := makeSut(t) + assert.False(t, sut.HasFloatingResources(map[string]resource.Quantity{})) + assert.False(t, sut.HasFloatingResources(map[string]resource.Quantity{"some-other-resource": resource.MustParse("10")})) + assert.False(t, sut.HasFloatingResources(map[string]resource.Quantity{"floating-resource-1": resource.MustParse("0")})) + assert.True(t, sut.HasFloatingResources(map[string]resource.Quantity{"floating-resource-1": resource.MustParse("10")})) + assert.True(t, sut.HasFloatingResources(map[string]resource.Quantity{"some-other-resource": resource.MustParse("10"), "floating-resource-1": resource.MustParse("10")})) +} + +func TestAllPools(t *testing.T) { + sut := makeSut(t) + assert.Equal(t, []string{"cpu", "gpu"}, sut.AllPools()) +} + +func TestRemoveFloatingResources(t *testing.T) { + sut := makeSut(t) + input := map[string]resource.Quantity{"floating-resource-1": resource.MustParse("200"), "some-other-resource": resource.MustParse("300")} + inputBefore := maps.DeepCopy(input) + result := sut.RemoveFloatingResources(input) + assert.Equal(t, map[string]resource.Quantity{"some-other-resource": resource.MustParse("300")}, result) + assert.Equal(t, inputBefore, input) +} + +func TestGetTotalAvailableForPool(t *testing.T) { + sut := makeSut(t) + zero := resource.Quantity{} + assert.Equal(t, map[string]resource.Quantity{"floating-resource-1": resource.MustParse("200"), "floating-resource-2": resource.MustParse("300")}, sut.GetTotalAvailableForPool("cpu").Resources) + assert.Equal(t, map[string]resource.Quantity{"floating-resource-1": resource.MustParse("100"), "floating-resource-2": zero}, sut.GetTotalAvailableForPool("gpu").Resources) + assert.Equal(t, map[string]resource.Quantity{"floating-resource-1": zero, "floating-resource-2": zero}, sut.GetTotalAvailableForPool("some-other-pool").Resources) +} + +func TestWithinLimits_WhenWithinLimits_ReturnsTrue(t *testing.T) { + sut := makeSut(t) + withinLimits, errorMessage := sut.WithinLimits("cpu", + schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"floating-resource-1": resource.MustParse("199")}}, + ) + assert.True(t, withinLimits) + assert.Empty(t, errorMessage) +} + +func TestWithinLimits_WhenAtLimit_ReturnsTrue(t *testing.T) { + sut := makeSut(t) + withinLimits, errorMessage := sut.WithinLimits("cpu", + schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"floating-resource-1": resource.MustParse("200")}}, + ) + assert.True(t, withinLimits) + assert.Empty(t, errorMessage) +} + +func TestWithinLimits_WhenExceedsLimit_ReturnsFalse(t *testing.T) { + sut := makeSut(t) + withinLimits, errorMessage := sut.WithinLimits("cpu", + schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"floating-resource-1": resource.MustParse("201")}}, + ) + assert.False(t, withinLimits) + assert.NotEmpty(t, errorMessage) +} + +func TestWithinLimits_IgnoresNonFloatingResources(t *testing.T) { + sut := makeSut(t) + withinLimits, errorMessage := sut.WithinLimits("cpu", + schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"some-other-resource": resource.MustParse("1000")}}, + ) + assert.True(t, withinLimits) + assert.Empty(t, errorMessage) +} + +func TestWithinLimits_WhenResourceNotSpecifiedForAPool_ReturnsFalse(t *testing.T) { + sut := makeSut(t) + withinLimits, errorMessage := sut.WithinLimits("gpu", + schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"floating-resource-2": resource.MustParse("1")}}, + ) + assert.False(t, withinLimits) + assert.NotEmpty(t, errorMessage) +} + +func TestWithinLimits_WhenPoolDoesNotExist_ReturnsFalse(t *testing.T) { + sut := makeSut(t) + withinLimits, errorMessage := sut.WithinLimits("some-other-pool", + schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"floating-resource-1": resource.MustParse("1")}}, + ) + assert.False(t, withinLimits) + assert.NotEmpty(t, errorMessage) +} + +func testConfig() []configuration.FloatingResourceConfig { + return []configuration.FloatingResourceConfig{ + { + Name: "floating-resource-1", + Pools: []configuration.FloatingResourcePoolConfig{ + { + Name: "cpu", + Quantity: resource.MustParse("200"), + }, + { + Name: "gpu", + Quantity: resource.MustParse("100"), + }, + }, + }, + { + Name: "floating-resource-2", + Pools: []configuration.FloatingResourcePoolConfig{ + { + Name: "cpu", + Quantity: resource.MustParse("300"), + }, + }, + }, + } +} + +func makeSut(t *testing.T) *FloatingResourceTypes { + sut, err := NewFloatingResourceTypes(testConfig()) + assert.Nil(t, err) + return sut +} diff --git a/internal/scheduler/gang_scheduler.go b/internal/scheduler/gang_scheduler.go index 5fb28d49fbe..6b611704b24 100644 --- a/internal/scheduler/gang_scheduler.go +++ b/internal/scheduler/gang_scheduler.go @@ -10,6 +10,7 @@ import ( "github.com/armadaproject/armada/internal/common/slices" schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/constraints" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -17,9 +18,10 @@ import ( // GangScheduler schedules one gang at a time. GangScheduler is not aware of queues. type GangScheduler struct { - constraints schedulerconstraints.SchedulingConstraints - schedulingContext *schedulercontext.SchedulingContext - nodeDb *nodedb.NodeDb + constraints schedulerconstraints.SchedulingConstraints + floatingResourceTypes *floatingresources.FloatingResourceTypes + schedulingContext *schedulercontext.SchedulingContext + nodeDb *nodedb.NodeDb // If true, the unsuccessfulSchedulingKeys check is omitted. skipUnsuccessfulSchedulingKeyCheck bool } @@ -27,12 +29,14 @@ type GangScheduler struct { func NewGangScheduler( sctx *schedulercontext.SchedulingContext, constraints schedulerconstraints.SchedulingConstraints, + floatingResourceTypes *floatingresources.FloatingResourceTypes, nodeDb *nodedb.NodeDb, ) (*GangScheduler, error) { return &GangScheduler{ - constraints: constraints, - schedulingContext: sctx, - nodeDb: nodeDb, + constraints: constraints, + floatingResourceTypes: floatingResourceTypes, + schedulingContext: sctx, + nodeDb: nodeDb, }, nil } @@ -138,6 +142,13 @@ func (sch *GangScheduler) Schedule(ctx *armadacontext.Context, gctx *schedulerco return } } + + if gctx.RequestsFloatingResources { + if ok, unschedulableReason = sch.floatingResourceTypes.WithinLimits(sch.schedulingContext.Pool, sch.schedulingContext.Allocated); !ok { + return + } + } + return sch.trySchedule(ctx, gctx) } diff --git a/internal/scheduler/gang_scheduler_test.go b/internal/scheduler/gang_scheduler_test.go index 09ce9fc04dd..fdb8cd8ae39 100644 --- a/internal/scheduler/gang_scheduler_test.go +++ b/internal/scheduler/gang_scheduler_test.go @@ -20,6 +20,7 @@ import ( schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/constraints" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" "github.com/armadaproject/armada/internal/scheduler/fairness" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -90,6 +91,22 @@ func TestGangScheduler(t *testing.T) { ExpectedCumulativeScheduledJobs: []int{64}, ExpectedRuntimeGangCardinality: []int{64}, }, + "floating resources": { + SchedulingConfig: func() configuration.SchedulingConfig { + cfg := testfixtures.TestSchedulingConfig() + cfg.ExperimentalFloatingResources = testfixtures.TestFloatingResourceConfig + return cfg + }(), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Gangs: [][]*jobdb.Job{ + // we have 10 of test-floating-resource so only the first of these two jobs should fit + addFloatingResourceRequest("6", testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1))), + addFloatingResourceRequest("6", testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1))), + }, + ExpectedScheduledIndices: testfixtures.IntRange(0, 0), + ExpectedCumulativeScheduledJobs: []int{1, 1}, + ExpectedRuntimeGangCardinality: []int{1, 0}, + }, "MaximumResourceFractionToSchedule": { SchedulingConfig: testfixtures.WithRoundLimitsConfig( map[string]float64{"cpu": 0.5}, @@ -571,7 +588,9 @@ func TestGangScheduler(t *testing.T) { tc.SchedulingConfig, nil, ) - sch, err := NewGangScheduler(sctx, constraints, nodeDb) + floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(tc.SchedulingConfig.ExperimentalFloatingResources) + require.NoError(t, err) + sch, err := NewGangScheduler(sctx, constraints, floatingResourceTypes, nodeDb) require.NoError(t, err) var actualScheduledIndices []int @@ -638,3 +657,13 @@ func TestGangScheduler(t *testing.T) { }) } } + +func addFloatingResourceRequest(request string, jobs []*jobdb.Job) []*jobdb.Job { + return testfixtures.WithRequestsJobs( + schedulerobjects.ResourceList{ + Resources: map[string]resource.Quantity{ + "test-floating-resource": resource.MustParse(request), + }, + }, + jobs) +} diff --git a/internal/scheduler/internaltypes/resource_list_factory.go b/internal/scheduler/internaltypes/resource_list_factory.go index c610b8040dc..771e4b59669 100644 --- a/internal/scheduler/internaltypes/resource_list_factory.go +++ b/internal/scheduler/internaltypes/resource_list_factory.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" k8sResource "k8s.io/apimachinery/pkg/api/resource" "github.com/armadaproject/armada/internal/scheduler/configuration" @@ -82,13 +81,10 @@ func (factory *ResourceListFactory) FromJobResourceListIgnoreUnknown(resources m } // Fail on unknown resources, round up. -func (factory *ResourceListFactory) FromJobResourceListFailOnUnknown(resources v1.ResourceList) (ResourceList, error) { - if resources == nil { - return ResourceList{}, nil - } +func (factory *ResourceListFactory) FromJobResourceListFailOnUnknown(resources map[string]k8sResource.Quantity) (ResourceList, error) { result := make([]int64, len(factory.indexToName)) for k, v := range resources { - index, ok := factory.nameToIndex[string(k)] + index, ok := factory.nameToIndex[k] if ok { result[index] = QuantityToInt64RoundUp(v, factory.scales[index]) } else { diff --git a/internal/scheduler/internaltypes/resource_list_factory_test.go b/internal/scheduler/internaltypes/resource_list_factory_test.go index 5224a553e7b..1010c9e8cdb 100644 --- a/internal/scheduler/internaltypes/resource_list_factory_test.go +++ b/internal/scheduler/internaltypes/resource_list_factory_test.go @@ -4,8 +4,6 @@ import ( "math" "testing" - v1 "k8s.io/api/core/v1" - "github.com/stretchr/testify/assert" k8sResource "k8s.io/apimachinery/pkg/api/resource" @@ -48,7 +46,7 @@ func TestFromNodeProto(t *testing.T) { func TestFromJobResourceListFailOnUnknown(t *testing.T) { factory := testFactory() - result, err := factory.FromJobResourceListFailOnUnknown(map[v1.ResourceName]k8sResource.Quantity{ + result, err := factory.FromJobResourceListFailOnUnknown(map[string]k8sResource.Quantity{ "memory": k8sResource.MustParse("100Mi"), "cpu": k8sResource.MustParse("9999999n"), }) @@ -60,7 +58,7 @@ func TestFromJobResourceListFailOnUnknown(t *testing.T) { func TestFromJobResourceListFailOnUnknownErrorsIfMissing(t *testing.T) { factory := testFactory() - _, err := factory.FromJobResourceListFailOnUnknown(map[v1.ResourceName]k8sResource.Quantity{ + _, err := factory.FromJobResourceListFailOnUnknown(map[string]k8sResource.Quantity{ "memory": k8sResource.MustParse("100Mi"), "missing": k8sResource.MustParse("1"), }) diff --git a/internal/scheduler/jobdb/comparison_test.go b/internal/scheduler/jobdb/comparison_test.go index 875c139ab39..502f7ccf613 100644 --- a/internal/scheduler/jobdb/comparison_test.go +++ b/internal/scheduler/jobdb/comparison_test.go @@ -7,9 +7,11 @@ import ( "github.com/armadaproject/armada/internal/common/stringinterner" "github.com/armadaproject/armada/internal/common/types" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" ) func TestJobPriorityComparer(t *testing.T) { + emptyFloatingResourceTypes, _ := floatingresources.NewFloatingResourceTypes(nil) tests := map[string]struct { a *Job b *Job @@ -52,7 +54,7 @@ func TestJobPriorityComparer(t *testing.T) { }, "Running jobs come before queued jobs": { a: &Job{id: "a", priority: 1}, - b: (&Job{id: "b", priority: 2, jobDb: NewJobDb(map[string]types.PriorityClass{"foo": {}}, "foo", stringinterner.New(1), TestResourceListFactory)}).WithNewRun("", "", "", 0), + b: (&Job{id: "b", priority: 2, jobDb: NewJobDb(map[string]types.PriorityClass{"foo": {}}, "foo", stringinterner.New(1), TestResourceListFactory, emptyFloatingResourceTypes)}).WithNewRun("", "", "", 0), expected: 1, }, "Running jobs are ordered third by runtime": { diff --git a/internal/scheduler/jobdb/job.go b/internal/scheduler/jobdb/job.go index 8f9e8bdb7d9..c2ca8a45acc 100644 --- a/internal/scheduler/jobdb/job.go +++ b/internal/scheduler/jobdb/job.go @@ -14,6 +14,7 @@ import ( armadamaps "github.com/armadaproject/armada/internal/common/maps" "github.com/armadaproject/armada/internal/common/types" + "github.com/armadaproject/armada/internal/scheduler/adapters" "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) @@ -644,7 +645,8 @@ func (job *Job) ValidateResourceRequests() error { return nil } - _, err := job.jobDb.resourceListFactory.FromJobResourceListFailOnUnknown(req) + resourcesExclFloating := job.jobDb.floatingResourceTypes.RemoveFloatingResources(adapters.K8sResourceListToMap(req)) + _, err := job.jobDb.resourceListFactory.FromJobResourceListFailOnUnknown(resourcesExclFloating) return err } @@ -765,6 +767,11 @@ func (job *Job) Validated() bool { return job.validated } +// Does this job request any floating resources? +func (job *Job) RequestsFloatingResources() bool { + return job.jobDb.floatingResourceTypes.HasFloatingResources(safeGetRequirements(job.jobSchedulingInfo)) +} + // WithJobSchedulingInfo returns a copy of the job with the job scheduling info updated. func (job *Job) WithJobSchedulingInfo(jobSchedulingInfo *schedulerobjects.JobSchedulingInfo) (*Job, error) { j := copyJob(*job) diff --git a/internal/scheduler/jobdb/job_run_test.go b/internal/scheduler/jobdb/job_run_test.go index f00ee4b0782..91a5b7e3f01 100644 --- a/internal/scheduler/jobdb/job_run_test.go +++ b/internal/scheduler/jobdb/job_run_test.go @@ -8,6 +8,7 @@ import ( "github.com/armadaproject/armada/internal/common/stringinterner" "github.com/armadaproject/armada/internal/common/types" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) @@ -35,6 +36,7 @@ var ( SchedulingKeyGenerator, stringinterner.New(1024), MakeTestResourceListFactory(), + makeTestEmptyFloatingResources(), ) scheduledAtPriority = int32(5) ) @@ -168,3 +170,8 @@ func TestDeepCopy(t *testing.T) { run.executor = "new executor" assert.Equal(t, expected, actual) } + +func makeTestEmptyFloatingResources() *floatingresources.FloatingResourceTypes { + result, _ := floatingresources.NewFloatingResourceTypes(nil) + return result +} diff --git a/internal/scheduler/jobdb/jobdb.go b/internal/scheduler/jobdb/jobdb.go index 8541d599bed..dead632849d 100644 --- a/internal/scheduler/jobdb/jobdb.go +++ b/internal/scheduler/jobdb/jobdb.go @@ -9,10 +9,13 @@ import ( "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "golang.org/x/exp/maps" + k8sResource "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/stringinterner" "github.com/armadaproject/armada/internal/common/types" + "github.com/armadaproject/armada/internal/scheduler/adapters" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) @@ -41,6 +44,8 @@ type JobDb struct { uuidProvider UUIDProvider // Used to make efficient ResourceList types. resourceListFactory *internaltypes.ResourceListFactory + // Info about floating resources + floatingResourceTypes *floatingresources.FloatingResourceTypes } // UUIDProvider is an interface used to mock UUID generation for tests. @@ -55,7 +60,11 @@ func (_ RealUUIDProvider) New() uuid.UUID { return uuid.New() } -func NewJobDb(priorityClasses map[string]types.PriorityClass, defaultPriorityClassName string, stringInterner *stringinterner.StringInterner, resourceListFactory *internaltypes.ResourceListFactory, +func NewJobDb(priorityClasses map[string]types.PriorityClass, + defaultPriorityClassName string, + stringInterner *stringinterner.StringInterner, + resourceListFactory *internaltypes.ResourceListFactory, + floatingResourceTypes *floatingresources.FloatingResourceTypes, ) *JobDb { return NewJobDbWithSchedulingKeyGenerator( priorityClasses, @@ -63,6 +72,7 @@ func NewJobDb(priorityClasses map[string]types.PriorityClass, defaultPriorityCla schedulerobjects.NewSchedulingKeyGenerator(), stringInterner, resourceListFactory, + floatingResourceTypes, ) } @@ -72,6 +82,7 @@ func NewJobDbWithSchedulingKeyGenerator( skg *schedulerobjects.SchedulingKeyGenerator, stringInterner *stringinterner.StringInterner, resourceListFactory *internaltypes.ResourceListFactory, + floatingResourceTypes *floatingresources.FloatingResourceTypes, ) *JobDb { defaultPriorityClass, ok := priorityClasses[defaultPriorityClassName] if !ok { @@ -91,6 +102,7 @@ func NewJobDbWithSchedulingKeyGenerator( clock: clock.RealClock{}, uuidProvider: RealUUIDProvider{}, resourceListFactory: resourceListFactory, + floatingResourceTypes: floatingResourceTypes, } } @@ -139,8 +151,6 @@ func (jobDb *JobDb) NewJob( priorityClass = jobDb.defaultPriorityClass } - rr := jobDb.getResourceRequirements(schedulingInfo) - job := &Job{ jobDb: jobDb, id: jobId, @@ -152,7 +162,7 @@ func (jobDb *JobDb) NewJob( requestedPriority: priority, submittedTime: created, jobSchedulingInfo: jobDb.internJobSchedulingInfoStrings(schedulingInfo), - resourceRequirements: rr, + resourceRequirements: jobDb.getResourceRequirements(schedulingInfo), priorityClass: priorityClass, cancelRequested: cancelRequested, cancelByJobSetRequested: cancelByJobSetRequested, @@ -167,17 +177,21 @@ func (jobDb *JobDb) NewJob( } func (jobDb *JobDb) getResourceRequirements(schedulingInfo *schedulerobjects.JobSchedulingInfo) internaltypes.ResourceList { + return jobDb.resourceListFactory.FromJobResourceListIgnoreUnknown(safeGetRequirements(schedulingInfo)) +} + +func safeGetRequirements(schedulingInfo *schedulerobjects.JobSchedulingInfo) map[string]k8sResource.Quantity { pr := schedulingInfo.GetPodRequirements() if pr == nil { - return internaltypes.ResourceList{} + return map[string]k8sResource.Quantity{} } req := pr.ResourceRequirements.Requests if req == nil { - return internaltypes.ResourceList{} + return map[string]k8sResource.Quantity{} } - return jobDb.resourceListFactory.FromJobResourceListIgnoreUnknown(schedulerobjects.ResourceListFromV1ResourceList(req).Resources) + return adapters.K8sResourceListToMap(req) } func (jobDb *JobDb) internJobSchedulingInfoStrings(info *schedulerobjects.JobSchedulingInfo) *schedulerobjects.JobSchedulingInfo { diff --git a/internal/scheduler/jobdb/jobdb_test.go b/internal/scheduler/jobdb/jobdb_test.go index 4919456a36a..8183439bcd7 100644 --- a/internal/scheduler/jobdb/jobdb_test.go +++ b/internal/scheduler/jobdb/jobdb_test.go @@ -18,10 +18,12 @@ import ( "github.com/armadaproject/armada/internal/common/stringinterner" "github.com/armadaproject/armada/internal/common/types" "github.com/armadaproject/armada/internal/common/util" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) func NewTestJobDb() *JobDb { + emptyFloatingResourceTypes, _ := floatingresources.NewFloatingResourceTypes(nil) return NewJobDb( map[string]types.PriorityClass{ "foo": {}, @@ -30,6 +32,7 @@ func NewTestJobDb() *JobDb { "foo", stringinterner.New(1024), TestResourceListFactory, + emptyFloatingResourceTypes, ) } diff --git a/internal/scheduler/metrics.go b/internal/scheduler/metrics.go index d2ab9d34318..edda06c530c 100644 --- a/internal/scheduler/metrics.go +++ b/internal/scheduler/metrics.go @@ -15,6 +15,7 @@ import ( commonmetrics "github.com/armadaproject/armada/internal/common/metrics" "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/scheduler/database" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/queue" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -57,13 +58,14 @@ func (m metricProvider) GetRunningJobMetrics(queueName string) []*commonmetrics. // MetricsCollector is a Prometheus Collector that handles scheduler metrics. // The metrics themselves are calculated asynchronously every refreshPeriod type MetricsCollector struct { - jobDb *jobdb.JobDb - queueCache queue.QueueCache - executorRepository database.ExecutorRepository - poolAssigner PoolAssigner - refreshPeriod time.Duration - clock clock.WithTicker - state atomic.Value + jobDb *jobdb.JobDb + queueCache queue.QueueCache + executorRepository database.ExecutorRepository + poolAssigner PoolAssigner + refreshPeriod time.Duration + clock clock.WithTicker + state atomic.Value + floatingResourceTypes *floatingresources.FloatingResourceTypes } func NewMetricsCollector( @@ -72,15 +74,17 @@ func NewMetricsCollector( executorRepository database.ExecutorRepository, poolAssigner PoolAssigner, refreshPeriod time.Duration, + floatingResourceTypes *floatingresources.FloatingResourceTypes, ) *MetricsCollector { return &MetricsCollector{ - jobDb: jobDb, - queueCache: queueCache, - executorRepository: executorRepository, - poolAssigner: poolAssigner, - refreshPeriod: refreshPeriod, - clock: clock.RealClock{}, - state: atomic.Value{}, + jobDb: jobDb, + queueCache: queueCache, + executorRepository: executorRepository, + poolAssigner: poolAssigner, + refreshPeriod: refreshPeriod, + clock: clock.RealClock{}, + state: atomic.Value{}, + floatingResourceTypes: floatingResourceTypes, } } @@ -318,6 +322,17 @@ func (c *MetricsCollector) updateClusterMetrics(ctx *armadacontext.Context) ([]p } } + for _, pool := range c.floatingResourceTypes.AllPools() { + totalFloatingResources := c.floatingResourceTypes.GetTotalAvailableForPool(pool) + clusterKey := clusterMetricKey{ + cluster: "floating", + pool: pool, + nodeType: "", + } + addToResourceListMap(availableResourceByCluster, clusterKey, totalFloatingResources) + addToResourceListMap(totalResourceByCluster, clusterKey, totalFloatingResources) + } + clusterMetrics := make([]prometheus.Metric, 0, len(phaseCountByQueue)) for k, v := range phaseCountByQueue { clusterMetrics = append(clusterMetrics, commonmetrics.NewQueueLeasedPodCount(float64(v), k.cluster, k.pool, k.queueName, k.phase, k.nodeType)) diff --git a/internal/scheduler/metrics_test.go b/internal/scheduler/metrics_test.go index 53ad50e9c68..11a8ee455a3 100644 --- a/internal/scheduler/metrics_test.go +++ b/internal/scheduler/metrics_test.go @@ -13,6 +13,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" commonmetrics "github.com/armadaproject/armada/internal/common/metrics" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/jobdb" schedulermocks "github.com/armadaproject/armada/internal/scheduler/mocks" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -111,6 +112,7 @@ func TestMetricsCollector_TestCollect_QueueMetrics(t *testing.T) { executorRepository, poolAssigner, 2*time.Second, + testfixtures.TestEmptyFloatingResources, ) collector.clock = testClock err = collector.refresh(ctx) @@ -155,9 +157,10 @@ func TestMetricsCollector_TestCollect_ClusterMetrics(t *testing.T) { executorWithJobs := createExecutor("cluster-1", nodeWithJobs) tests := map[string]struct { - jobDbJobs []*jobdb.Job - executors []*schedulerobjects.Executor - expected []prometheus.Metric + jobDbJobs []*jobdb.Job + floatingResourceTypes *floatingresources.FloatingResourceTypes + executors []*schedulerobjects.Executor + expected []prometheus.Metric }{ "empty cluster single node type": { jobDbJobs: []*jobdb.Job{}, @@ -233,6 +236,15 @@ func TestMetricsCollector_TestCollect_ClusterMetrics(t *testing.T) { commonmetrics.NewClusterTotalCapacity(1, "cluster-1", testfixtures.TestPool, "nodes", "type-1"), }, }, + "floating resources": { + jobDbJobs: []*jobdb.Job{}, + floatingResourceTypes: testfixtures.TestFloatingResources, + executors: []*schedulerobjects.Executor{}, + expected: []prometheus.Metric{ + commonmetrics.NewClusterAvailableCapacity(10, "floating", "pool", "test-floating-resource", ""), + commonmetrics.NewClusterTotalCapacity(10, "floating", "pool", "test-floating-resource", ""), + }, + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -255,12 +267,17 @@ func TestMetricsCollector_TestCollect_ClusterMetrics(t *testing.T) { executorRepository := schedulermocks.NewMockExecutorRepository(ctrl) executorRepository.EXPECT().GetExecutors(ctx).Return(tc.executors, nil) + if tc.floatingResourceTypes == nil { + tc.floatingResourceTypes = testfixtures.TestEmptyFloatingResources + } + collector := NewMetricsCollector( jobDb, queueCache, executorRepository, poolAssigner, 2*time.Second, + tc.floatingResourceTypes, ) collector.clock = testClock err = collector.refresh(ctx) diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 5c98c3bc770..dc5e577e7c1 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -4,6 +4,8 @@ import ( "fmt" "testing" + "github.com/armadaproject/armada/internal/scheduler/adapters" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" @@ -123,7 +125,7 @@ func TestNodeBindingEvictionUnbinding(t *testing.T) { jobFilter := func(job *jobdb.Job) bool { return true } job := testfixtures.Test1GpuJob("A", testfixtures.PriorityClass0) request := job.EfficientResourceRequirements() - requestInternalRl, err := nodeDb.resourceListFactory.FromJobResourceListFailOnUnknown(job.ResourceRequirements().Requests) + requestInternalRl, err := nodeDb.resourceListFactory.FromJobResourceListFailOnUnknown(adapters.K8sResourceListToMap(job.ResourceRequirements().Requests)) assert.Nil(t, err) jobId := job.Id() diff --git a/internal/scheduler/nodedb/nodeiteration_test.go b/internal/scheduler/nodedb/nodeiteration_test.go index 9a61a6b1e92..af39f364e4a 100644 --- a/internal/scheduler/nodedb/nodeiteration_test.go +++ b/internal/scheduler/nodedb/nodeiteration_test.go @@ -437,7 +437,7 @@ func TestNodeTypeIterator(t *testing.T) { require.NoError(t, nodeDb.UpsertMany(entries)) indexedResourceRequests := make([]int64, len(testfixtures.TestResources)) - rr, err := testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(schedulerobjects.V1ResourceListFromResourceList(tc.resourceRequests)) + rr, err := testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(tc.resourceRequests.Resources) assert.Nil(t, err) for i, resourceName := range nodeDb.indexedResources { indexedResourceRequests[i], err = rr.GetByName(resourceName) @@ -830,7 +830,7 @@ func TestNodeTypesIterator(t *testing.T) { } require.NoError(t, nodeDb.UpsertMany(entries)) - rr, err := testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(schedulerobjects.V1ResourceListFromResourceList(tc.resourceRequests)) + rr, err := testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(tc.resourceRequests.Resources) assert.Nil(t, err) indexedResourceRequests := make([]int64, len(testfixtures.TestResources)) diff --git a/internal/scheduler/nodedb/nodematching_test.go b/internal/scheduler/nodedb/nodematching_test.go index 5ae54c65f8a..f84ff20699d 100644 --- a/internal/scheduler/nodedb/nodematching_test.go +++ b/internal/scheduler/nodedb/nodematching_test.go @@ -665,12 +665,12 @@ func makeTestNodeTaintsLabels(taints []v1.Taint, labels map[string]string) *inte } func makeTestNodeResources(t *testing.T, allocatableByPriority schedulerobjects.AllocatableByPriorityAndResourceType, totalResources schedulerobjects.ResourceList) *internaltypes.Node { - tr, err := testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(schedulerobjects.V1ResourceListFromResourceList(totalResources)) + tr, err := testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(totalResources.Resources) assert.Nil(t, err) abp := map[int32]internaltypes.ResourceList{} for pri, rl := range allocatableByPriority { - abp[pri], err = testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(schedulerobjects.V1ResourceListFromResourceList(rl)) + abp[pri], err = testfixtures.TestResourceListFactory.FromJobResourceListFailOnUnknown(rl.Resources) assert.Nil(t, err) } diff --git a/internal/scheduler/preempting_queue_scheduler.go b/internal/scheduler/preempting_queue_scheduler.go index 8732836cbc0..46d11625e0c 100644 --- a/internal/scheduler/preempting_queue_scheduler.go +++ b/internal/scheduler/preempting_queue_scheduler.go @@ -18,6 +18,7 @@ import ( schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/constraints" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" "github.com/armadaproject/armada/internal/scheduler/fairness" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" @@ -29,6 +30,7 @@ import ( type PreemptingQueueScheduler struct { schedulingContext *schedulercontext.SchedulingContext constraints schedulerconstraints.SchedulingConstraints + floatingResourceTypes *floatingresources.FloatingResourceTypes protectedFractionOfFairShare float64 useAdjustedFairShareProtection bool jobRepo JobRepository @@ -50,6 +52,7 @@ type PreemptingQueueScheduler struct { func NewPreemptingQueueScheduler( sctx *schedulercontext.SchedulingContext, constraints schedulerconstraints.SchedulingConstraints, + floatingResourceTypes *floatingresources.FloatingResourceTypes, protectedFractionOfFairShare float64, useAdjustedFairShareProtection bool, jobRepo JobRepository, @@ -74,6 +77,7 @@ func NewPreemptingQueueScheduler( return &PreemptingQueueScheduler{ schedulingContext: sctx, constraints: constraints, + floatingResourceTypes: floatingResourceTypes, protectedFractionOfFairShare: protectedFractionOfFairShare, useAdjustedFairShareProtection: useAdjustedFairShareProtection, jobRepo: jobRepo, @@ -155,7 +159,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche maps.Copy(sch.nodeIdByJobId, evictorResult.NodeIdByJobId) // Re-schedule evicted jobs/schedule new jobs. - ctx.WithField("stage", "scheduling-algo").Info("Performing initial scheduling jobs onto nodes") + ctx.WithField("stage", "scheduling-algo").Info("Performing initial scheduling of jobs onto nodes") schedulerResult, err := sch.schedule( armadacontext.WithLogField(ctx, "stage", "re-schedule after balancing eviction"), inMemoryJobRepo, @@ -552,6 +556,7 @@ func (sch *PreemptingQueueScheduler) schedule(ctx *armadacontext.Context, inMemo sched, err := NewQueueScheduler( sch.schedulingContext, sch.constraints, + sch.floatingResourceTypes, sch.nodeDb, jobIteratorByQueue, ) diff --git a/internal/scheduler/preempting_queue_scheduler_test.go b/internal/scheduler/preempting_queue_scheduler_test.go index 5e31e10553a..cfdb75d837b 100644 --- a/internal/scheduler/preempting_queue_scheduler_test.go +++ b/internal/scheduler/preempting_queue_scheduler_test.go @@ -51,7 +51,7 @@ func TestEvictOversubscribed(t *testing.T) { err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(nodeDbTxn, jobs, node) require.NoError(t, err) - jobDb := jobdb.NewJobDb(config.PriorityClasses, config.DefaultPriorityClassName, stringInterner, testfixtures.TestResourceListFactory) + jobDb := jobdb.NewJobDb(config.PriorityClasses, config.DefaultPriorityClassName, stringInterner, testfixtures.TestResourceListFactory, testfixtures.TestEmptyFloatingResources) jobDbTxn := jobDb.WriteTxn() err = jobDbTxn.Upsert(jobs) require.NoError(t, err) @@ -1737,7 +1737,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { priorities := types.AllowedPriorities(tc.SchedulingConfig.PriorityClasses) - jobDb := jobdb.NewJobDb(tc.SchedulingConfig.PriorityClasses, tc.SchedulingConfig.DefaultPriorityClassName, stringinterner.New(1024), testfixtures.TestResourceListFactory) + jobDb := jobdb.NewJobDb(tc.SchedulingConfig.PriorityClasses, tc.SchedulingConfig.DefaultPriorityClassName, stringinterner.New(1024), testfixtures.TestResourceListFactory, testfixtures.TestEmptyFloatingResources) jobDbTxn := jobDb.WriteTxn() // Accounting across scheduling rounds. @@ -1871,6 +1871,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { sch := NewPreemptingQueueScheduler( sctx, constraints, + testfixtures.TestEmptyFloatingResources, tc.SchedulingConfig.ProtectedFractionOfFairShare, tc.SchedulingConfig.UseAdjustedFairShareProtection, NewSchedulerJobRepositoryAdapter(jobDbTxn), @@ -2175,7 +2176,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { } txn.Commit() - jobDb := jobdb.NewJobDb(tc.SchedulingConfig.PriorityClasses, tc.SchedulingConfig.DefaultPriorityClassName, stringinterner.New(1024), testfixtures.TestResourceListFactory) + jobDb := jobdb.NewJobDb(tc.SchedulingConfig.PriorityClasses, tc.SchedulingConfig.DefaultPriorityClassName, stringinterner.New(1024), testfixtures.TestResourceListFactory, testfixtures.TestEmptyFloatingResources) jobDbTxn := jobDb.WriteTxn() var queuedJobs []*jobdb.Job for _, jobs := range jobsByQueue { @@ -2227,6 +2228,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { sch := NewPreemptingQueueScheduler( sctx, constraints, + testfixtures.TestEmptyFloatingResources, tc.SchedulingConfig.ProtectedFractionOfFairShare, tc.SchedulingConfig.UseAdjustedFairShareProtection, NewSchedulerJobRepositoryAdapter(jobDbTxn), @@ -2288,6 +2290,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { sch := NewPreemptingQueueScheduler( sctx, constraints, + testfixtures.TestEmptyFloatingResources, tc.SchedulingConfig.ProtectedFractionOfFairShare, tc.SchedulingConfig.UseAdjustedFairShareProtection, NewSchedulerJobRepositoryAdapter(jobDbTxn), diff --git a/internal/scheduler/queue_scheduler.go b/internal/scheduler/queue_scheduler.go index de7cb68dd5c..1251e3ea1da 100644 --- a/internal/scheduler/queue_scheduler.go +++ b/internal/scheduler/queue_scheduler.go @@ -13,6 +13,7 @@ import ( schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/constraints" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" "github.com/armadaproject/armada/internal/scheduler/fairness" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/nodedb" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) @@ -28,6 +29,7 @@ type QueueScheduler struct { func NewQueueScheduler( sctx *schedulercontext.SchedulingContext, constraints schedulerconstraints.SchedulingConstraints, + floatingResourceTypes *floatingresources.FloatingResourceTypes, nodeDb *nodedb.NodeDb, jobIteratorByQueue map[string]JobIterator, ) (*QueueScheduler, error) { @@ -36,7 +38,7 @@ func NewQueueScheduler( return nil, errors.Errorf("no scheduling context for queue %s", queue) } } - gangScheduler, err := NewGangScheduler(sctx, constraints, nodeDb) + gangScheduler, err := NewGangScheduler(sctx, constraints, floatingResourceTypes, nodeDb) if err != nil { return nil, err } diff --git a/internal/scheduler/queue_scheduler_test.go b/internal/scheduler/queue_scheduler_test.go index 7ac1bb43b64..582e0543bd4 100644 --- a/internal/scheduler/queue_scheduler_test.go +++ b/internal/scheduler/queue_scheduler_test.go @@ -589,7 +589,7 @@ func TestQueueScheduler(t *testing.T) { it := jobRepo.GetJobIterator(q.Name) jobIteratorByQueue[q.Name] = it } - sch, err := NewQueueScheduler(sctx, constraints, nodeDb, jobIteratorByQueue) + sch, err := NewQueueScheduler(sctx, constraints, testfixtures.TestEmptyFloatingResources, nodeDb, jobIteratorByQueue) require.NoError(t, err) result, err := sch.Schedule(armadacontext.Background()) diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 130a36d9af4..4750475a458 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -32,11 +32,13 @@ import ( "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/serve" + "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/common/stringinterner" "github.com/armadaproject/armada/internal/common/types" schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" "github.com/armadaproject/armada/internal/scheduler/database" "github.com/armadaproject/armada/internal/scheduler/failureestimator" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/leader" @@ -84,6 +86,12 @@ func Run(config schedulerconfig.Configuration) error { } ctx.Infof("Supported resource types: %s", resourceListFactory.SummaryString()) + floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(config.Scheduling.ExperimentalFloatingResources) + if err != nil { + return err + } + ctx.Infof("Floating resource types: %s", floatingResourceTypes.SummaryString()) + // List of services to run concurrently. // Because we want to start services only once all input validation has been completed, // we add all services to a slice and start them together at the end of this function. @@ -179,6 +187,7 @@ func Run(config schedulerconfig.Configuration) error { jobRepository, executorRepository, types.AllowedPriorities(config.Scheduling.PriorityClasses), + slices.Map(config.Scheduling.SupportedResourceTypes, func(rt schedulerconfig.ResourceType) string { return rt.Name }), config.Scheduling.NodeIdLabel, config.Scheduling.PriorityClassNameOverride, config.Scheduling.PriorityClasses, @@ -281,6 +290,7 @@ func Run(config schedulerconfig.Configuration) error { queueQuarantiner, stringInterner, resourceListFactory, + floatingResourceTypes, ) if err != nil { return errors.WithMessage(err, "error creating scheduling algo") @@ -290,6 +300,7 @@ func Run(config schedulerconfig.Configuration) error { config.Scheduling.DefaultPriorityClassName, stringInterner, resourceListFactory, + floatingResourceTypes, ) schedulingRoundMetrics := NewSchedulerMetrics(config.Metrics.Metrics) if err := prometheus.Register(schedulingRoundMetrics); err != nil { @@ -338,6 +349,7 @@ func Run(config schedulerconfig.Configuration) error { executorRepository, poolAssigner, config.Metrics.RefreshInterval, + floatingResourceTypes, ) if err := prometheus.Register(metricsCollector); err != nil { return errors.WithStack(err) diff --git a/internal/scheduler/scheduling_algo.go b/internal/scheduler/scheduling_algo.go index 0af6e58c435..f9e1e01f8e1 100644 --- a/internal/scheduler/scheduling_algo.go +++ b/internal/scheduler/scheduling_algo.go @@ -23,6 +23,7 @@ import ( schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" "github.com/armadaproject/armada/internal/scheduler/database" "github.com/armadaproject/armada/internal/scheduler/fairness" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" @@ -61,10 +62,11 @@ type FairSchedulingAlgo struct { // Used to reduce the rate at which jobs are scheduled from misbehaving queues. queueQuarantiner *quarantine.QueueQuarantiner // Function that is called every time an executor is scheduled. Useful for testing. - onExecutorScheduled func(executor *schedulerobjects.Executor) - clock clock.Clock - stringInterner *stringinterner.StringInterner - resourceListFactory *internaltypes.ResourceListFactory + onExecutorScheduled func(executor *schedulerobjects.Executor) + clock clock.Clock + stringInterner *stringinterner.StringInterner + resourceListFactory *internaltypes.ResourceListFactory + floatingResourceTypes *floatingresources.FloatingResourceTypes } func NewFairSchedulingAlgo( @@ -77,6 +79,7 @@ func NewFairSchedulingAlgo( queueQuarantiner *quarantine.QueueQuarantiner, stringInterner *stringinterner.StringInterner, resourceListFactory *internaltypes.ResourceListFactory, + floatingResourceTypes *floatingresources.FloatingResourceTypes, ) (*FairSchedulingAlgo, error) { if _, ok := config.PriorityClasses[config.DefaultPriorityClassName]; !ok { return nil, errors.Errorf( @@ -98,6 +101,7 @@ func NewFairSchedulingAlgo( clock: clock.RealClock{}, stringInterner: stringInterner, resourceListFactory: resourceListFactory, + floatingResourceTypes: floatingResourceTypes, }, nil } @@ -291,6 +295,10 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con } } + for pool, poolCapacity := range totalCapacityByPool { + poolCapacity.Add(l.floatingResourceTypes.GetTotalAvailableForPool(pool)) + } + // Create a map of jobs associated with each executor. jobsByExecutorId := make(map[string][]*jobdb.Job) nodeIdByJobId := make(map[string]string) @@ -495,6 +503,7 @@ func (l *FairSchedulingAlgo) scheduleOnExecutors( scheduler := NewPreemptingQueueScheduler( sctx, constraints, + l.floatingResourceTypes, l.schedulingConfig.ProtectedFractionOfFairShare, l.schedulingConfig.UseAdjustedFairShareProtection, NewSchedulerJobRepositoryAdapter(fsctx.txn), diff --git a/internal/scheduler/scheduling_algo_test.go b/internal/scheduler/scheduling_algo_test.go index 94825c2a287..558022a8782 100644 --- a/internal/scheduler/scheduling_algo_test.go +++ b/internal/scheduler/scheduling_algo_test.go @@ -380,6 +380,7 @@ func TestSchedule(t *testing.T) { nil, stringinterner.New(1024), testfixtures.TestResourceListFactory, + testfixtures.TestEmptyFloatingResources, ) require.NoError(t, err) @@ -532,6 +533,7 @@ func BenchmarkNodeDbConstruction(b *testing.B) { nil, stringInterner, testfixtures.TestResourceListFactory, + testfixtures.TestEmptyFloatingResources, ) require.NoError(b, err) b.StartTimer() diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index 059e4ee0deb..c901dc0388a 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -26,6 +26,7 @@ import ( schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/constraints" schedulercontext "github.com/armadaproject/armada/internal/scheduler/context" "github.com/armadaproject/armada/internal/scheduler/fairness" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" @@ -102,6 +103,11 @@ func NewSimulator(clusterSpec *ClusterSpec, workloadSpec *WorkloadSpec, scheduli if err != nil { return nil, errors.WithMessage(err, "Error with the .scheduling.supportedResourceTypes field in config") } + floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(schedulingConfig.ExperimentalFloatingResources) + if err != nil { + return nil, err + } + clusterSpec = proto.Clone(clusterSpec).(*ClusterSpec) workloadSpec = proto.Clone(workloadSpec).(*WorkloadSpec) initialiseClusterSpec(clusterSpec) @@ -117,6 +123,7 @@ func NewSimulator(clusterSpec *ClusterSpec, workloadSpec *WorkloadSpec, scheduli schedulingConfig.DefaultPriorityClassName, stringinterner.New(1024), resourceListFactory, + floatingResourceTypes, ) randomSeed := workloadSpec.RandomSeed if randomSeed == 0 { @@ -503,9 +510,16 @@ func (s *Simulator) handleScheduleEvent(ctx *armadacontext.Context) error { s.schedulingConfig, nil, ) + + nloatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(s.schedulingConfig.ExperimentalFloatingResources) + if err != nil { + return err + } + sch := scheduler.NewPreemptingQueueScheduler( sctx, constraints, + nloatingResourceTypes, s.schedulingConfig.ProtectedFractionOfFairShare, s.schedulingConfig.UseAdjustedFairShareProtection, scheduler.NewSchedulerJobRepositoryAdapter(txn), diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index 36bac3dfb58..fffd7d7a082 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -21,6 +21,7 @@ import ( "github.com/armadaproject/armada/internal/common/types" "github.com/armadaproject/armada/internal/common/util" schedulerconfiguration "github.com/armadaproject/armada/internal/scheduler/configuration" + "github.com/armadaproject/armada/internal/scheduler/floatingresources" "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -41,9 +42,22 @@ const ( ) var ( - TestResourceListFactory = MakeTestResourceListFactory() - BaseTime, _ = time.Parse("2006-01-02T15:04:05.000Z", "2022-03-01T15:04:05.000Z") - TestPriorityClasses = map[string]types.PriorityClass{ + TestResourceListFactory = MakeTestResourceListFactory() + TestEmptyFloatingResources = MakeTestFloatingResourceTypes(nil) + TestFloatingResources = MakeTestFloatingResourceTypes(TestFloatingResourceConfig) + TestFloatingResourceConfig = []schedulerconfiguration.FloatingResourceConfig{ + { + Name: "test-floating-resource", + Pools: []schedulerconfiguration.FloatingResourcePoolConfig{ + { + Name: "pool", + Quantity: resource.MustParse("10"), + }, + }, + }, + } + BaseTime, _ = time.Parse("2006-01-02T15:04:05.000Z", "2022-03-01T15:04:05.000Z") + TestPriorityClasses = map[string]types.PriorityClass{ PriorityClass0: {Priority: 0, Preemptible: true}, PriorityClass1: {Priority: 1, Preemptible: true}, PriorityClass2: {Priority: 2, Preemptible: true}, @@ -100,6 +114,7 @@ func NewJobDb(resourceListFactory *internaltypes.ResourceListFactory) *jobdb.Job SchedulingKeyGenerator, stringinterner.New(1024), resourceListFactory, + TestFloatingResources, ) // Mock out the clock and uuid provider to ensure consistent ids and timestamps are generated. jobDb.SetClock(NewMockPassiveClock()) @@ -954,6 +969,11 @@ func MakeTestResourceListFactory() *internaltypes.ResourceListFactory { return result } +func MakeTestFloatingResourceTypes(config []schedulerconfiguration.FloatingResourceConfig) *floatingresources.FloatingResourceTypes { + result, _ := floatingresources.NewFloatingResourceTypes(config) + return result +} + func GetTestSupportedResourceTypes() []schedulerconfiguration.ResourceType { return []schedulerconfiguration.ResourceType{ {Name: "memory", Resolution: resource.MustParse("1")},