Merge pull request #96 from sapcc/metrics

introducing basic metrics

charts/kubernikus/templates/operator.yaml

@@ -14,6 +14,9 @@ spec:
       labels:
         app: kubernikus
         type: operator
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: {{ .Values.operator.metrics_port | quote }}
     spec:
       containers:
         - name: operator
@@ -31,4 +34,8 @@ spec:
             - --auth-project-domain={{ .Values.openstack.auth_project_domain }}
             - --kubernikus-domain={{ .Values.domain }}
             - --namespace={{ default "kubernikus" .Values.namespace }}
+            - --metric-port={{ default 9091 .Values.operator.metrics_port }}
             - --v={{ default 1 .Values.groundctl.log_level }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.operator.metrics_port }}

charts/kubernikus/values.yaml

@@ -22,3 +22,6 @@ revisionHistoryLimit: 3
 k8sniff:
   enabled: true
   #external_service_ip: ""
+
+operator:
+  metrics_port: 9091

glide.yaml

@@ -26,6 +26,7 @@ import:
 - package: github.com/imdario/mergo
   version: 0.2.2
 # Dependencies extracted from go-swagger 0.12.0
+- package: github.com/databus23/guttle
 - package: github.com/go-openapi/jsonpointer
   version: 779f45308c19820f1a69e9a4cd965f496e0da10f
 - package: github.com/go-openapi/loads
@@ -60,3 +61,10 @@ import:
   version: 03cfca65330da08a5a440053faf994a3c682b5bf
 - package: github.com/go-openapi/jsonreference
   version: 36d33bfe519efae5632669801b180bf1a245da3b
+- package: github.com/prometheus/prometheus
+  version: ^2.0.0
+  subpackages:
+  - client_go
+  - client_model
+  - common
+  - procfs

pkg/cmd/kubernikus/operator.go

@@ -49,6 +49,7 @@ func NewOperatorOptions() *Options {
 	options.AuthDomain = "Default"
 	options.KubernikusDomain = "kluster.staging.cloud.sap"
 	options.Namespace = "kubernikus"
+	options.MetricPort = 9091
 	options.Controllers = []string{"groundctl", "launchctl"}
 	return options
 }
@@ -68,6 +69,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.KubernikusProjectID, "kubernikus-projectid", o.KubernikusProjectID, "ID of the project the k*s control plane.")
 	flags.StringVar(&o.KubernikusNetworkID, "kubernikus-networkid", o.KubernikusNetworkID, "ID of the network the k*s control plane.")
 	flags.StringVar(&o.Namespace, "namespace", o.Namespace, "Restrict operator to resources in the given namespace")
+	flags.IntVar(&o.MetricPort, "metric-port", o.MetricPort, "Port on which metrics are exposed")
 	flags.StringSliceVar(&o.Controllers, "controllers", o.Controllers, "A list of controllers to enable.  Default is to enable all. controllers: groundctl, launchctl")
 }
 
@@ -90,6 +92,7 @@ func (o *Options) Run(c *cobra.Command) error {
 	wg := &sync.WaitGroup{}                            // Goroutines can add themselves to this to be waited on
 
 	go controller.NewKubernikusOperator(&o.KubernikusOperatorOptions).Run(stop, wg)
+	go controller.ExposeMetrics(o.MetricPort, stop, wg)
 
 	<-sigs // Wait for signals (this hangs until a signal arrives)
 	glog.Info("Shutting down...")

pkg/controller/ground.go

@@ -136,6 +136,7 @@ func (op *GroundControl) handler(key string) error {
 	} else {
 		kluster := obj.(*v1.Kluster)
 		glog.V(5).Infof("Handling kluster %v in phase %q", kluster.Name, kluster.Status.Phase)
+		setMetricStatusPhaseForKluster(kluster.GetName(), kluster.Status.Phase)
 
 		switch phase := kluster.Status.Phase; phase {
 		case models.KlusterPhasePending:

pkg/controller/launch.go

@@ -170,6 +170,9 @@ func (launchctl *LaunchControl) syncPool(kluster *v1.Kluster, pool *models.NodeP
 	starting := starting(nodes)
 	ready := running + starting
 
+	setMetricNodePoolSize(kluster.GetName(), pool.Name, pool.Image, pool.Flavor, pool.Size)
+	setMetricNodePoolStatus(kluster.GetName(), pool.Name, map[string]int64{"running": running, "starting": starting, "ready": ready})
+
 	switch {
 	case ready < pool.Size:
 		glog.V(3).Infof("[%v] Pool %v: Starting/Running/Total: %v/%v/%v. Too few nodes. Need to spawn more.", kluster.Name, pool.Name, starting, running, pool.Size)

pkg/controller/metrics.go

@@ -0,0 +1,137 @@
+package controller
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+
+	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sapcc/kubernikus/pkg/api/models"
+)
+
+const (
+	metricNamespace = "kubernikus"
+)
+
+var klusterInstancesTotal = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: metricNamespace,
+		Name:      "kluster_instances_total",
+		Help:      "total number of klusters",
+	},
+	[]string{"domain_id", "project_id"},
+)
+
+var klusterStatusPhase = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: metricNamespace,
+		Name:      "kluster_status_phase",
+		Help:      "The phase the kluster is currently in",
+	},
+	[]string{"kluster_id", "phase"},
+)
+
+var nodePoolInfo = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: metricNamespace,
+		Name:      "node_pool_info",
+		Help:      "information for a node pool",
+	},
+	[]string{"kluster_id", "node_pool", "image_name", "flavor_name"},
+)
+
+var nodePoolStatus = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: metricNamespace,
+		Name:      "node_pool_status",
+		Help:      "status of the node pool",
+	},
+	[]string{"kluster_id", "node_pool", "pool_size", "status"},
+)
+
+/*
+kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="<phase>"} 			< 1|0 >
+kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="creating"} 		1
+kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="running"} 			0
+kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="pending"} 			0
+kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="terminating"} 	0
+*/
+func setMetricStatusPhaseForKluster(klusterName string, klusterPhase models.KlusterPhase) {
+	// Set current phase to 1, others to 0 if it is set.
+	if klusterPhase != "" {
+		labels := prometheus.Labels{
+			"kluster_id": klusterName,
+			"phase":      string(klusterPhase),
+		}
+		klusterStatusPhase.With(labels).Set(boolToFloat64(klusterPhase == models.KlusterPhaseCreating))
+		klusterStatusPhase.With(labels).Set(boolToFloat64(klusterPhase == models.KlusterPhaseRunning))
+		klusterStatusPhase.With(labels).Set(boolToFloat64(klusterPhase == models.KlusterPhasePending))
+		klusterStatusPhase.With(labels).Set(boolToFloat64(klusterPhase == models.KlusterPhaseTerminating))
+	}
+}
+
+/*
+kubernikus_node_pool_info{"kluster_id"="<id", "node_pool"="<name>", "image_name"="<name>", "flavor_name"="<name>"} <node_pool_size>
+*/
+func setMetricNodePoolSize(klusterID, nodePoolName, imageName, flavorName string, nodePoolSize int64) {
+	nodePoolInfo.With(prometheus.Labels{
+		"kluster_id":  klusterID,
+		"node_pool":   nodePoolName,
+		"image_name":  imageName,
+		"flavor_name": flavorName,
+	}).Set(float64(nodePoolSize))
+}
+
+/*
+kubernikus_node_pool_status{"kluster_id"="<id", "node_pool"="<name>", "status"="<status>"} < number of nodes in that status >
+kubernikus_node_pool_status{"kluster_id"="<id", "node_pool"="<name>", "status"="ready"} 1
+kubernikus_node_pool_status{"kluster_id"="<id", "node_pool"="<name>", "status"="running"} 1
+kubernikus_node_pool_status{"kluster_id"="<id", "node_pool"="<name>", "status"="healthy"} 1
+*/
+func setMetricNodePoolStatus(klusterID, nodePoolName string, status map[string]int64) {
+	if status != nil {
+		for s, v := range status {
+			nodePoolStatus.With(prometheus.Labels{
+				"kluster_id": klusterID,
+				"node_pool":  nodePoolName,
+				"status":     s,
+			}).Set(float64(v))
+		}
+	}
+}
+
+func boolToFloat64(b bool) float64 {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+func init() {
+	prometheus.MustRegister(
+		klusterInstancesTotal,
+		klusterStatusPhase,
+		nodePoolInfo,
+		nodePoolStatus,
+	)
+}
+
+func ExposeMetrics(metricPort int, stopCh <-chan struct{}, wg *sync.WaitGroup) error {
+	glog.Infof("Exposing metrics on localhost:%v/metrics ", metricPort)
+	defer wg.Done()
+	wg.Add(1)
+	for {
+		select {
+		case <-stopCh:
+			return nil
+		default:
+			http.Handle("/metrics", promhttp.Handler())
+			return http.ListenAndServe(
+				fmt.Sprintf("0.0.0.0:%v", metricPort),
+				nil,
+			)
+		}
+	}
+}

pkg/controller/operator.go

@@ -51,6 +51,7 @@ type KubernikusOperatorOptions struct {
 	KubernikusNetworkID string
 	Namespace           string
 	Controllers         []string
+	MetricPort          int
 }
 
 type Clients struct {