From 8c0570d6d227a58be16455241999673f5d8c662d Mon Sep 17 00:00:00 2001 From: Daniel Czerwonk Date: Mon, 5 Aug 2019 20:15:48 +0200 Subject: [PATCH] add VM disks metrics --- disk/disk.go | 23 +++++++++ disk/helper.go | 28 +++++++++++ go.mod | 1 + go.sum | 1 + host/host.go | 2 + main.go | 27 ++++++++--- metric/helper.go | 1 + network/helper.go | 2 + network/nic.go | 3 ++ statistic/staticstic.go | 2 + storagedomain/helper.go | 52 ++++++++++++++++++++ storagedomain/storage_domain.go | 22 +++++---- vm/disk_attachment.go | 14 ++++++ vm/snapshot.go | 4 +- vm/vm.go | 2 + vm/vm_collector.go | 86 ++++++++++++++++++++++++++++----- 16 files changed, 242 insertions(+), 28 deletions(-) create mode 100644 disk/disk.go create mode 100644 disk/helper.go create mode 100644 storagedomain/helper.go create mode 100644 vm/disk_attachment.go diff --git a/disk/disk.go b/disk/disk.go new file mode 100644 index 0000000..199e8c6 --- /dev/null +++ b/disk/disk.go @@ -0,0 +1,23 @@ +package disk + +import "github.com/czerwonk/ovirt_exporter/storagedomain" + +// Disk represents the disk resource +type Disk struct { + ID string `xml:"id,attr"` + Name string `xml:"name,omitempty"` + Alias string `xml:"alias,omitempty"` + ProvisionedSize uint64 `xml:"provisioned_size,omitempty"` + ActualSize uint64 `xml:"actual_size,omitempty"` + TotalSize uint64 `xml:"total_size,omitempty"` + StorageDomains *storagedomain.StorageDomains `xml:"storage_domains,omitempty"` +} + +// StorageDomainName returns the name of the storage domain of the disk +func (d *Disk) StorageDomainName() string { + if len(d.StorageDomains.Domains) == 0 { + return "" + } + + return d.StorageDomains.Domains[0].Name +} diff --git a/disk/helper.go b/disk/helper.go new file mode 100644 index 0000000..7c12ed8 --- /dev/null +++ b/disk/helper.go @@ -0,0 +1,28 @@ +package disk + +import ( + "fmt" + + "github.com/czerwonk/ovirt_api/api" + "github.com/czerwonk/ovirt_exporter/storagedomain" +) + +// Get retrieves disk information +func Get(id string, client *api.Client) (*Disk, error) { + path := fmt.Sprintf("disks/%s", id) + + d := &Disk{} + err := client.GetAndParse(path, &d) + if err != nil { + return nil, err + } + + for i, dom := range d.StorageDomains.Domains { + d.StorageDomains.Domains[i] = storagedomain.StorageDomain{ + ID: dom.ID, + Name: storagedomain.Name(dom.ID, client), + } + } + + return d, nil +} diff --git a/go.mod b/go.mod index 485beca..b8be10b 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/czerwonk/ovirt_exporter require ( github.com/czerwonk/ovirt_api v0.0.0-20180321161247-63e3f014686c + github.com/pkg/errors v0.8.0 github.com/prometheus/client_golang v0.9.4 github.com/prometheus/common v0.4.1 ) diff --git a/go.sum b/go.sum index e169c0c..d420fdb 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/host/host.go b/host/host.go index c1de583..197700a 100644 --- a/host/host.go +++ b/host/host.go @@ -1,9 +1,11 @@ package host +// Hosts is a collection of Host type Hosts struct { Hosts []Host `xml:"host"` } +// Host represents the host resource type Host struct { ID string `xml:"id,attr"` Name string `xml:"name"` diff --git a/main.go b/main.go index 35c51ff..58fc0be 100644 --- a/main.go +++ b/main.go @@ -12,12 +12,13 @@ import ( "github.com/czerwonk/ovirt_exporter/host" "github.com/czerwonk/ovirt_exporter/storagedomain" "github.com/czerwonk/ovirt_exporter/vm" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/log" ) -const version string = "0.8.6" +const version string = "0.9.0" var ( showVersion = flag.Bool("version", false, "Print version information.") @@ -25,10 +26,12 @@ var ( metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") apiURL = flag.String("api.url", "https://localhost/ovirt-engine/api/", "API REST Endpoint") apiUser = flag.String("api.username", "user@internal", "API username") + apiPass = flag.String("api.password", "", "API password") apiPassFile = flag.String("api.password-file", "", "File containing the API password") apiInsecureCert = flag.Bool("api.insecure-cert", false, "Skip verification for untrusted SSL/TLS certificates") withSnapshots = flag.Bool("with-snapshots", true, "Collect snapshot metrics (can be time consuming in some cases)") withNetwork = flag.Bool("with-network", true, "Collect network metrics (can be time consuming in some cases)") + withDisks = flag.Bool("with-disks", true, "Collect disk metrics (can be time consuming in some cases)") debug = flag.Bool("debug", false, "Show verbose output (e.g. body of each response received from API)") collectorDuration = prometheus.NewHistogramVec( @@ -112,13 +115,12 @@ func connectAPI() (*api.Client, error) { opts = append(opts, api.WithInsecure()) } - b, err := ioutil.ReadFile(*apiPassFile) + pass, err := apiPassword() if err != nil { - return nil, err + return nil, errors.Wrap(err, "error while reading password file") } - apiPass := strings.Trim(string(b), "\n") - client, err := api.NewClient(*apiURL, *apiUser, apiPass, opts...) + client, err := api.NewClient(*apiURL, *apiUser, pass, opts...) if err != nil { return nil, err } @@ -126,9 +128,22 @@ func connectAPI() (*api.Client, error) { return client, err } +func apiPassword() (string, error) { + if *apiPassFile == "" { + return *apiPass, nil + } + + b, err := ioutil.ReadFile(*apiPassFile) + if err != nil { + return "", err + } + + return strings.Trim(string(b), "\n"), nil +} + func handleMetricsRequest(w http.ResponseWriter, r *http.Request, client *api.Client, appReg *prometheus.Registry) { reg := prometheus.NewRegistry() - reg.MustRegister(vm.NewCollector(client, *withSnapshots, *withNetwork, collectorDuration.WithLabelValues("vm"))) + reg.MustRegister(vm.NewCollector(client, *withSnapshots, *withNetwork, *withDisks, collectorDuration.WithLabelValues("vm"))) reg.MustRegister(host.NewCollector(client, *withNetwork, collectorDuration.WithLabelValues("host"))) reg.MustRegister(storagedomain.NewCollector(client, collectorDuration.WithLabelValues("storage"))) diff --git a/metric/helper.go b/metric/helper.go index 596f3b9..92a062b 100644 --- a/metric/helper.go +++ b/metric/helper.go @@ -2,6 +2,7 @@ package metric import "github.com/prometheus/client_golang/prometheus" +// MustCreate creates a new prometheus metric func MustCreate(desc *prometheus.Desc, v float64, labelValues []string) prometheus.Metric { return prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(v), labelValues...) } diff --git a/network/helper.go b/network/helper.go index 3175728..70cded5 100644 --- a/network/helper.go +++ b/network/helper.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +// CollectMetricsForHost collects net metrics for a specific Host func CollectMetricsForHost(path, prefix string, labelNames, labelValues []string, client *api.Client, ch chan<- prometheus.Metric) error { nics := &HostNics{} err := client.GetAndParse(path, nics) @@ -20,6 +21,7 @@ func CollectMetricsForHost(path, prefix string, labelNames, labelValues []string return collectForNics(nics.Nics, path, prefix, labelNames, labelValues, client, ch) } +// CollectMetricsForVM collects net metrics for a specific VM func CollectMetricsForVM(path, prefix string, labelNames, labelValues []string, client *api.Client, ch chan<- prometheus.Metric) error { nics := &VMNics{} err := client.GetAndParse(path, nics) diff --git a/network/nic.go b/network/nic.go index e6a38b5..36bb43d 100644 --- a/network/nic.go +++ b/network/nic.go @@ -1,13 +1,16 @@ package network +// HostNics is a collection of NICs of a host type HostNics struct { Nics []Nic `xml:"host_nic"` } +// VMNics is a collection of NICs of a VM type VMNics struct { Nics []Nic `xml:"nic"` } +// Nic represents the network interface card resource type Nic struct { ID string `xml:"id,attr"` Name string `xml:"name"` diff --git a/statistic/staticstic.go b/statistic/staticstic.go index cc6a7de..955bf5a 100644 --- a/statistic/staticstic.go +++ b/statistic/staticstic.go @@ -1,9 +1,11 @@ package statistic +// Statistics is a collection of Statistics type Statistics struct { Statistic []Statistic `xml:"statistic"` } +// Statistic represents the statistic resource type Statistic struct { Name string `xml:"name"` Description string `xml:"description"` diff --git a/storagedomain/helper.go b/storagedomain/helper.go new file mode 100644 index 0000000..0e1f237 --- /dev/null +++ b/storagedomain/helper.go @@ -0,0 +1,52 @@ +package storagedomain + +import ( + "sync" + + "fmt" + + "github.com/czerwonk/ovirt_api/api" + "github.com/prometheus/common/log" +) + +var ( + cacheMutex = sync.Mutex{} + nameCache = make(map[string]string) +) + +// Get retrieves domain information +func Get(id string, client *api.Client) (*StorageDomain, error) { + path := fmt.Sprintf("storagedomains/%s", id) + + d := StorageDomain{} + err := client.GetAndParse(path, &d) + if err != nil { + return nil, err + } + + return &d, nil +} + +// Name retrieves domain name +func Name(id string, client *api.Client) string { + cacheMutex.Lock() + defer cacheMutex.Unlock() + + if n, found := nameCache[id]; found { + return n + } + + d, err := Get(id, client) + if err != nil { + log.Error(err) + return "" + } + + if (d == nil) { + log.Errorf("could not find name for storage domain with ID %s", id) + return "" + } + + nameCache[id] = d.Name + return d.Name +} \ No newline at end of file diff --git a/storagedomain/storage_domain.go b/storagedomain/storage_domain.go index 16fedfb..815219e 100644 --- a/storagedomain/storage_domain.go +++ b/storagedomain/storage_domain.go @@ -1,25 +1,27 @@ package storagedomain +// StorageDomains is a collection of storage domains type StorageDomains struct { Domains []StorageDomain `xml:"storage_domain"` } +// StorageDomain represents the storage domain resource type StorageDomain struct { ID string `xml:"id,attr"` - Name string `xml:"name"` + Name string `xml:"name,omitempty"` Storage struct { - Path string `xml:"path"` - Type string `xml:"type"` + Path string `xml:"path,omitempty"` + Type string `xml:"type,omitempty"` } `xml:"storage,omitempty"` - Type string `xml:"type"` - Available float64 ` xml:"available"` - Committed float64 `xml:"committed"` - Used float64 `xml:"used"` - ExternalStatus string `xml:"external_status"` - Master bool `xml:"master"` + Type string `xml:"type,omitempty"` + Available float64 ` xml:"available,omitempty"` + Committed float64 `xml:"committed,omitempty"` + Used float64 `xml:"used,omitempty"` + ExternalStatus string `xml:"external_status,omitempty"` + Master bool `xml:"master,omitempty"` DataCenters struct { DataCenter struct { ID string `xml:"id,attr"` } `xml:"data_center"` - } `xml:"data_centers"` + } `xml:"data_centers,omitempty"` } diff --git a/vm/disk_attachment.go b/vm/disk_attachment.go new file mode 100644 index 0000000..0a8517e --- /dev/null +++ b/vm/disk_attachment.go @@ -0,0 +1,14 @@ +package vm + +import "github.com/czerwonk/ovirt_exporter/disk" + +// DiskAttachments is a collection of diskattachments +type DiskAttachments struct { + Attachment []DiskAttachment `xml:"disk_attachment"` +} + +// DiskAttachment represents the diskattachment resource +type DiskAttachment struct { + LogicalName string `xml:"logical_name"` + Disk disk.Disk `xml:"disk"` +} diff --git a/vm/snapshot.go b/vm/snapshot.go index 7d335a6..21b7763 100644 --- a/vm/snapshot.go +++ b/vm/snapshot.go @@ -2,12 +2,14 @@ package vm import "time" +// Snapshots is a collection of snapshots type Snapshots struct { Snapshot []Snapshot `xml:"snapshot"` } +// Snapshot repesents the snapshot resource type Snapshot struct { - Id string `xml:"id,attr"` + ID string `xml:"id,attr"` Description string `xml:"description"` Date time.Time `xml:"date"` PersistMemorystate bool `xml:"persist_memorystate"` diff --git a/vm/vm.go b/vm/vm.go index 2fa2a2a..6b766fc 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -1,9 +1,11 @@ package vm +// VMs is a collection of virtual machines type VMs struct { VMs []VM `xml:"vm"` } +// VM represents the virutal machine resource type VM struct { ID string `xml:"id,attr"` Name string `xml:"name"` diff --git a/vm/vm_collector.go b/vm/vm_collector.go index 9644228..b120b30 100644 --- a/vm/vm_collector.go +++ b/vm/vm_collector.go @@ -9,6 +9,7 @@ import ( "github.com/czerwonk/ovirt_api/api" "github.com/czerwonk/ovirt_exporter/cluster" + "github.com/czerwonk/ovirt_exporter/disk" "github.com/czerwonk/ovirt_exporter/host" "github.com/czerwonk/ovirt_exporter/metric" "github.com/czerwonk/ovirt_exporter/network" @@ -20,15 +21,18 @@ import ( const prefix = "ovirt_vm_" var ( - upDesc *prometheus.Desc - cpuCoresDesc *prometheus.Desc - cpuSocketsDesc *prometheus.Desc - cpuThreadsDesc *prometheus.Desc - snapshotCount *prometheus.Desc - minSnapshotAge *prometheus.Desc - maxSnapshotAge *prometheus.Desc - illegalImages *prometheus.Desc - labelNames []string + upDesc *prometheus.Desc + cpuCoresDesc *prometheus.Desc + cpuSocketsDesc *prometheus.Desc + cpuThreadsDesc *prometheus.Desc + snapshotCount *prometheus.Desc + minSnapshotAge *prometheus.Desc + maxSnapshotAge *prometheus.Desc + illegalImages *prometheus.Desc + diskProvisionedSize *prometheus.Desc + diskActualSize *prometheus.Desc + diskTotalSize *prometheus.Desc + labelNames []string ) func init() { @@ -41,6 +45,12 @@ func init() { maxSnapshotAge = prometheus.NewDesc(prefix+"snapshot_max_age_seconds", "Age of the oldest snapshot in seconds", labelNames, nil) minSnapshotAge = prometheus.NewDesc(prefix+"snapshot_min_age_seconds", "Age of the newest snapshot in seconds", labelNames, nil) illegalImages = prometheus.NewDesc(prefix+"illegal_images", "Health status of the disks attatched to the VM (1 if one or more disk is in illegal state)", labelNames, nil) + + diskLabelNames := append(labelNames, "disk_name", "disk_alias", "disk_logical_name", "storage_domain") + diskProvisionedSize = prometheus.NewDesc(prefix+"disk_provisioned_size_bytes", "Provisioned size of the disk in bytes", diskLabelNames, nil) + diskActualSize = prometheus.NewDesc(prefix+"disk_actual_size_bytes", "Actual size of the disk in bytes", diskLabelNames, nil) + diskTotalSize = prometheus.NewDesc(prefix+"disk_total_size_bytes", "Total size of the disk in bytes", diskLabelNames, nil) + } // VMCollector collects virtual machine statistics from oVirt @@ -50,12 +60,18 @@ type VMCollector struct { metrics []prometheus.Metric collectSnapshots bool collectNetwork bool + collectDisks bool mutex sync.Mutex } // NewCollector creates a new collector -func NewCollector(client *api.Client, collectSnaphots, collectNetwork bool, collectDuration prometheus.Observer) prometheus.Collector { - return &VMCollector{client: client, collectSnapshots: collectSnaphots, collectNetwork: collectNetwork, collectDuration: collectDuration} +func NewCollector(client *api.Client, collectSnaphots, collectNetwork bool, collectDisks bool, collectDuration prometheus.Observer) prometheus.Collector { + return &VMCollector{ + client: client, + collectSnapshots: collectSnaphots, + collectNetwork: collectNetwork, + collectDisks: collectDisks, + collectDuration: collectDuration} } // Collect implements Prometheus Collector interface @@ -135,6 +151,10 @@ func (c *VMCollector) collectForVM(vm VM, ch chan<- prometheus.Metric, wg *sync. if c.collectSnapshots { c.collectSnapshotMetrics(v, ch, l) } + + if c.collectDisks { + c.collectDiskMetrics(v, ch, l) + } } func (c *VMCollector) collectCPUMetrics(vm *VM, ch chan<- prometheus.Metric, l []string) { @@ -193,3 +213,47 @@ func (c *VMCollector) collectSnapshotMetrics(vm *VM, ch chan<- prometheus.Metric max := s[len(s)-1] ch <- metric.MustCreate(minSnapshotAge, time.Since(max.Date).Seconds(), l) } + +func (c *VMCollector) collectDiskMetrics(vm *VM, ch chan<- prometheus.Metric, l []string) { + attchs := DiskAttachments{} + path := fmt.Sprintf("vms/%s/diskattachments", vm.ID) + + err := c.client.GetAndParse(path, &attchs) + if err != nil { + log.Error(err) + return + } + + if len(attchs.Attachment) == 0 { + return + } + + wg := &sync.WaitGroup{} + wg.Add(len(attchs.Attachment)) + + for _, a := range attchs.Attachment { + go c.collectForAttachment(a, ch, l, wg) + } + + wg.Wait() +} + +func (c *VMCollector) collectForAttachment(attachment DiskAttachment, ch chan<- prometheus.Metric, l []string, wg *sync.WaitGroup) { + defer wg.Done() + + d, err := disk.Get(attachment.Disk.ID, c.client) + if err != nil { + log.Error(err) + return + } + + if d == nil { + log.Error("could not find disk with ID " + attachment.Disk.ID) + return + } + + l = append(l, d.Name, d.Alias, attachment.LogicalName, d.StorageDomainName()) + ch <- metric.MustCreate(diskProvisionedSize, float64(d.ProvisionedSize), l) + ch <- metric.MustCreate(diskActualSize, float64(d.ActualSize), l) + ch <- metric.MustCreate(diskTotalSize, float64(d.TotalSize), l) +}