Merge pull request #626 from agrare/collect_vsphere_tags

Collect vsphere tags and categories

app/models/manageiq/providers/vmware/infra_manager/inventory/collector.rb

@@ -3,11 +3,11 @@ class ManageIQ::Providers::Vmware::InfraManager::Inventory::Collector
   include Vmdb::Logging
 
   def initialize(ems)
-    @ems             = ems
-    @exit_requested  = false
-    @inventory_cache = ems.class::Inventory::Cache.new
-    @saver           = ems.class::Inventory::Saver.new
-    @vim_thread      = nil
+    @ems            = ems
+    @exit_requested = false
+    @cache          = ems.class::Inventory::Cache.new
+    @saver          = ems.class::Inventory::Saver.new
+    @vim_thread     = nil
   end
 
   def refresh
@@ -44,9 +44,11 @@ def restart(join_timeout = 2.minutes)
     self.vim_thread     = vim_collector_thread
   end
 
+  attr_accessor :cache, :categories_by_id, :tags_by_id, :tag_ids_by_attached_object
+
   private
 
-  attr_reader   :ems, :inventory_cache, :saver
+  attr_reader   :ems, :saver
   attr_accessor :exit_requested, :vim_thread, :last_full_refresh
 
   def vim_collector_thread
@@ -56,7 +58,7 @@ def vim_collector_thread
   def vim_collector
     _log.info("#{log_header} Monitor updates thread started")
 
-    vim = connect
+    vim = vim_connect
     property_filter = create_property_filter(vim, ems_inventory_filter_spec(vim))
 
     _log.info("#{log_header} Refreshing initial inventory")
@@ -80,15 +82,23 @@ def vim_collector
 
   def full_refresh(vim, property_filter)
     persister = full_persister_klass.new(ems)
-    parser    = parser_klass.new(inventory_cache, persister)
+    parser    = parser_klass.new(self, persister)
 
     version, updated_objects = monitor_updates(vim, property_filter, "")
 
+    if vim.rev >= '6.0' && vim.serviceContent.about.apiType == 'VirtualCenter'
+      cis_api_client = cis_connect
+      collect_cis_taggings(cis_api_client)
+    end
+
     parse_updates(vim, parser, updated_objects)
     parse_storage_profiles(vim, parser)
+
     if vim.rev >= '6.0' && vim.serviceContent.about.apiType == 'VirtualCenter'
-      parse_content_libraries(parser)
+      cis_api_client = cis_connect
+      parse_content_libraries(cis_api_client, parser)
     end
+
     save_inventory(persister)
 
     self.last_full_refresh = Time.now.utc
@@ -98,7 +108,7 @@ def full_refresh(vim, property_filter)
 
   def targeted_refresh(vim, property_filter, version)
     persister = targeted_persister_klass.new(ems)
-    parser    = parser_klass.new(inventory_cache, persister)
+    parser    = parser_klass.new(self, persister)
 
     version, updated_objects = monitor_updates(vim, property_filter, version)
 
@@ -122,7 +132,7 @@ def monitor_updates(vim, property_filter, version)
     return version, updated_objects
   end
 
-  def connect
+  def vim_connect
     host = ems.hostname
     username, password = ems.auth_user_pwd
 
@@ -153,6 +163,26 @@ def pbm_connect(vim)
     RbVmomi::PBM.connect(vim, :insecure => true)
   end
 
+  def cis_connect
+    require 'vsphere-automation-content'
+    require 'vsphere-automation-cis'
+
+    configuration = VSphereAutomation::Configuration.new.tap do |c|
+      c.host = ems.hostname
+      c.username = ems.auth_user_pwd.first
+      c.password = ems.auth_user_pwd.last
+      c.scheme = 'https'
+      c.verify_ssl = false
+      c.verify_ssl_host = false
+    end
+
+    api_client = VSphereAutomation::ApiClient.new(configuration)
+
+    VSphereAutomation::CIS::SessionApi.new(api_client).create('')
+
+    api_client
+  end
+
   def disconnect(vim)
     return if vim.nil?
 
@@ -227,17 +257,17 @@ def process_object_update(object_update)
   end
 
   def process_object_update_enter(obj, change_set, _missing_set = [])
-    inventory_cache.insert(obj, process_change_set(change_set))
+    cache.insert(obj, process_change_set(change_set))
   end
 
   def process_object_update_modify(obj, change_set, _missing_set = [])
-    inventory_cache.update(obj) do |props|
+    cache.update(obj) do |props|
       process_change_set(change_set, props)
     end
   end
 
   def process_object_update_leave(obj)
-    inventory_cache.delete(obj)
+    cache.delete(obj)
   end
 
   def retrieve_uncached_props(obj)
@@ -285,31 +315,40 @@ def retrieve_customization_spec(spec_manager, cached_props)
     end
   end
 
-  def parse_content_libraries(parser)
-    require 'vsphere-automation-content'
-    require 'vsphere-automation-cis'
+  def parse_content_libraries(api_client, parser)
+    library_api      = VSphereAutomation::Content::LibraryApi.new(api_client)
+    library_item_api = VSphereAutomation::Content::LibraryItemApi.new(api_client)
 
-    configuration = VSphereAutomation::Configuration.new.tap do |c|
-      c.host = ems.hostname
-      c.username = ems.auth_user_pwd.first
-      c.password = ems.auth_user_pwd.last
-      c.scheme = 'https'
-      c.verify_ssl = false
-      c.verify_ssl_host = false
+    library_api.list.value.each do |lib_id|
+      library_item_api.list(lib_id).value.each do |item_id|
+        parser.parse_content_library_item(library_item_api.get(item_id).value)
+      end
     end
+  rescue VSphereAutomation::ApiError
+    nil
+  end
 
-    api_client = VSphereAutomation::ApiClient.new(configuration)
-    VSphereAutomation::CIS::SessionApi.new(api_client).create('')
-    api_libs = VSphereAutomation::Content::LibraryApi.new(api_client)
-    api_items = VSphereAutomation::Content::LibraryItemApi.new(api_client)
+  def collect_cis_taggings(api_client)
+    tagging_category_api        = VSphereAutomation::CIS::TaggingCategoryApi.new(api_client)
+    tagging_tag_api             = VSphereAutomation::CIS::TaggingTagApi.new(api_client)
+    tagging_tag_association_api = VSphereAutomation::CIS::TaggingTagAssociationApi.new(api_client)
+
+    category_ids     = tagging_category_api.list.value.to_a
+    tag_ids          = tagging_tag_api.list.value.to_a
+
+    categories       = category_ids.map { |category_id| tagging_category_api.get(category_id).value }
+    tags             = tag_ids.map { |tag_id| tagging_tag_api.get(tag_id).value }
+
+    self.categories_by_id = categories.index_by(&:id)
+    self.tags_by_id       = tags.index_by(&:id)
+
+    self.tag_ids_by_attached_object = Hash.new { |h, k| h[k] = Hash.new { |h1, k1| h1[k1] = [] } }
 
-    api_libs.list.value.each do |lib_id|
-      api_items.list(lib_id).value.each do |item_id|
-        parser.parse_content_library_item(api_items.get(item_id).value)
+    tags.each do |tag|
+      tagging_tag_association_api.list_attached_objects(tag.id).value.to_a.each do |obj|
+        tag_ids_by_attached_object[obj.type][obj.id] << tag.id
       end
     end
-  rescue VSphereAutomation::ApiError
-    nil
   end
 
   def parse_storage_profiles(vim, parser)
@@ -349,11 +388,14 @@ def log_object_update(object_update)
     _log.debug do
       object_str = "#{object_update.obj.class.wsdl_name}:#{object_update.obj._ref}"
 
-      prop_changes = object_update.changeSet.map(&:name).take(5).join(", ")
-      prop_changes << ", ..." if object_update.changeSet.length > 5
-
       s =  "#{log_header} Object: [#{object_str}] Kind: [#{object_update.kind}]"
-      s << " Props: [#{prop_changes}]" if object_update.kind == "modify"
+      if object_update.kind == "modify"
+        prop_changes = object_update.changeSet.map(&:name).take(5).join(", ")
+        prop_changes << ", ..." if object_update.changeSet.length > 5
+
+        s << " Props: [#{prop_changes}]"
+      end
+
       s
     end
   end
@@ -367,7 +409,7 @@ def full_refresh_interval
   end
 
   def full_persister_klass
-    @full_persister_klass ||= ems.class::Inventory::Persister
+    @full_persister_klass ||= ems.class::Inventory::Persister::Full
   end
 
   def targeted_persister_klass

app/models/manageiq/providers/vmware/infra_manager/inventory/parser.rb

@@ -10,11 +10,12 @@ class ManageIQ::Providers::Vmware::InfraManager::Inventory::Parser
   include_concern :ResourcePool
   include_concern :VirtualMachine
 
-  attr_reader :cache, :persister
-  private     :cache, :persister
+  attr_reader :cache, :collector, :persister
+  private     :cache, :collector, :persister
 
-  def initialize(cache, persister)
-    @cache     = cache
+  def initialize(collector, persister)
+    @cache     = collector.cache
+    @collector = collector
     @persister = persister
   end
 
@@ -386,6 +387,7 @@ def parse_virtual_machine(object, kind, props)
     parse_virtual_machine_operating_system(vm, props)
     parse_virtual_machine_hardware(vm, props)
     parse_virtual_machine_custom_attributes(vm, props)
+    parse_virtual_machine_labels(vm, props)
     parse_virtual_machine_snapshots(vm, props)
   end
 

app/models/manageiq/providers/vmware/infra_manager/inventory/parser/virtual_machine.rb

@@ -354,6 +354,33 @@ def parse_virtual_machine_custom_attributes(vm, props)
       end
     end
 
+    def parse_virtual_machine_labels(vm, _props)
+      # Tags are only fetched on full refresh so during a targeted refresh the
+      # tag_ids_by_attached_object/tags_by_id/categories_by_id will be nil
+      tag_ids = collector.tag_ids_by_attached_object&.dig("VirtualMachine", vm.ems_ref)
+      return if tag_ids.blank?
+
+      persister_labels = tag_ids.map do |tag_id|
+        tag      = collector.tags_by_id[tag_id]
+        category = collector.categories_by_id[tag&.category_id]
+
+        next if tag.nil? || category.nil?
+
+        persister.vm_and_template_labels.build(
+          :resource    => vm,
+          :name        => category.name,
+          :section     => "labels",
+          :source      => "vmware",
+          :value       => tag.name,
+          :description => tag.description
+        )
+      end.compact
+
+      persister.tag_mapper.map_labels("VmOrTemplate", persister_labels).each do |tag|
+        persister.vm_and_template_taggings.build(:taggable => vm, :tag => tag)
+      end
+    end
+
     def parse_virtual_machine_snapshots(vm, props)
       snapshots = props[:snapshot]
       return if snapshots.blank?

app/models/manageiq/providers/vmware/infra_manager/inventory/persister.rb

@@ -1,5 +1,6 @@
 class ManageIQ::Providers::Vmware::InfraManager::Inventory::Persister < ManageIQ::Providers::Inventory::Persister
   require_nested :Batch
+  require_nested :Full
   require_nested :Targeted
 
   def initialize_inventory_collections
@@ -9,6 +10,8 @@ def initialize_inventory_collections
     add_collection(infra, :distributed_virtual_lans)
     add_collection(infra, :clusters)
     add_collection(infra, :ems_custom_attributes, :parent_inventory_collections => %i[vms_and_templates])
+    add_collection(infra, :vm_and_template_labels, :parent_inventory_collections => %i[vms_and_templates])
+    add_collection(infra, :vm_and_template_taggings, :parent_inventory_collections => %i[vms_and_templates])
     add_collection(infra, :ems_extensions)
     add_collection(infra, :ems_folders)
     add_collection(infra, :ems_licenses)

app/models/manageiq/providers/vmware/infra_manager/inventory/persister/full.rb

@@ -0,0 +1,7 @@
+class ManageIQ::Providers::Vmware::InfraManager::Inventory::Persister::Full < ManageIQ::Providers::Vmware::InfraManager::Inventory::Persister
+  def initialize_inventory_collections
+    super
+
+    initialize_tag_mapper
+  end
+end

spec/models/manageiq/providers/vmware/infra_manager/refresher_spec.rb

@@ -39,6 +39,61 @@
           assert_specific_vm
         end
       end
+
+      context "with taggings and labels" do
+        let(:category) do
+          require "vsphere-automation-cis"
+          VSphereAutomation::CIS::CisTaggingCategoryModel.new(
+            :id          => "urn:vmomi:InventoryServiceCategory:aece75c1-0157-498c-b7d9-43e0532ddce8:GLOBAL",
+            :name        => "Category1",
+            :description => "Description",
+            :cardinality => "SINGLE",
+            :used_by     => []
+          )
+        end
+
+        let(:tag) do
+          require "vsphere-automation-cis"
+          VSphereAutomation::CIS::CisTaggingTagModel.new(
+            :id          => "urn:vmomi:InventoryServiceTag:43b0c084-4e91-4950-8cc4-c81cb46b701f:GLOBAL",
+            :category_id => "urn:vmomi:InventoryServiceCategory:aece75c1-0157-498c-b7d9-43e0532ddce8:GLOBAL",
+            :name        => "Tag1",
+            :description => "Tag Description",
+            :used_by     => []
+          )
+        end
+
+        let!(:env_tag_mapping)         { FactoryBot.create(:tag_mapping_with_category, :label_name => "Category1") }
+        let(:env_tag_mapping_category) { env_tag_mapping.tag.classification }
+
+        before do
+          collector.categories_by_id           = {category.id => category}
+          collector.tags_by_id                 = {tag.id => tag}
+          collector.tag_ids_by_attached_object = {"VirtualMachine" => {"vm-21" => [tag.id]}}
+        end
+
+        it "saves vm labels" do
+          2.times { with_vcr { collector.refresh } }
+
+          ems.reload
+
+          expect(ems.vm_and_template_labels.count).to eq(1)
+
+          vm = ems.vms.find_by(:ems_ref => "vm-21")
+          expect(vm.labels.count).to eq(1)
+          expect(vm.labels.first).to have_attributes(
+            :section     => "labels",
+            :name        => "Category1",
+            :value       => "Tag1",
+            :resource    => vm,
+            :source      => "vmware",
+            :description => "Tag Description"
+          )
+          expect(vm.tags.count).to eq(1)
+          expect(vm.tags.first.category).to eq(env_tag_mapping_category)
+          expect(vm.tags.first.classification.description).to eq("Tag1")
+        end
+      end
     end
 
     context "targeted refresh" do
@@ -54,7 +109,6 @@
         end
       end
       let(:property_filter) { RbVmomi::VIM.PropertyFilter(vim, "session[6f2dcefd-41de-6dfb-0160-1ee1cc024553]") }
-      let(:cache)           { collector.send(:inventory_cache) }
 
       before do
         # Use the VCR to prime the cache and do the initial save_inventory
@@ -247,7 +301,7 @@
 
       def run_targeted_refresh(update_set)
         persister       = ems.class::Inventory::Persister::Targeted.new(ems)
-        parser          = ems.class::Inventory::Parser.new(cache, persister)
+        parser          = ems.class::Inventory::Parser.new(collector, persister)
         updated_objects = collector.send(:process_update_set, property_filter, update_set)
 
         collector.send(:parse_updates, vim, parser, updated_objects)