Merge pull request #41 from gubbe505/UpdateInventoryRefresh

Update of Inventory Refresh

app/models/manageiq/providers/nsxt/inventory/parser/network_manager.rb

@@ -12,21 +12,26 @@ def network_services
       network_service = persister.network_services.find_or_build(service['id'])
       network_service.name = service['display_name']
       network_service.description = service['description']
-      network_service_entries(service)
+      network_service.shared = service['is_default']
+      network_service_entries(service, network_service)
     end
   end
 
-  def network_service_entries(service)
+  def network_service_entries(service, network_service)
     service['service_entries'].each do |service_entry|
-      id = "#{service['id']}-#{service_entry['id']}"
-      network_service_entry = persister.network_service_entries.find_or_build(id)
+      service_entry_id = service_entry['unique_id'] || service_entry['id']
+      network_service_entry = persister.network_service_entries.find_or_build("#{service['id']}-#{service_entry_id}")
       network_service_entry.name = service_entry['display_name']
-      network_service_entry.network_service = persister.network_services.lazy_find(service['id'])
+      network_service_entry.network_service = network_service
       if service_entry['resource_type'] == 'L4PortSetServiceEntry'
         network_service_entry.protocol = service_entry['l4_protocol']
         network_service_entry.source_ports = network_service_entry_ports(service_entry['source_ports'])
         network_service_entry.destination_ports = network_service_entry_ports(service_entry['destination_ports'])
-      else
+      elsif service_entry['resource_type'] == 'ALGTypeServiceEntry'
+        network_service_entry.protocol = service_entry['alg']
+        network_service_entry.source_ports = network_service_entry_ports(service_entry['source_ports'])
+        network_service_entry.destination_ports = network_service_entry_ports(service_entry['destination_ports'])
+      elsif service_entry['resource_type'] == 'ICMPTypeServiceEntry'
         network_service_entry.protocol = service_entry['protocol']
       end
     end
@@ -48,81 +53,122 @@ def network_routers
 
   def cloud_networks
     collector.segments.each do |segment|
+      next if segment['type'] == 'DISCONNECTED'
       cloud_network = persister.cloud_networks.find_or_build(segment['id'])
       cloud_network.name = segment['display_name']
       cloud_network.description = segment['description']
       cloud_network.cloud_tenant = cloud_tenant(segment['tags'])
       cloud_network.status = 'active'
       cloud_network.enabled = true
-      cloud_network.extra_attributes = { :security_zone => get_tag_value_by_scope(segment['tags'], 'network_zone') }
-      cloud_subnets(segment)
+      network_router_id = segment['connectivity_path']&.split('/tier-1s/')&.last
+      network_router = persister.network_routers.lazy_find(network_router_id) unless network_router_id.nil?
+      cloud_subnets(segment, network_router)
     end
   end
 
-  def cloud_subnets(segment)
+  def cloud_subnets(segment, network_router)
     return if segment['subnets'].nil?
     segment['subnets'].each do |segment_subnet|
-      id = "#{segment['id']}-#{segment_subnet['gateway_address']}"
-      network_router_id = segment['connectivity_path'].split('/tier-1s/').last
+      id = "#{segment['id']}-#{segment_subnet['network']}"
       cloud_subnet = persister.cloud_subnets.find_or_build(id)
-      cloud_subnet.name = "#{segment['display_name']}-#{segment_subnet['gateway_address']}"
+      cloud_subnet.name = "#{segment['display_name']}-#{segment_subnet['network']}"
       cloud_subnet.cloud_tenant = cloud_tenant(segment['tags'])
       cloud_subnet.cidr = segment_subnet['network']
-      cloud_subnet.gateway = segment_subnet['gateway_address']
+      cloud_subnet.gateway = segment_subnet['gateway_address']&.split('/')&.first
       cloud_subnet.dhcp_enabled = false
       cloud_subnet.cloud_network = persister.cloud_networks.lazy_find(segment['id'])
-      cloud_subnet.network_router = persister.network_routers.lazy_find(network_router_id)
+      cloud_subnet.network_router = network_router
 
-      # TODO: this depends on vm.instance_uuid which doesn't exist yet
-      # network_ports(segment, cloud_subnet)
+      network_ports(segment, cloud_subnet)
     end
   end
 
   def network_ports(segment, cloud_subnet)
-    Lan.where(:ems_ref => segment['id']).each do |lan|
-      lan.vms.each do |vm|
-        return if cloud_subnet.cloud_tenant.nil?
-        return if vm.tenant_id != cloud_subnet.cloud_tenant.source_tenant.id
-        network_port = persister.network_ports.find_or_build(vm.instance_uuid)
-        network_port.name = vm.name
-        network_port.cloud_tenant = cloud_subnet.cloud_tenant
-        network_port.cloud_subnets = [] if network_port.cloud_subnets.nil?
-        network_port.cloud_subnets << cloud_subnet
-        network_port.status = 'active'
-        network_port.device = vm
-      end
-    end
+  #   # TODO: this depends on vm.instance_uuid which doesn't exist yet
+  #   Lan.where(:ems_ref => segment['id']).each do |lan|
+  #     lan.vms.each do |vm|
+  #       next if cloud_subnet&.cloud_tenant&.source_tenant.nil?
+  #       next if vm.tenant_id != cloud_subnet.cloud_tenant.source_tenant.id
+  #       network_port = persister.network_ports.find_or_build("#{segment['id']}_#{vm.instance_uuid}")
+  #       network_port.name = vm.name
+  #       network_port.device = vm
+  #       network_port.device_ref = vm.instance_uuid
+  #       network_port.cloud_tenant = cloud_subnet.cloud_tenant
+  #       network_port.cloud_subnets = [] if network_port.cloud_subnets.nil?
+  #       network_port.cloud_subnets << cloud_subnet
+  #       network_port.status = 'active'
+  #     end
+  #   end
   end
 
   def security_groups
     collector.groups.each do |group|
       next if group['id'] == 'ANY'
+
       security_group = persister.security_groups.find_or_build(group['id'])
       security_group.name = group['display_name']
       security_group.description = group['description']
       security_group.cloud_tenant = cloud_tenant(group['tags'])
       security_group.network_ports = []
-      security_groups_network_ports(security_group)
+      security_groups_network_ports_vm(group, security_group)
+      security_groups_network_ports_subnets(group, security_group)
     end
   end
 
-  def security_groups_network_ports(security_group)
+  def security_groups_network_ports_vm(group, security_group)
     # TODO: This depends on vm.instance_uuid which doesn't exist yet
-    # group_members = collector.group_members(security_group.ems_ref)
-    # group_members.each do |group_member|
-    #   vm = Vm.find_by(:instance_uuid => group_member['id'])
-    #   next if vm.nil?
-    #   network_port = persister.network_ports.find_or_build(group_member['id'])
-    #   network_port.name = group_member['display_name']
-    #   network_port.cloud_tenant = security_group.cloud_tenant
-    #   network_port.status = 'active'
-    #   network_port.device = vm
-    #   network_port.device_ref = group_member['id']
-    #   network_port.security_groups = [] if network_port.security_groups.nil?
-    #   network_port.security_groups << security_group
+    # return if group['expression'].nil?
+
+    # expressions = group['expression'].select do |expression|
+    #   expression['resource_type'] == 'ExternalIDExpression' && expression['member_type'] == 'VirtualMachine'
+    # end
+    # expressions.each do |expression|
+    #   expression['external_ids'].each do |external_id|
+    #     vm = Vm.find_by(:instance_uuid => external_id)
+    #     next if vm.nil?
+
+    #     network_port = persister.network_ports.find_or_build("#{group['id']}#{external_id}")
+    #     network_port.name = vm.name
+    #     network_port.cloud_tenant = security_group.cloud_tenant
+    #     network_port.status = 'active'
+    #     network_port.device = vm
+    #     network_port.device_ref = external_id
+    #     network_port.security_groups = [] if network_port.security_groups.nil?
+    #     network_port.security_groups << security_group
+    #   end
     # end
   end
 
+  def security_groups_network_ports_subnets(group, security_group)
+    return if group['expression'].nil?
+
+    expressions = group['expression'].select do |expression|
+      expression['resource_type'] == 'IPAddressExpression'
+    end
+    expressions.each_with_index do |expression, index|
+      network_port = persister.network_ports.find_or_build(expression['id'])
+      network_port.name = "#{group['display_name']} #{index + 1}"
+      network_port.cloud_tenant = security_group.cloud_tenant
+      network_port.status = 'active'
+      network_port.device = security_group
+      network_port.device_ref = group['id']
+      security_group.network_ports = [] if security_group.network_ports.nil?
+      security_group.network_ports << network_port
+
+      expression['ip_addresses'].each do |ip_address|
+        cloud_subnet = persister.cloud_subnets.find_or_build("external_#{ip_address}")
+        cloud_subnet.name = ip_address
+        cloud_subnet.cidr = ip_address
+        cloud_subnet.dhcp_enabled = false
+        cloud_subnet.network_ports = [] if cloud_subnet.network_ports.nil?
+        cloud_subnet.network_ports << network_port
+
+        network_port.cloud_subnets = [] if network_port.cloud_subnets.nil?
+        network_port.cloud_subnets << cloud_subnet
+      end
+    end
+  end
+
   def security_policies
     collector.security_policies.each do |policy|
       security_policy = persister.security_policies.find_or_build(policy['id'])
@@ -146,24 +192,17 @@ def security_policy_rules(security_policy)
       security_policy_rule.action = rule['action']
       security_policy_rule.direction = rule['direction']
       security_policy_rule.ip_protocol = rule['ip_protocol']
-      security_policy_rule.source_security_groups = [] if security_policy_rule.source_security_groups.nil?
-      rule['source_groups'].each do |group|
-        next if group == 'ANY'
-        security_group = persister.security_groups.lazy_find(group.split('/groups/').last)
-        security_policy_rule.source_security_groups << security_group unless security_group.nil?
-      end
-      security_policy_rule.destination_security_groups = [] if security_policy_rule.destination_security_groups.nil?
-      rule['destination_groups'].each do |group|
-        next if group == 'ANY'
-        security_group = persister.security_groups.lazy_find(group.split('/groups/').last)
-        security_policy_rule.destination_security_groups << security_group unless security_group.nil?
-      end
-      security_policy_rule.network_services = [] if security_policy_rule.network_services.nil?
-      rule['services'].each do |service|
-        next if service == 'ANY'
-        network_service = persister.network_services.lazy_find(service.split('/services/').last)
-        security_policy_rule.network_services << network_service unless network_service.nil?
-      end
+      security_policy_rule.sources_excluded = rule['sources_excluded'].present? ? rule['sources_excluded'] : false
+      security_policy_rule.destinations_excluded = rule['destinations_excluded'].present? ? rule['destinations_excluded'] : false
+      security_policy_rule.source_security_groups = rule['source_groups']
+        .map{|group| persister.security_groups.lazy_find(group.split('/groups/').last)}
+        .compact
+      security_policy_rule.destination_security_groups = rule['destination_groups']
+        .map{|group| persister.security_groups.lazy_find(group.split('/groups/').last)}
+        .compact
+      security_policy_rule.network_services = rule['services']
+        .map{|service| persister.network_services.lazy_find(service.split('/services/').last)}
+        .compact
     end
   end
 
@@ -178,11 +217,9 @@ def get_tag_value_by_scope(tags, scope)
   def cloud_tenant(tags)
     cloud_tenant_tag = get_tag_value_by_scope(tags, 'tenant')
     return nil if cloud_tenant_tag.nil?
-
     cloud_tenant_id = cloud_tenant_tag.upcase
     cloud_tenant = persister.cloud_tenants.find_or_build(cloud_tenant_id)
     cloud_tenant.name = cloud_tenant_id
-    cloud_tenant.source_tenant = Tenant.find_by_name(cloud_tenant_id)
     return cloud_tenant
   end
 end

app/models/manageiq/providers/nsxt/network_manager/security_policy_rule.rb

@@ -11,35 +11,5 @@ def self.nsxt_type
     def self.refresh_type
       :security_policies
     end
-
-    private
-
-    def self.map_groups_to_list(ids, options)
-      raise "The list must contain at least 1 group" [] if ids.nil? || 1 < ids.size == 0
-      external_ids = []
-      ids.each do |id|
-        group = SecurityGroup.find(id)
-        raise "Security Group with #{id} does not exists (anymore)" if group.nil?
-        external_ids << group.ems_ref
-      end
-      return external_ids
-    rescue StandardError => error
-      self.create_notification(options, error)
-      raise error
-    end
-
-    def self.map_services_to_list(ids, options)
-      raise "The list must contain at least 1 service" [] if ids.nil? || ids.size == 0
-      external_ids = []
-      ids.each do |id|
-        service = NetworkService.find(id)
-        raise "Network Service with #{id} does not exists (anymore)" if service.nil?
-        external_ids << service.ems_ref
-      end
-      return external_ids
-    rescue StandardError => error
-      self.create_notification(options, error)
-      raise error
-    end
   end
 end

app/models/manageiq/providers/nsxt/nsxt_client.rb

@@ -8,7 +8,7 @@ def initialize(server, path, user, password, verify_ssl = false)
     @user = user
     @password = password
     @client = Rest.new(server, user, password, verify_ssl)
-    connected, data = @client.login
+    connected = @client.login
     return if connected
 
     raise 'NSX-T Authentication failed'

app/models/manageiq/providers/nsxt/nsxt_client/rest.rb

@@ -12,7 +12,6 @@ def login
     RestClient::Request.execute(:method => :get, :url => @login_url, :user => @user, :password => @password, :verify_ssl => @verify_ssl) do |response|
       case response.code
       when 200
-        data = JSON.parse(response.body)
         return true
       when 403
         raise MiqException::MiqInvalidCredentialsError, "Login failed due to a bad username or password."

spec/models/manageiq/providers/nsxt/network_manager/refresher_spec.rb

@@ -31,8 +31,8 @@
     end
 
     def assert_table_counts
-      expect(CloudNetwork.count).to        eq(39)
-      expect(CloudSubnet.count).to         eq(32)
+      expect(CloudNetwork.count).to        eq(32)
+      expect(CloudSubnet.count).to         eq(33)
       expect(NetworkService.count).to      eq(410)
       expect(NetworkServiceEntry.count).to eq(802)
       expect(NetworkRouter.count).to       eq(10)
@@ -77,14 +77,14 @@ def assert_specific_cloud_network
     end
 
     def assert_specific_cloud_subnet
-      cloud_subnet = ems.cloud_subnets.find_by(:ems_ref => "d5204c40-6a90-11ea-a5b5-f9815823238c-192.168.12.1/24")
+      cloud_subnet = ems.cloud_subnets.find_by(:ems_ref => "d5204c40-6a90-11ea-a5b5-f9815823238c-192.168.12.0/24")
 
       expect(cloud_subnet).to have_attributes(
-        :name         => "192.168.12.0-192.168.12.1/24",
-        :ems_ref      => "d5204c40-6a90-11ea-a5b5-f9815823238c-192.168.12.1/24",
+        :name         => "192.168.12.0-192.168.12.0/24",
+        :ems_ref      => "d5204c40-6a90-11ea-a5b5-f9815823238c-192.168.12.0/24",
         :cidr         => "192.168.12.0/24",
         :dhcp_enabled => false,
-        :gateway      => "192.168.12.1/24",
+        :gateway      => "192.168.12.1",
         :type         => "ManageIQ::Providers::Nsxt::NetworkManager::CloudSubnet",
       )
 
@@ -101,7 +101,7 @@ def assert_specific_network_router
       )
 
       expect(network_router.cloud_subnets.count).to eq(2)
-      expect(network_router.cloud_subnets.pluck(:ems_ref)).to include("d5204c40-6a90-11ea-a5b5-f9815823238c-192.168.12.1/24")
+      expect(network_router.cloud_subnets.pluck(:ems_ref)).to include("d5204c40-6a90-11ea-a5b5-f9815823238c-192.168.12.0/24")
     end
 
     def assert_specific_security_group