Merge branch '5.x' into 5.5

.gitignore

@@ -7,6 +7,7 @@
 .coverage
 .eggs
 .idea
+.vscode
 Elastic.ico
 Vagrant/centos/6/.vagrant
 Vagrant/centos/7/.vagrant

curator/_version.py

@@ -1,2 +1 @@
-__version__ = '5.5.2'
-
+__version__ = '5.5.3'

curator/actions.py

@@ -1486,15 +1486,16 @@ def get_state(self):
 
     def report_state(self):
         """
-        Log the state of the snapshot
+        Log the state of the snapshot and raise an exception if the state is
+        not ``SUCCESS``
         """
         self.get_state()
         if self.state == 'SUCCESS':
-            self.loggit.info(
-                'Snapshot {0} successfully completed.'.format(self.name))
+            self.loggit.info('Snapshot {0} successfully completed.'.format(self.name))
         else:
-            self.loggit.warn(
-                'Snapshot {0} completed with state: {0}'.format(self.state))
+            msg = 'Snapshot {0} completed with state: {0}'.format(self.state)
+            self.loggit.error(msg)
+            raise exceptions.FailedSnapshot(msg)
 
     def do_dry_run(self):
         """
@@ -1531,6 +1532,7 @@ def do_action(self):
                     repository=self.repository,
                     wait_interval=self.wait_interval, max_wait=self.max_wait
                 )
+                self.report_state()
             else:
                 self.loggit.warn(
                     '"wait_for_completion" set to {0}.'
@@ -1686,9 +1688,7 @@ def _get_expected_output(self):
                     index
                 )
             )
-            self.loggit.debug('index: {0} replacement: '
-                '{1}'.format(index, self.expected_output[-1])
-            )
+            self.loggit.debug('index: {0} replacement: {1}'.format(index, self.expected_output[-1]))
 
     def report_state(self):
         """
@@ -1708,10 +1708,9 @@ def report_state(self):
         if found_count == len(self.expected_output):
             self.loggit.info('All indices appear to have been restored.')
         else:
-            self.loggit.error(
-                'Some of the indices do not appear to have been restored. '
-                'Missing: {0}'.format(missing)
-            )
+            msg = 'Some of the indices do not appear to have been restored. Missing: {0}'.format(missing)
+            self.loggit.error(msg)
+            raise exceptions.FailedRestore(msg)
 
     def do_dry_run(self):
         """
@@ -1741,16 +1740,14 @@ def do_dry_run(self):
                 'DRY-RUN: restore: Index {0} {1}'.format(index, replacement_msg)
             )
 
-
     def do_action(self):
         """
         Restore indices with options passed.
         """
         if not self.skip_repo_fs_check:
             utils.test_repo_fs(self.client, self.repository)
         if utils.snapshot_running(self.client):
-            raise exceptions.SnapshotInProgress(
-                'Cannot restore while a snapshot is in progress.')
+            raise exceptions.SnapshotInProgress('Cannot restore while a snapshot is in progress.')
         try:
             self.loggit.info('Restoring indices "{0}" from snapshot: '
                 '{1}'.format(self.indices, self.name)
@@ -1767,6 +1764,7 @@ def do_action(self):
                     self.client, 'restore', index_list=self.expected_output,
                     wait_interval=self.wait_interval, max_wait=self.max_wait
                 )
+                self.report_state()
             else:
                 self.loggit.warn(
                     '"wait_for_completion" set to {0}. '

curator/cli.py

@@ -132,6 +132,8 @@ def run(config, action_file, dry_run=False):
         logger.debug('timeout_override = {0}'.format(timeout_override))
         ignore_empty_list = actions[idx]['options'].pop('ignore_empty_list')
         logger.debug('ignore_empty_list = {0}'.format(ignore_empty_list))
+        allow_ilm = actions[idx]['options'].pop('allow_ilm_indices')
+        logger.debug('allow_ilm_indices = {0}'.format(allow_ilm))
 
         ### Skip to next action if 'disabled'
         if action_disabled:

curator/cli_singletons/alias.py

@@ -8,14 +8,16 @@
 @click.option('--remove', callback=validate_filter_json, help='JSON array of filters selecting indices to REMOVE from alias', default=None)
 @click.option('--warn_if_no_indices', is_flag=True, help='Do not raise exception if there are no actionable indices in add/remove')
 @click.option('--extra_settings', help='JSON version of extra_settings (see documentation)', callback=json_to_dict)
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.pass_context
-def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings):
+def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_indices):
     """
     Add/Remove Indices to/from Alias
     """
     manual_options = {
         'name': name,
         'extra_settings': extra_settings,
+        'allow_ilm_indices': allow_ilm_indices,
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     ignore_empty_list = warn_if_no_indices

curator/cli_singletons/allocation.py

@@ -10,9 +10,10 @@
 @click.option('--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion', show_default=True)
 @click.option('--wait_interval', default=9, type=int, help='Seconds to wait between completion checks.', show_default=True)
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
-def allocation(ctx, key, value, allocation_type, wait_for_completion, max_wait, wait_interval, ignore_empty_list, filter_list):
+def allocation(ctx, key, value, allocation_type, wait_for_completion, max_wait, wait_interval, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Shard Routing Allocation
     """
@@ -23,6 +24,7 @@ def allocation(ctx, key, value, allocation_type, wait_for_completion, max_wait,
         'wait_for_completion': wait_for_completion,
         'max_wait': max_wait,
         'wait_interval': wait_interval,
+        'allow_ilm_indices': allow_ilm_indices,
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list)

curator/cli_singletons/close.py

@@ -5,13 +5,17 @@
 @click.command(context_settings=get_width())
 @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed')
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
-def close(ctx, delete_aliases, ignore_empty_list, filter_list):
+def close(ctx, delete_aliases, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Close Indices
     """
-    manual_options = { 'delete_aliases': delete_aliases }
+    manual_options = { 
+        'delete_aliases': delete_aliases,
+        'allow_ilm_indices': allow_ilm_indices,
+    }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list)
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])

curator/cli_singletons/delete.py

@@ -5,14 +5,15 @@
 #### Indices ####
 @click.command(context_settings=get_width())
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
-def delete_indices(ctx, ignore_empty_list, filter_list):
+def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Delete Indices
     """
     # ctx.info_name is the name of the function or name specified in @click.command decorator
-    action = cli_action(ctx.info_name, ctx.obj['config']['client'], {}, filter_list, ignore_empty_list)
+    action = cli_action(ctx.info_name, ctx.obj['config']['client'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list)
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])
 
 #### Snapshots ####
@@ -21,15 +22,17 @@ def delete_indices(ctx, ignore_empty_list, filter_list):
 @click.option('--retry_count', type=int, help='Number of times to retry (max 3)')
 @click.option('--retry_interval', type=int, help='Time in seconds between retries')
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable snapshots')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting snapshots to act on.', required=True)
 @click.pass_context
-def delete_snapshots(ctx, repository, retry_count, retry_interval, ignore_empty_list, filter_list):
+def delete_snapshots(ctx, repository, retry_count, retry_interval, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Delete Snapshots
     """
     manual_options = {
         'retry_count': retry_count, 
-        'retry_interval': retry_interval
+        'retry_interval': retry_interval,
+        'allow_ilm_indices': allow_ilm_indices,
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list, repository=repository)

curator/cli_singletons/forcemerge.py

@@ -6,15 +6,17 @@
 @click.option('--max_num_segments', type=int, required=True, help='Maximum number of segments per shard (minimum of 1)')
 @click.option('--delay', type=float, help='Time in seconds to delay between operations. Default 0, maximum 3600')
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
-def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, filter_list):
+def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     forceMerge Indices (reduce segment count)
     """
     manual_options = { 
         'max_num_segments': max_num_segments,
         'delay': delay,
+        'allow_ilm_indices': allow_ilm_indices,
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list)

curator/cli_singletons/object_class.py

@@ -50,6 +50,13 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l
             except KeyError:
                 self.logger.critical('Action must be one of {0}'.format(list(CLASS_MAP.keys())))
             self.check_options(option_dict)
+        else:
+            self.options = option_dict
+        # Extract allow_ilm_indices so it can be handled separately.
+        if 'allow_ilm_indices' in self.options:
+            self.allow_ilm = self.options.pop('allow_ilm_indices')
+        else:
+            self.allow_ilm = False
         if action == 'alias':
             self.alias = {
                 'name': option_dict['name'],
@@ -61,6 +68,8 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l
                     self.alias[k] = {}
                     self.check_filters(kwargs[k], loc='alias singleton', key=k)
                     self.alias[k]['filters'] = self.filters
+                    if self.allow_ilm:
+                        self.alias[k]['filters'].append({'filtertype':'ilm'})
         elif action in [ 'cluster_routing', 'create_index', 'rollover']:
             self.action_kwargs = {}
             # No filters for these actions
@@ -119,6 +128,8 @@ def check_filters(self, filter_dict, loc='singleton', key='filters'):
 
     def do_filters(self):
         self.logger.debug('Running filters and testing for empty list object')
+        if self.allow_ilm:
+            self.filters.append({'filtertype':'ilm','exclude':True})
         try:
             self.list_object.iterate_filters({'filters':self.filters})
             self.list_object.empty_list_check()
@@ -132,7 +143,7 @@ def do_filters(self):
                 sys.exit(1)
 
     def get_list_object(self):
-        if self.action in snapshot_actions():
+        if self.action in snapshot_actions() or self.action == 'show_snapshots':
             self.list_object = SnapshotList(self.client, repository=self.repository)
         else:
             self.list_object = IndexList(self.client)
@@ -164,6 +175,7 @@ def do_singleton_action(self, dry_run=False):
             else:
                 self.get_list_object()
                 self.do_filters()
+                self.logger.debug('OPTIONS = {0}'.format(self.options))
                 action_obj = self.action_class(self.list_object, **self.options)
             try:
                 if dry_run:

curator/cli_singletons/open_indices.py

@@ -4,12 +4,13 @@
 
 @click.command(name='open', context_settings=get_width())
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
-def open_indices(ctx, ignore_empty_list, filter_list):
+def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Open Indices
     """
     # ctx.info_name is the name of the function or name specified in @click.command decorator
-    action = cli_action(ctx.info_name, ctx.obj['config']['client'], {}, filter_list, ignore_empty_list)
+    action = cli_action(ctx.info_name, ctx.obj['config']['client'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list)
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])

curator/cli_singletons/replicas.py

@@ -6,15 +6,17 @@
 @click.option('--count', type=int, required=True, help='Number of replicas (max 10)')
 @click.option('--wait_for_completion/--no-wait_for_completion', default=False, help='Wait for replication to complete', show_default=True)
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
-def replicas(ctx, count, wait_for_completion, ignore_empty_list, filter_list):
+def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Change Replica Count
     """
     manual_options = { 
         'count': count,
         'wait_for_completion': wait_for_completion,
+        'allow_ilm_indices': allow_ilm_indices,
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list)

curator/cli_singletons/restore.py

@@ -18,11 +18,12 @@
 @click.option('--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion')
 @click.option('--skip_repo_fs_check', is_flag=True, show_default=True, help='Skip repository filesystem access validation.')
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
+@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting snapshots to act on.', required=True)
 @click.pass_context
 def restore(ctx, repository, name, index, rename_pattern, rename_replacement, extra_settings,
     include_aliases, ignore_unavailable, include_global_state, partial, wait_for_completion, 
-    wait_interval, max_wait, skip_repo_fs_check, ignore_empty_list, filter_list):
+    wait_interval, max_wait, skip_repo_fs_check, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Restore Indices
     """
@@ -40,6 +41,7 @@ def restore(ctx, repository, name, index, rename_pattern, rename_replacement, ex
         'wait_for_completion': wait_for_completion,
         'max_wait': max_wait,
         'wait_interval': wait_interval,
+        'allow_ilm_indices': allow_ilm_indices,
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list, repository=repository)