Release prep for 8.0.14 (#1710)

* WIP branch for debugging cold2frozen weirdness

* Release prep for 8.0.14

**Announcement**

  * A long awaited feature has been added, stealthily. It's fully in the documentation, but I do
    not yet plan to make a big announcement about it. In actions that search through indices, you
    can now specify a ``search_pattern`` to limit the number of indices that will be filtered. If
    no search pattern is specified, the behavior will be the same as it ever was: it will search
    through ``_all`` indices. The actions that support this option are: allocation, close,
    cold2frozen, delete_indices, forcemerge, index_settings, open, replicas, shrink, and snapshot.

**Bugfix**

  * A mixup with naming conventions from the PII redacter tool got in the way of the cold2frozen
    action completing properly.

**Changes**

  * Version bump: ``es_client==8.13.0``
      * With the version bump to ``es_client`` comes a necessary change to calls to create a
        repository. In elastic/elasticsearch-specification#2255 it became
        clear that using ``type`` and ``settings`` as it has been was insufficient for repository
        settings, so we go back to using a request ``body`` as in older times. This change affects
        ``esrepomgr`` in one place, and otherwise only in snapshot/restore testing.
  * Added the curator.helpers.getters.meta_getter to reduce near duplicate functions.
  * Changed curator.helpers.getters.get_indices to use the _cat API to pull indices. The primary
    driver for this is that it avoids pulling in the full mapping and index settings when all we
    really need to return is a list of index names. This should help keep memory from ballooning
    quite as much. The function also now allows for a search_pattern kwarg to search only for
    indices matching a pattern. This will also potentially make the initial index return list much
    smaller, and the list of indices needing to be filtered that much smaller.
  * Tests were added to ensure that the changes for ``get_indices`` work everywhere.
  * Tests were added to ensure that the new ``search_pattern`` did not break anything, and does
    behave as expected.

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.13'
+__version__ = '8.0.14'

curator/actions/alias.py

@@ -47,7 +47,7 @@ def add(self, ilo, warn_if_no_indices=False):
         :type ilo: :py:class:`~.curator.indexlist.IndexList`
         """
         verify_index_list(ilo)
-        self.loggit.debug('ADD -> ILO = %s', ilo)
+        self.loggit.debug('ADD -> ILO = %s', ilo.indices)
         if not self.client:
             self.client = ilo.client
         self.name = parse_datemath(self.client, self.name)
@@ -80,7 +80,7 @@ def remove(self, ilo, warn_if_no_indices=False):
         :type ilo: :py:class:`~.curator.indexlist.IndexList`
         """
         verify_index_list(ilo)
-        self.loggit.debug('REMOVE -> ILO = %s', ilo)
+        self.loggit.debug('REMOVE -> ILO = %s', ilo.indices)
         if not self.client:
             self.client = ilo.client
         self.name = parse_datemath(self.client, self.name)

curator/actions/cold2frozen.py

@@ -1,6 +1,6 @@
 """Snapshot and Restore action classes"""
 import logging
-from curator.helpers.getters import get_alias_actions, get_frozen_prefix, get_tier_preference
+from curator.helpers.getters import get_alias_actions, get_tier_preference, meta_getter
 from curator.helpers.testers import has_lifecycle_name, is_idx_partial, verify_index_list
 from curator.helpers.utils import report_failure
 from curator.exceptions import CuratorException, FailedExecution, SearchableSnapshotException
@@ -76,7 +76,8 @@ def action_generator(self):
         :rtype: dict
         """
         for idx in self.index_list.indices:
-            idx_settings = self.client.indices.get(index=idx)[idx]['settings']['index']
+            idx_settings = meta_getter(self.client, idx, get='settings')
+            self.loggit.debug('Index %s has settings: %s', idx, idx_settings)
             if has_lifecycle_name(idx_settings):
                 self.loggit.critical(
                     'Index %s is associated with an ILM policy and this action will never work on '
@@ -85,13 +86,16 @@ def action_generator(self):
             if is_idx_partial(idx_settings):
                 self.loggit.critical('Index %s is already in the frozen tier', idx)
                 raise SearchableSnapshotException('Index is already in frozen tier')
+
             snap = idx_settings['store']['snapshot']['snapshot_name']
             snap_idx = idx_settings['store']['snapshot']['index_name']
             repo = idx_settings['store']['snapshot']['repository_name']
-            aliases = self.client.indices.get(index=idx)[idx]['aliases']
+            msg = f'Index {idx} Snapshot name: {snap}, Snapshot index: {snap_idx}, repo: {repo}'
+            self.loggit.debug(msg)
+
+            aliases = meta_getter(self.client, idx, get='alias')
 
-            prefix = get_frozen_prefix(snap_idx, idx)
-            renamed = f'{prefix}{snap_idx}'
+            renamed = f'partial-{idx}'
 
             if not self.index_settings:
                 self.index_settings = {
@@ -187,7 +191,7 @@ def cleanup(self, current_idx, newidx):
     def do_action(self):
         """
         Do the actions outlined:
-
+        Extract values from generated kwargs
         Mount
         Verify
         Update Aliases

curator/cli.py

@@ -86,14 +86,22 @@ def process_action(client, action_def, dry_run=False):
     logger = logging.getLogger(__name__)
     logger.debug('Configuration dictionary: %s', action_def.action_dict)
     mykwargs = {}
+    search_pattern = '_all'
 
+    logger.critical('INITIAL Action kwargs: %s', mykwargs)
     # Add some settings to mykwargs...
     if action_def.action == 'delete_indices':
         mykwargs['master_timeout'] = 30
 
     ### Update the defaults with whatever came with opts, minus any Nones
     mykwargs.update(prune_nones(action_def.options))
+
+    # Pop out the search_pattern option, if present.
+    if 'search_pattern' in mykwargs:
+        search_pattern = mykwargs.pop('search_pattern')
+
     logger.debug('Action kwargs: %s', mykwargs)
+    logger.critical('Post search_pattern Action kwargs: %s', mykwargs)
 
     ### Set up the action ###
     logger.debug('Running "%s"', action_def.action.upper())
@@ -120,8 +128,9 @@ def process_action(client, action_def, dry_run=False):
             mykwargs.pop('repository') # We don't need to send this value to the action
             action_def.instantiate('list_obj', client, repository=action_def.options['repository'])
         else:
-            action_def.instantiate('list_obj', client)
+            action_def.instantiate('list_obj', client, search_pattern=search_pattern)
         action_def.list_obj.iterate_filters({'filters': action_def.filters})
+        logger.critical('Pre Instantiation Action kwargs: %s', mykwargs)
         action_def.instantiate('action_cls', action_def.list_obj, **mykwargs)
     ### Do the action
     if dry_run:

curator/cli_singletons/allocation.py

@@ -4,6 +4,7 @@
 from curator.cli_singletons.utils import validate_filter_json
 
 @click.command()
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option('--key', type=str, required=True, help='Node identification tag')
 @click.option('--value', type=str, default=None, help='Value associated with --key')
 @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude']))
@@ -47,6 +48,7 @@
 @click.pass_context
 def allocation(
         ctx,
+        search_pattern,
         key,
         value,
         allocation_type,
@@ -61,6 +63,7 @@ def allocation(
     Shard Routing Allocation
     """
     manual_options = {
+        'search_pattern': search_pattern,
         'key': key,
         'value': value,
         'allocation_type': allocation_type,

curator/cli_singletons/close.py

@@ -4,6 +4,7 @@
 from curator.cli_singletons.utils import validate_filter_json
 
 @click.command()
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed')
 @click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed')
 @click.option(
@@ -24,11 +25,14 @@
     required=True
 )
 @click.pass_context
-def close(ctx, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, filter_list):
+def close(
+        ctx, search_pattern, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices,
+        filter_list):
     """
     Close Indices
     """
     manual_options = {
+        'search_pattern': search_pattern,
         'skip_flush': skip_flush,
         'delete_aliases': delete_aliases,
         'allow_ilm_indices': allow_ilm_indices,

curator/cli_singletons/delete.py

@@ -5,6 +5,7 @@
 
 #### Indices ####
 @click.command()
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option(
     '--ignore_empty_list',
     is_flag=True,
@@ -23,15 +24,15 @@
     required=True
 )
 @click.pass_context
-def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
+def delete_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Delete Indices
     """
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
         'delete_indices',
         ctx.obj['configdict'],
-        {'allow_ilm_indices':allow_ilm_indices},
+        {'search_pattern': search_pattern, 'allow_ilm_indices':allow_ilm_indices},
         filter_list,
         ignore_empty_list
     )

curator/cli_singletons/forcemerge.py

@@ -4,6 +4,7 @@
 from curator.cli_singletons.utils import validate_filter_json
 
 @click.command()
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option(
     '--max_num_segments',
     type=int,
@@ -32,11 +33,14 @@
     help='JSON array of filters selecting indices to act on.',
     required=True)
 @click.pass_context
-def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, filter_list):
+def forcemerge(
+        ctx, search_pattern, max_num_segments, delay, ignore_empty_list, allow_ilm_indices,
+        filter_list):
     """
     forceMerge Indices (reduce segment count)
     """
     manual_options = {
+        'search_pattern': search_pattern,
         'max_num_segments': max_num_segments,
         'delay': delay,
         'allow_ilm_indices': allow_ilm_indices,

curator/cli_singletons/object_class.py

@@ -76,6 +76,12 @@ def __init__(self, action, configdict, option_dict, filter_list, ignore_empty_li
             self.check_options(option_dict)
         else:
             self.options = option_dict
+
+        # Pop out search_pattern if it's there
+        self.search_pattern = '_all'
+        if 'search_pattern' in self.options:
+            self.search_pattern = self.options.pop('search_pattern')
+
         # Extract allow_ilm_indices so it can be handled separately.
         if 'allow_ilm_indices' in self.options:
             self.allow_ilm = self.options.pop('allow_ilm_indices')

curator/cli_singletons/open_indices.py

@@ -4,6 +4,7 @@
 from curator.cli_singletons.utils import validate_filter_json
 
 @click.command(name='open')
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option(
     '--ignore_empty_list',
     is_flag=True,
@@ -22,15 +23,15 @@
     required=True
 )
 @click.pass_context
-def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
+def open_indices(ctx, search_pattern, ignore_empty_list, allow_ilm_indices, filter_list):
     """
     Open Indices
     """
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
         ctx.info_name,
         ctx.obj['configdict'],
-        {'allow_ilm_indices':allow_ilm_indices},
+        {'search_pattern': search_pattern, 'allow_ilm_indices':allow_ilm_indices},
         filter_list,
         ignore_empty_list
     )

curator/cli_singletons/replicas.py

@@ -4,6 +4,7 @@
 from curator.cli_singletons.utils import validate_filter_json
 
 @click.command()
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option('--count', type=int, required=True, help='Number of replicas (max 10)')
 @click.option(
     '--wait_for_completion/--no-wait_for_completion',
@@ -29,11 +30,14 @@
     required=True
 )
 @click.pass_context
-def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, filter_list):
+def replicas(
+        ctx, search_pattern, count, wait_for_completion, ignore_empty_list, allow_ilm_indices,
+        filter_list):
     """
     Change Replica Count
     """
     manual_options = {
+        'search_pattern': search_pattern,
         'count': count,
         'wait_for_completion': wait_for_completion,
         'allow_ilm_indices': allow_ilm_indices,

curator/cli_singletons/show.py

@@ -11,22 +11,25 @@
 
 # pylint: disable=line-too-long
 @click.command(epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots'))
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option('--verbose', help='Show verbose output.', is_flag=True, show_default=True)
 @click.option('--header', help='Print header if --verbose', is_flag=True, show_default=True)
 @click.option('--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True)
 @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices')
 @click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True)
 @click.option('--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.')
 @click.pass_context
-def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indices, filter_list):
+def show_indices(
+        ctx, search_pattern, verbose, header, epoch, ignore_empty_list, allow_ilm_indices,
+        filter_list):
     """
     Show Indices
     """
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
         'show_indices',
         ctx.obj['configdict'],
-        {'allow_ilm_indices': allow_ilm_indices},
+        {'search_pattern': search_pattern, 'allow_ilm_indices': allow_ilm_indices},
         filter_list,
         ignore_empty_list
     )

curator/cli_singletons/shrink.py

@@ -5,6 +5,7 @@
 
 # pylint: disable=line-too-long
 @click.command()
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option('--shrink_node', default='DETERMINISTIC', type=str, help='Named node, or DETERMINISTIC', show_default=True)
 @click.option('--node_filters', help='JSON version of node_filters (see documentation)', callback=json_to_dict)
 @click.option('--number_of_shards', default=1, type=int, help='Shrink to this many shards per index')
@@ -25,15 +26,16 @@
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
 def shrink(
-        ctx, shrink_node, node_filters, number_of_shards, number_of_replicas, shrink_prefix,
-        shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings,
+        ctx, search_pattern, shrink_node, node_filters, number_of_shards, number_of_replicas,
+        shrink_prefix, shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings,
         wait_for_active_shards, wait_for_rebalance, wait_for_completion, wait_interval, max_wait,
         ignore_empty_list, allow_ilm_indices, filter_list
     ):
     """
     Shrink Indices to --number_of_shards
     """
     manual_options = {
+        'search_pattern': search_pattern,
         'shrink_node': shrink_node,
         'node_filters': node_filters,
         'number_of_shards': number_of_shards,

curator/cli_singletons/snapshot.py

@@ -5,6 +5,7 @@
 
 # pylint: disable=line-too-long
 @click.command()
+@click.option('--search_pattern', type=str, default='_all', help='Elasticsearch Index Search Pattern')
 @click.option('--repository', type=str, required=True, help='Snapshot repository')
 @click.option('--name', type=str, help='Snapshot name', show_default=True, default='curator-%Y%m%d%H%M%S')
 @click.option('--ignore_unavailable', is_flag=True, show_default=True, help='Ignore unavailable shards/indices.')
@@ -19,14 +20,15 @@
 @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True)
 @click.pass_context
 def snapshot(
-        ctx, repository, name, ignore_unavailable, include_global_state, partial,
+        ctx, search_pattern, repository, name, ignore_unavailable, include_global_state, partial,
         skip_repo_fs_check, wait_for_completion, wait_interval, max_wait, ignore_empty_list,
         allow_ilm_indices, filter_list
     ):
     """
     Snapshot Indices
     """
     manual_options = {
+        'search_pattern': search_pattern,
         'name': name,
         'repository': repository,
         'ignore_unavailable': ignore_unavailable,

curator/defaults/option_defaults.py

@@ -380,6 +380,12 @@ def cluster_routing_value():
     """
     return {Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas')}
 
+def search_pattern():
+    """
+    :returns: ``{Optional('search_pattern', default='_all'): Any(*string_types)}``
+    """
+    return {Optional('search_pattern', default='_all'): Any(*string_types)}
+
 def shrink_node():
     """
     :returns: ``{Required('shrink_node'): Any(*string_types)}``