diff --git a/curator/_version.py b/curator/_version.py index 434b2f25..9d236b0a 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1,2 @@ """Curator Version""" -__version__ = '8.0.10' +__version__ = '8.0.11' diff --git a/curator/actions/cold2frozen.py b/curator/actions/cold2frozen.py index e4512e5f..20daaf90 100644 --- a/curator/actions/cold2frozen.py +++ b/curator/actions/cold2frozen.py @@ -127,56 +127,85 @@ def do_dry_run(self): ) self.loggit.info(msg) - - def do_action(self): + def mount_index(self, newidx, kwargs): """ Call :py:meth:`~.elasticsearch.client.SearchableSnapshotsClient.mount` to mount the indices in :py:attr:`ilo` in the Frozen tier. + """ + try: + self.loggit.debug('Mounting new index %s in frozen tier...', newidx) + self.client.searchable_snapshots.mount(**kwargs) + # pylint: disable=broad-except + except Exception as err: + report_failure(err) - Verify index looks good - + def verify_mount(self, newidx): + """ + Verify that newidx is a mounted index + """ + self.loggit.debug('Verifying new index %s is mounted properly...', newidx) + idx_settings = self.client.indices.get(index=newidx)[newidx] + if is_idx_partial(idx_settings['settings']['index']): + self.loggit.info('Index %s is mounted for frozen tier', newidx) + else: + report_failure(SearchableSnapshotException( + f'Index {newidx} not a mounted searchable snapshot')) + + def update_aliases(self, current_idx, newidx, aliases): + """ Call :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` to update each new frozen index with the aliases from the old cold-tier index. Verify aliases look good. - + """ + alias_names = aliases.keys() + if not alias_names: + self.loggit.warning('No aliases associated with index %s', current_idx) + else: + self.loggit.debug('Transferring aliases to new index %s', newidx) + self.client.indices.update_aliases( + actions=get_alias_actions(current_idx, newidx, aliases)) + verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys() + if alias_names != verify: + self.loggit.error( + 'Alias names do not match! %s does not match: %s', alias_names, verify) + report_failure(FailedExecution('Aliases failed to transfer to new index')) + + def cleanup(self, current_idx, newidx): + """ Call :py:meth:`~.elasticsearch.client.IndicesClient.delete` to delete the cold tier index. """ + self.loggit.debug('Deleting old index: %s', current_idx) try: - for kwargs in self.action_generator(): - aliases = kwargs.pop('aliases') - current_idx = kwargs.pop('current_idx') - newidx = kwargs['renamed_index'] - # Actually do the mount - self.loggit.debug('Mounting new index %s in frozen tier...', newidx) - self.client.searchable_snapshots.mount(**kwargs) - # Verify it's mounted as a partial now: - self.loggit.debug('Verifying new index %s is mounted properly...', newidx) - idx_settings = self.client.indices.get(index=newidx)[newidx] - if is_idx_partial(idx_settings['settings']['index']): - self.loggit.info('Index %s is mounted for frozen tier', newidx) - else: - raise SearchableSnapshotException( - f'Index {newidx} not a mounted searchable snapshot') - # Update Aliases - alias_names = aliases.keys() - if not alias_names: - self.loggit.warning('No aliases associated with index %s', current_idx) - else: - self.loggit.debug('Transferring aliases to new index %s', newidx) - self.client.indices.update_aliases( - actions=get_alias_actions(current_idx, newidx, aliases)) - verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys() - if alias_names != verify: - self.loggit.error( - 'Alias names do not match! %s does not match: %s', alias_names, verify) - raise FailedExecution('Aliases failed to transfer to new index') - # Clean up old index - self.loggit.debug('Deleting old index: %s', current_idx) - self.client.indices.delete(index=current_idx) - self.loggit.info( - 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx) - + self.client.indices.delete(index=current_idx) # pylint: disable=broad-except except Exception as err: report_failure(err) + self.loggit.info( + 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx) + + def do_action(self): + """ + Do the actions outlined: + + Mount + Verify + Update Aliases + Cleanup + """ + for kwargs in self.action_generator(): + aliases = kwargs.pop('aliases') + current_idx = kwargs.pop('current_idx') + newidx = kwargs['renamed_index'] + + # Mount the index + self.mount_index(newidx, kwargs) + + # Verify it's mounted as a partial now: + self.verify_mount(newidx) + + # Update Aliases + self.update_aliases(current_idx, newidx, aliases) + + # Clean up old index + self.cleanup(current_idx, newidx) diff --git a/curator/actions/create_index.py b/curator/actions/create_index.py index b6ff6240..1ef0563e 100644 --- a/curator/actions/create_index.py +++ b/curator/actions/create_index.py @@ -74,10 +74,14 @@ def do_action(self): # Most likely error is a 400, `resource_already_exists_exception` except RequestError as err: match_list = ["index_already_exists_exception", "resource_already_exists_exception"] - if err.error in match_list and self.ignore_existing: - self.loggit.warning('Index %s already exists.', self.name) + if err.error in match_list: + if self.ignore_existing: + self.loggit.warning('Index %s already exists.', self.name) + else: + raise FailedExecution(f'Index {self.name} already exists.') from err else: - raise FailedExecution(f'Index {self.name} already exists.') from err + msg = f'Unable to create index "{self.name}". Error: {err.error}' + raise FailedExecution(msg) from err # pylint: disable=broad-except except Exception as err: report_failure(err) diff --git a/curator/cli.py b/curator/cli.py index 95ab05dd..b7c7f244 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -2,8 +2,9 @@ import sys import logging import click -from es_client.defaults import LOGGING_SETTINGS -from es_client.helpers.config import cli_opts, context_settings, get_args, get_client, get_config +from es_client.defaults import OPTION_DEFAULTS +from es_client.helpers.config import ( + cli_opts, context_settings, generate_configdict, get_client, get_config, options_from_dict) from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper, prune_nones from curator.exceptions import ClientException @@ -129,24 +130,17 @@ def process_action(client, action_def, dry_run=False): logger.debug('Doing the action here.') action_def.action_cls.do_action() -def run(client_args, other_args, action_file, dry_run=False): +def run(ctx: click.Context) -> None: """ - Called by :py:func:`cli` to execute what was collected at the command-line + :param ctx: The Click command context - :param client_args: The ClientArgs arguments object - :param other_args: The OtherArgs arguments object - :param action_file: The action configuration file - :param dry_run: Do not perform any changes + :type ctx: :py:class:`Context ` - :type client_args: :py:class:`~.es_client.ClientArgs` - :type other_args: :py:class:`~.es_client.OtherArgs` - :type action_file: str - :type dry_run: bool + Called by :py:func:`cli` to execute what was collected at the command-line """ logger = logging.getLogger(__name__) - - logger.debug('action_file: %s', action_file) - all_actions = ActionsFile(action_file) + logger.debug('action_file: %s', ctx.params['action_file']) + all_actions = ActionsFile(ctx.params['action_file']) for idx in sorted(list(all_actions.actions.keys())): action_def = all_actions.actions[idx] ### Skip to next action if 'disabled' @@ -160,7 +154,7 @@ def run(client_args, other_args, action_file, dry_run=False): # Override the timeout, if specified, otherwise use the default. if action_def.timeout_override: - client_args.request_timeout = action_def.timeout_override + ctx.obj['client_args'].request_timeout = action_def.timeout_override # Create a client object for each action... logger.info('Creating client object and testing connection') @@ -168,8 +162,8 @@ def run(client_args, other_args, action_file, dry_run=False): try: client = get_client(configdict={ 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) + 'client': prune_nones(ctx.obj['client_args'].asdict()), + 'other_settings': prune_nones(ctx.obj['other_args'].asdict()) } }) except ClientException as exc: @@ -187,7 +181,7 @@ def run(client_args, other_args, action_file, dry_run=False): msg = f'Trying Action ID: {idx}, "{action_def.action}": {action_def.description}' try: logger.info(msg) - process_action(client, action_def, dry_run=dry_run) + process_action(client, action_def, dry_run=ctx.params['dry_run']) # pylint: disable=broad-except except Exception as err: exception_handler(action_def, err) @@ -196,31 +190,8 @@ def run(client_args, other_args, action_file, dry_run=False): # pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals, line-too-long @click.command(context_settings=context_settings(), epilog=footer(__version__, tail='command-line.html')) -@click_opt_wrap(*cli_opts('config')) -@click_opt_wrap(*cli_opts('hosts')) -@click_opt_wrap(*cli_opts('cloud_id')) -@click_opt_wrap(*cli_opts('api_token')) -@click_opt_wrap(*cli_opts('id')) -@click_opt_wrap(*cli_opts('api_key')) -@click_opt_wrap(*cli_opts('username')) -@click_opt_wrap(*cli_opts('password')) -@click_opt_wrap(*cli_opts('bearer_auth')) -@click_opt_wrap(*cli_opts('opaque_id')) -@click_opt_wrap(*cli_opts('request_timeout')) -@click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('ca_certs')) -@click_opt_wrap(*cli_opts('client_cert')) -@click_opt_wrap(*cli_opts('client_key')) -@click_opt_wrap(*cli_opts('ssl_assert_hostname')) -@click_opt_wrap(*cli_opts('ssl_assert_fingerprint')) -@click_opt_wrap(*cli_opts('ssl_version')) -@click_opt_wrap(*cli_opts('master-only', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF)) +@options_from_dict(OPTION_DEFAULTS) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) -@click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS)) @click.argument('action_file', type=click.Path(exists=True), nargs=1) @click.version_option(__version__, '-v', '--version', prog_name="curator") @click.pass_context @@ -228,7 +199,7 @@ def cli( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, - dry_run, loglevel, logfile, logformat, action_file + loglevel, logfile, logformat, blacklist, dry_run, action_file ): """ Curator for Elasticsearch indices @@ -244,8 +215,8 @@ def cli( curator_cli -h """ ctx.obj = {} - ctx.obj['dry_run'] = dry_run - cfg = get_config(ctx.params, default_config_file()) - configure_logging(cfg, ctx.params) - client_args, other_args = get_args(ctx.params, cfg) - run(client_args, other_args, action_file, dry_run) + ctx.obj['default_config'] = default_config_file() + get_config(ctx) + configure_logging(ctx) + generate_configdict(ctx) + run(ctx) diff --git a/curator/cli_singletons/alias.py b/curator/cli_singletons/alias.py index f6e77046..803ab99c 100644 --- a/curator/cli_singletons/alias.py +++ b/curator/cli_singletons/alias.py @@ -1,11 +1,10 @@ """Alias Singleton""" import logging import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--name', type=str, help='Alias name', required=True) @click.option( '--add', @@ -51,7 +50,7 @@ def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_ ignore_empty_list = warn_if_no_indices action = CLIAction( ctx.info_name, - ctx.obj['config'], + ctx.obj['configdict'], manual_options, [], # filter_list is empty in our case ignore_empty_list, diff --git a/curator/cli_singletons/allocation.py b/curator/cli_singletons/allocation.py index 1ede28bc..260d8a65 100644 --- a/curator/cli_singletons/allocation.py +++ b/curator/cli_singletons/allocation.py @@ -1,10 +1,9 @@ """Allocation Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--key', type=str, required=True, help='Node identification tag') @click.option('--value', type=str, default=None, help='Value associated with --key') @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude'])) @@ -72,5 +71,5 @@ def allocation( } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/close.py b/curator/cli_singletons/close.py index 8039b50c..c6d11d3f 100644 --- a/curator/cli_singletons/close.py +++ b/curator/cli_singletons/close.py @@ -1,10 +1,9 @@ """Close Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed') @click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed') @click.option( @@ -36,5 +35,5 @@ def close(ctx, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/delete.py b/curator/cli_singletons/delete.py index 57deedd9..e43546d3 100644 --- a/curator/cli_singletons/delete.py +++ b/curator/cli_singletons/delete.py @@ -1,11 +1,10 @@ """Delete Index and Delete Snapshot Singletons""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json #### Indices #### -@click.command(context_settings=context_settings()) +@click.command() @click.option( '--ignore_empty_list', is_flag=True, @@ -31,7 +30,7 @@ def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'delete_indices', - ctx.obj['config'], + ctx.obj['configdict'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list @@ -39,7 +38,7 @@ def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): action.do_singleton_action(dry_run=ctx.obj['dry_run']) #### Snapshots #### -@click.command(context_settings=context_settings()) +@click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository name') @click.option('--retry_count', type=int, help='Number of times to retry (max 3)') @click.option('--retry_interval', type=int, help='Time in seconds between retries') @@ -76,7 +75,7 @@ def delete_snapshots( # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'delete_snapshots', - ctx.obj['config'], + ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, diff --git a/curator/cli_singletons/forcemerge.py b/curator/cli_singletons/forcemerge.py index 95ff1211..ed6cb5bd 100644 --- a/curator/cli_singletons/forcemerge.py +++ b/curator/cli_singletons/forcemerge.py @@ -1,10 +1,9 @@ """ForceMerge Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option( '--max_num_segments', type=int, @@ -44,5 +43,5 @@ def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indice } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/object_class.py b/curator/cli_singletons/object_class.py index 622fd7a2..900282e3 100644 --- a/curator/cli_singletons/object_class.py +++ b/curator/cli_singletons/object_class.py @@ -45,10 +45,10 @@ class CLIAction(): """ Unified class for all CLI singleton actions """ - def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_list, **kwargs): + def __init__(self, action, configdict, option_dict, filter_list, ignore_empty_list, **kwargs): """Class setup :param action: The action name. - :param client_args: ``dict`` containing everything needed for + :param configdict: ``dict`` containing everything needed for :py:class:`~.es_client.builder.Builder` to build an :py:class:`~.elasticsearch.Elasticsearch` client object. :param option_dict: Options for ``action``. @@ -57,7 +57,7 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l :param kwargs: Other keyword args to pass to ``action``. :type action: str - :type client_args: dict + :type configdict: dict :type option_dict: dict :type filter_list: list :type ignore_empty_list: bool @@ -102,9 +102,8 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l self.logger.debug('rollover option_dict = %s', option_dict) else: self.check_filters(filter_list) - try: - builder = Builder(configdict=client_args) + builder = Builder(configdict=configdict) builder.connect() # pylint: disable=broad-except except Exception as exc: @@ -212,13 +211,10 @@ def do_singleton_action(self, dry_run=False): self.do_filters() self.logger.debug('OPTIONS = %s', self.options) action_obj = self.action_class(self.list_object, **self.options) - try: - if dry_run: - action_obj.do_dry_run() - else: - action_obj.do_action() - except Exception as exc: - raise Exception from exc # pass it on? + if dry_run: + action_obj.do_dry_run() + else: + action_obj.do_action() # pylint: disable=broad-except except Exception as exc: self.logger.critical( diff --git a/curator/cli_singletons/open_indices.py b/curator/cli_singletons/open_indices.py index 4ba94750..61faa31d 100644 --- a/curator/cli_singletons/open_indices.py +++ b/curator/cli_singletons/open_indices.py @@ -1,10 +1,9 @@ """Open (closed) Index Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(name='open', context_settings=context_settings()) +@click.command(name='open') @click.option( '--ignore_empty_list', is_flag=True, @@ -30,7 +29,7 @@ def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( ctx.info_name, - ctx.obj['config'], + ctx.obj['configdict'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list diff --git a/curator/cli_singletons/replicas.py b/curator/cli_singletons/replicas.py index 9ae792d2..70d039f0 100644 --- a/curator/cli_singletons/replicas.py +++ b/curator/cli_singletons/replicas.py @@ -1,10 +1,9 @@ """Change Replica Count Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--count', type=int, required=True, help='Number of replicas (max 10)') @click.option( '--wait_for_completion/--no-wait_for_completion', @@ -41,5 +40,5 @@ def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indic } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/restore.py b/curator/cli_singletons/restore.py index 8ff4921b..313fb5db 100644 --- a/curator/cli_singletons/restore.py +++ b/curator/cli_singletons/restore.py @@ -1,11 +1,10 @@ """Snapshot Restore Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository') @click.option('--name', type=str, help='Snapshot name', required=False, default=None) @click.option('--index', multiple=True, help='Index name to restore. (Can invoke repeatedly for multiple indices)') @@ -53,7 +52,7 @@ def restore( # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( ctx.info_name, - ctx.obj['config'], + ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, diff --git a/curator/cli_singletons/rollover.py b/curator/cli_singletons/rollover.py index dd1cc94e..7c6539b6 100644 --- a/curator/cli_singletons/rollover.py +++ b/curator/cli_singletons/rollover.py @@ -1,12 +1,11 @@ """Index Rollover Singleton""" import click -from es_client.helpers.config import context_settings from es_client.helpers.utils import prune_nones from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--name', type=str, help='Alias name', required=True) @click.option('--max_age', type=str, help='max_age condition value (see documentation)') @click.option('--max_docs', type=str, help='max_docs condition value (see documentation)') @@ -35,7 +34,7 @@ def rollover( } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, [], True, + ctx.info_name, ctx.obj['configdict'], manual_options, [], True, extra_settings=extra_settings, new_index=new_index, wait_for_active_shards=wait_for_active_shards diff --git a/curator/cli_singletons/show.py b/curator/cli_singletons/show.py index 51b5e72a..3751b0f7 100644 --- a/curator/cli_singletons/show.py +++ b/curator/cli_singletons/show.py @@ -1,7 +1,6 @@ """Show Index/Snapshot Singletons""" from datetime import datetime import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json from curator.helpers.getters import byte_size @@ -11,7 +10,7 @@ #### Indices #### # pylint: disable=line-too-long -@click.command(context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) +@click.command(epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) @click.option('--verbose', help='Show verbose output.', is_flag=True, show_default=True) @click.option('--header', help='Print header if --verbose', is_flag=True, show_default=True) @click.option('--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True) @@ -26,7 +25,7 @@ def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indic # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'show_indices', - ctx.obj['config'], + ctx.obj['configdict'], {'allow_ilm_indices': allow_ilm_indices}, filter_list, ignore_empty_list @@ -83,7 +82,7 @@ def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indic #### Snapshots #### # pylint: disable=line-too-long -@click.command(context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) +@click.command(epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) @click.option('--repository', type=str, required=True, help='Snapshot repository name') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable snapshots') @click.option('--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.') @@ -95,7 +94,7 @@ def show_snapshots(ctx, repository, ignore_empty_list, filter_list): # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'show_snapshots', - ctx.obj['config'], + ctx.obj['configdict'], {}, filter_list, ignore_empty_list, diff --git a/curator/cli_singletons/shrink.py b/curator/cli_singletons/shrink.py index 0b209800..79653143 100644 --- a/curator/cli_singletons/shrink.py +++ b/curator/cli_singletons/shrink.py @@ -1,11 +1,10 @@ """Shrink Index Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--shrink_node', default='DETERMINISTIC', type=str, help='Named node, or DETERMINISTIC', show_default=True) @click.option('--node_filters', help='JSON version of node_filters (see documentation)', callback=json_to_dict) @click.option('--number_of_shards', default=1, type=int, help='Shrink to this many shards per index') @@ -53,5 +52,5 @@ def shrink( 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator - action = CLIAction(ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + action = CLIAction(ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/snapshot.py b/curator/cli_singletons/snapshot.py index 7ed26b1a..ad0e69fc 100644 --- a/curator/cli_singletons/snapshot.py +++ b/curator/cli_singletons/snapshot.py @@ -1,11 +1,10 @@ """Snapshot Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository') @click.option('--name', type=str, help='Snapshot name', show_default=True, default='curator-%Y%m%d%H%M%S') @click.option('--ignore_unavailable', is_flag=True, show_default=True, help='Ignore unavailable shards/indices.') @@ -40,5 +39,5 @@ def snapshot( 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator - action = CLIAction(ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + action = CLIAction(ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/helpers/testers.py b/curator/helpers/testers.py index 9147ea36..7109b8cf 100644 --- a/curator/helpers/testers.py +++ b/curator/helpers/testers.py @@ -301,8 +301,7 @@ def verify_client_object(test): """ logger = logging.getLogger(__name__) # Ignore mock type for testing - if str(type(test)) == "" or \ - str(type(test)) == "": + if str(type(test)) == "": pass elif not isinstance(test, Elasticsearch): msg = f'Not a valid client object. Type: {type(test)} was passed' diff --git a/curator/repomgrcli.py b/curator/repomgrcli.py index bc320b44..c2c12f36 100644 --- a/curator/repomgrcli.py +++ b/curator/repomgrcli.py @@ -6,7 +6,7 @@ from elasticsearch8 import ApiError, NotFoundError from es_client.defaults import LOGGING_SETTINGS, SHOW_OPTION from es_client.builder import Builder -from es_client.helpers.config import cli_opts, context_settings, get_config, get_args +from es_client.helpers.config import cli_opts, context_settings, generate_configdict, get_config from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper, prune_nones from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer @@ -57,7 +57,7 @@ def get_client(ctx): :returns: A client connection object :rtype: :py:class:`~.elasticsearch.Elasticsearch` """ - builder = Builder(configdict=ctx.obj['esconfig']) + builder = Builder(configdict=ctx.obj['configdict']) try: builder.connect() # pylint: disable=broad-except @@ -350,21 +350,13 @@ def repo_mgr_cli( es_repo_mgr show-all-options """ ctx.obj = {} - # Ensure a passable ctx object - ctx.ensure_object(dict) ctx.obj['dry_run'] = dry_run - cfg = get_config(ctx.params, default_config_file()) - configure_logging(cfg, ctx.params) + ctx.obj['default_config'] = default_config_file() + get_config(ctx) + configure_logging(ctx) logger = logging.getLogger('curator.repomgrcli') - client_args, other_args = get_args(ctx.params, cfg) - ctx.obj['esconfig'] = { - 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) - } - } + generate_configdict(ctx) logger.debug('Exiting initial command function...') - @repo_mgr_cli.command( 'show-all-options', diff --git a/curator/singletons.py b/curator/singletons.py index 4828deae..a531f7be 100644 --- a/curator/singletons.py +++ b/curator/singletons.py @@ -1,9 +1,10 @@ """CLI module for curator_cli""" import click -from es_client.defaults import LOGGING_SETTINGS, SHOW_OPTION -from es_client.helpers.config import cli_opts, context_settings, get_config, get_args +from es_client.defaults import SHOW_EVERYTHING +from es_client.helpers.config import ( + cli_opts, context_settings, generate_configdict, get_config, options_from_dict) from es_client.helpers.logging import configure_logging -from es_client.helpers.utils import option_wrapper, prune_nones +from es_client.helpers.utils import option_wrapper from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer from curator._version import __version__ from curator.cli_singletons import ( @@ -12,44 +13,20 @@ ) from curator.cli_singletons.show import show_indices, show_snapshots -ONOFF = {'on': '', 'off': 'no-'} click_opt_wrap = option_wrapper() # pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals @click.group( context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html')) -@click_opt_wrap(*cli_opts('config')) -@click_opt_wrap(*cli_opts('hosts')) -@click_opt_wrap(*cli_opts('cloud_id')) -@click_opt_wrap(*cli_opts('api_token')) -@click_opt_wrap(*cli_opts('id')) -@click_opt_wrap(*cli_opts('api_key')) -@click_opt_wrap(*cli_opts('username')) -@click_opt_wrap(*cli_opts('password')) -@click_opt_wrap(*cli_opts('bearer_auth', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('opaque_id', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('request_timeout')) -@click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF, override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('ca_certs')) -@click_opt_wrap(*cli_opts('client_cert')) -@click_opt_wrap(*cli_opts('client_key')) -@click_opt_wrap(*cli_opts('ssl_assert_hostname', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('ssl_assert_fingerprint', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('ssl_version', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('master-only', onoff=ONOFF, override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF, override=SHOW_OPTION)) +@options_from_dict(SHOW_EVERYTHING) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) -@click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS)) @click.version_option(__version__, '-v', '--version', prog_name='curator_cli') @click.pass_context def curator_cli( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, - dry_run, loglevel, logfile, logformat + loglevel, logfile, logformat, blacklist, dry_run ): """ Curator CLI (Singleton Tool) @@ -63,16 +40,10 @@ def curator_cli( """ ctx.obj = {} ctx.obj['dry_run'] = dry_run - cfg = get_config(ctx.params, default_config_file()) - configure_logging(cfg, ctx.params) - client_args, other_args = get_args(ctx.params, cfg) - final_config = { - 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) - } - } - ctx.obj['config'] = final_config + ctx.obj['default_config'] = default_config_file() + get_config(ctx) + configure_logging(ctx) + generate_configdict(ctx) # Add the subcommands curator_cli.add_command(alias) diff --git a/docs/Changelog.rst b/docs/Changelog.rst index 185bfc44..eb2c6683 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,44 @@ Changelog ========= +8.0.11 (20 March 2024) +---------------------- + +**Announcement** + + * With the advent of ``es_client==8.12.5``, environment variables can now be used to automatically + populate command-line options. The ``ESCLIENT_`` prefix just needs to prepend the capitalized + option name, and any hyphens need to be replaced by underscores. ``--http-compress True`` is + automatically settable by having ``ESCLIENT_HTTP_COMPRESS=1``. Boolean values are 1, 0, True, + or False (case-insensitive). Options like ``hosts`` which can have multiple values just need to + have whitespace between the values, e.g. + ``ESCLIENT_HOSTS='http://127.0.0.1:9200 http://localhost:9200'``. It splits perfectly. This is + tremendous news for the containerization/k8s community. You won't have to have all of the + options spelled out any more. Just have the environment variables assigned. + * Also, log blacklisting has made it to the command-line as well. It similarly can be set via + environment variable, e.g. ``ESCLIENT_BLACKLIST='elastic_transport urllib3'``, or by multiple + ``--blacklist`` entries at the command line. + * ``es_client`` has simplified things such that I can clean up arg sprawl in the command line + scripts. + +**Changes** + +Lots of pending pull requests have been merged. Thank you to the community +members who took the time to contribute to Curator's code. + + * DOCFIX - Update date math section to use ``y`` instead of ``Y`` (#1510) + * DOCFIX - Update period filtertype description (#1550) + * add .dockerignore to increase build speed (#1604) + * DOCFIX - clarification on prefix and suffix kinds (#1558) + The provided documentation was adapted and edited. + * Use builtin unittest.mock (#1695) + * Had to also update ``helpers.testers.verify_client_object``. + * Display proper error when mapping incorrect (#1526) - @namreg + Also assisting with this is @alexhornblake in #1537 + Apologies for needing to adapt the code manually since it's been so long. + * Version bumps: + * ``es_client==8.12.6`` + 8.0.10 (1 February 2024) ------------------------ diff --git a/docs/asciidoc/inc_kinds.asciidoc b/docs/asciidoc/inc_kinds.asciidoc index f9a8fcd6..7e7bcb9d 100644 --- a/docs/asciidoc/inc_kinds.asciidoc +++ b/docs/asciidoc/inc_kinds.asciidoc @@ -21,7 +21,8 @@ To match all indices _except_ those starting with `logstash-`: exclude: True ------------- -Note: Internally _regex_ pattern is constructed from prefix value as `^{0}.*$`. Special characters should be escaped with backslash to match literally. +NOTE: Internally, the `prefix` value is used to create a _regex_ pattern: `^{0}.*$`. Any special +characters should be escaped with a backslash to match literally. === suffix @@ -44,7 +45,8 @@ To match all indices _except_ those ending with `-prod`: exclude: True ------------- -Note: Internally _regex_ pattern is constructed from suffix value as `^.*{0}$`. Specials character should be escaped with backslash to match literally. +NOTE: Internally, the `suffix` value is used to create a _regex_ pattern: `^.*{0}$`. Any special +characters should be escaped with a backslash to match literally. === timestring diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index 75913364..4cf1819f 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,7 +1,7 @@ -:curator_version: 8.0.9 +:curator_version: 8.0.11 :curator_major: 8 :curator_doc_tree: 8.0 -:es_py_version: 8.12.0 +:es_py_version: 8.12.1 :es_doc_tree: 8.12 :stack_doc_tree: 8.12 :pybuild_ver: 3.11.7 diff --git a/pyproject.toml b/pyproject.toml index cf1b5776..6b7b7f43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ keywords = [ 'index-expiry' ] dependencies = [ - "es_client==8.12.4" + "es_client==8.12.6" ] [project.optional-dependencies] @@ -94,7 +94,7 @@ run = "run-coverage --no-cov" [[tool.hatch.envs.test.matrix]] python = ["3.9", "3.10", "3.11"] -version = ["8.0.9"] +version = ["8.0.11"] [tool.pytest.ini_options] pythonpath = [".", "curator"] diff --git a/tests/integration/test_alias.py b/tests/integration/test_alias.py index 915aeeed..ecf6468b 100644 --- a/tests/integration/test_alias.py +++ b/tests/integration/test_alias.py @@ -1,11 +1,13 @@ """Test Alias action""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long import os +import logging from datetime import datetime, timedelta import pytest from elasticsearch8.exceptions import NotFoundError from . import CuratorTestCase from . import testvars +LOGGER = logging.getLogger('test_alias') HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') @@ -231,9 +233,10 @@ def test_warn_if_no_indices(self): '--remove', '{"filtertype":"pattern","kind":"prefix","value":"my"}', '--warn_if_no_indices' ] - assert 0 == self.run_subprocess(args, logname='TestCLIAlias.test_warn_if_no_indices') + LOGGER.debug('ARGS = %s', args) + assert 0 == self.run_subprocess(args) expected = {idx1: {'aliases': {alias: {}}}, idx2: {'aliases': {alias: {}}}} - assert expected == self.client.indices.get_alias(name=alias) + assert expected == dict(self.client.indices.get_alias(name=alias)) def test_exit_1_on_empty_list(self): """test_exit_1_on_empty_list""" alias = 'testalias' @@ -246,4 +249,4 @@ def test_exit_1_on_empty_list(self): '--add', '{"filtertype":"pattern","kind":"prefix","value":"dum","exclude":false}', '--remove', '{"filtertype":"pattern","kind":"prefix","value":"my","exclude":false}', ] - assert 1 == self.run_subprocess(args, logname='TestCLIAlias.test_warn_if_no_indices') \ No newline at end of file + assert 1 == self.run_subprocess(args, logname='TestCLIAlias.test_exit_1_on_empty_list') \ No newline at end of file diff --git a/tests/integration/test_create_index.py b/tests/integration/test_create_index.py index 25875c75..51456570 100644 --- a/tests/integration/test_create_index.py +++ b/tests/integration/test_create_index.py @@ -85,3 +85,25 @@ def test_already_existing_pass(self): self.invoke_runner() assert [idx] == get_indices(self.client) assert 0 == self.result.exit_code + def test_incorrect_mapping_fail_with_propper_error(self): + config = ( + '---\n' + 'actions:\n' + ' 1:\n' + ' description: "Create index as named"\n' + ' action: create_index\n' + ' options:\n' + ' name: {0}\n' + ' extra_settings:\n' + ' mappings:\n' + ' properties:\n' + ' name: ["test"]\n' + ) + idx = 'testing' + self.write_config( + self.args['configfile'], testvars.none_logging_config.format(HOST)) + self.write_config(self.args['actionfile'], config.format(idx)) + self.invoke_runner() + assert not get_indices(self.client) + assert 'mapper_parsing_exception' in self.result.stdout + assert 1 == self.result.exit_code diff --git a/tests/integration/test_delete_indices.py b/tests/integration/test_delete_indices.py index 22c3541d..bbf1775e 100644 --- a/tests/integration/test_delete_indices.py +++ b/tests/integration/test_delete_indices.py @@ -332,4 +332,4 @@ def test_name_older_than_now_cli(self): '--filter_list', '{"filtertype":"age","source":"name","direction":"older","timestring":"%Y.%m.%d","unit":"days","unit_count":5}', ] self.assertEqual(0, self.run_subprocess(args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli')) - self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) \ No newline at end of file + self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) diff --git a/tests/integration/testvars.py b/tests/integration/testvars.py index dea4d11b..c7c7b0e9 100644 --- a/tests/integration/testvars.py +++ b/tests/integration/testvars.py @@ -453,6 +453,10 @@ ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' +' - filtertype: pattern\n' +' kind: prefix\n' +' value: ilm-history-\n' +' exclude: True\n' ' - filtertype: {0}\n' ' source: {1}\n' ' range_from: {2}\n'