diff --git a/.gitignore b/.gitignore index 08f2a066..00e94e50 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .coverage .eggs .idea +.vscode Elastic.ico Vagrant/centos/6/.vagrant Vagrant/centos/7/.vagrant diff --git a/Vagrant/centos/6/Vagrantfile b/Vagrant/centos/6/Vagrantfile deleted file mode 100644 index 3e23f382..00000000 --- a/Vagrant/centos/6/Vagrantfile +++ /dev/null @@ -1,18 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure(2) do |config| - config.vm.box = "elastic/centos-6-x86_64" - - config.vm.provision "shell", inline: <<-SHELL - sudo yum -y groupinstall "Development Tools" - sudo yum -y install python-devel zlib-devel bzip2-devel sqlite sqlite-devel openssl-devel - SHELL - - config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant" - config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant" - - config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--nictype1", "virtio"] - end -end diff --git a/Vagrant/centos/7/Vagrantfile b/Vagrant/centos/7/Vagrantfile deleted file mode 100644 index 49148805..00000000 --- a/Vagrant/centos/7/Vagrantfile +++ /dev/null @@ -1,18 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure(2) do |config| - config.vm.box = "elastic/centos-7-x86_64" - - config.vm.provision "shell", inline: <<-SHELL - sudo yum -y groupinstall "Development Tools" - sudo yum -y install python-devel zlib-devel bzip2-devel sqlite sqlite-devel openssl-devel - SHELL - - config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant" - config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant" - - config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--nictype1", "virtio"] - end -end diff --git a/Vagrant/ubuntu/14.04/Vagrantfile b/Vagrant/ubuntu/14.04/Vagrantfile deleted file mode 100644 index 7809d08e..00000000 --- a/Vagrant/ubuntu/14.04/Vagrantfile +++ /dev/null @@ -1,19 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure(2) do |config| - config.vm.box = "ubuntu/trusty64" - - config.vm.provision "shell", inline: <<-SHELL - sudo apt-get -y autoremove - sudo apt-get update - sudo apt-get install -y libxml2-dev zlib1g-dev pkg-config python-dev make build-essential libssl-dev libbz2-dev libsqlite3-dev - SHELL - - config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant" - config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant" - - config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--nictype1", "virtio"] - end -end diff --git a/curator/_version.py b/curator/_version.py index ebe51413..f7798467 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1 @@ -__version__ = '5.5.2' - +__version__ = '5.5.3' diff --git a/curator/actions.py b/curator/actions.py index 7e924161..9b5e07ad 100644 --- a/curator/actions.py +++ b/curator/actions.py @@ -1486,15 +1486,16 @@ def get_state(self): def report_state(self): """ - Log the state of the snapshot + Log the state of the snapshot and raise an exception if the state is + not ``SUCCESS`` """ self.get_state() if self.state == 'SUCCESS': - self.loggit.info( - 'Snapshot {0} successfully completed.'.format(self.name)) + self.loggit.info('Snapshot {0} successfully completed.'.format(self.name)) else: - self.loggit.warn( - 'Snapshot {0} completed with state: {0}'.format(self.state)) + msg = 'Snapshot {0} completed with state: {0}'.format(self.state) + self.loggit.error(msg) + raise exceptions.FailedSnapshot(msg) def do_dry_run(self): """ @@ -1531,6 +1532,7 @@ def do_action(self): repository=self.repository, wait_interval=self.wait_interval, max_wait=self.max_wait ) + self.report_state() else: self.loggit.warn( '"wait_for_completion" set to {0}.' @@ -1686,9 +1688,7 @@ def _get_expected_output(self): index ) ) - self.loggit.debug('index: {0} replacement: ' - '{1}'.format(index, self.expected_output[-1]) - ) + self.loggit.debug('index: {0} replacement: {1}'.format(index, self.expected_output[-1])) def report_state(self): """ @@ -1708,10 +1708,9 @@ def report_state(self): if found_count == len(self.expected_output): self.loggit.info('All indices appear to have been restored.') else: - self.loggit.error( - 'Some of the indices do not appear to have been restored. ' - 'Missing: {0}'.format(missing) - ) + msg = 'Some of the indices do not appear to have been restored. Missing: {0}'.format(missing) + self.loggit.error(msg) + raise exceptions.FailedRestore(msg) def do_dry_run(self): """ @@ -1741,7 +1740,6 @@ def do_dry_run(self): 'DRY-RUN: restore: Index {0} {1}'.format(index, replacement_msg) ) - def do_action(self): """ Restore indices with options passed. @@ -1749,8 +1747,7 @@ def do_action(self): if not self.skip_repo_fs_check: utils.test_repo_fs(self.client, self.repository) if utils.snapshot_running(self.client): - raise exceptions.SnapshotInProgress( - 'Cannot restore while a snapshot is in progress.') + raise exceptions.SnapshotInProgress('Cannot restore while a snapshot is in progress.') try: self.loggit.info('Restoring indices "{0}" from snapshot: ' '{1}'.format(self.indices, self.name) @@ -1767,6 +1764,7 @@ def do_action(self): self.client, 'restore', index_list=self.expected_output, wait_interval=self.wait_interval, max_wait=self.max_wait ) + self.report_state() else: self.loggit.warn( '"wait_for_completion" set to {0}. ' diff --git a/curator/cli.py b/curator/cli.py index e05e3736..1885c08c 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -132,6 +132,8 @@ def run(config, action_file, dry_run=False): logger.debug('timeout_override = {0}'.format(timeout_override)) ignore_empty_list = actions[idx]['options'].pop('ignore_empty_list') logger.debug('ignore_empty_list = {0}'.format(ignore_empty_list)) + allow_ilm = actions[idx]['options'].pop('allow_ilm_indices') + logger.debug('allow_ilm_indices = {0}'.format(allow_ilm)) ### Skip to next action if 'disabled' if action_disabled: diff --git a/curator/cli_singletons/alias.py b/curator/cli_singletons/alias.py index dba5cad3..16832891 100644 --- a/curator/cli_singletons/alias.py +++ b/curator/cli_singletons/alias.py @@ -8,14 +8,16 @@ @click.option('--remove', callback=validate_filter_json, help='JSON array of filters selecting indices to REMOVE from alias', default=None) @click.option('--warn_if_no_indices', is_flag=True, help='Do not raise exception if there are no actionable indices in add/remove') @click.option('--extra_settings', help='JSON version of extra_settings (see documentation)', callback=json_to_dict) +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.pass_context -def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings): +def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_indices): """ Add/Remove Indices to/from Alias """ manual_options = { 'name': name, 'extra_settings': extra_settings, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator ignore_empty_list = warn_if_no_indices diff --git a/curator/cli_singletons/allocation.py b/curator/cli_singletons/allocation.py index 7d678a7c..cd8cf3b8 100644 --- a/curator/cli_singletons/allocation.py +++ b/curator/cli_singletons/allocation.py @@ -10,9 +10,10 @@ @click.option('--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion', show_default=True) @click.option('--wait_interval', default=9, type=int, help='Seconds to wait between completion checks.', show_default=True) @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context -def allocation(ctx, key, value, allocation_type, wait_for_completion, max_wait, wait_interval, ignore_empty_list, filter_list): +def allocation(ctx, key, value, allocation_type, wait_for_completion, max_wait, wait_interval, ignore_empty_list, allow_ilm_indices, filter_list): """ Shard Routing Allocation """ @@ -23,6 +24,7 @@ def allocation(ctx, key, value, allocation_type, wait_for_completion, max_wait, 'wait_for_completion': wait_for_completion, 'max_wait': max_wait, 'wait_interval': wait_interval, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) diff --git a/curator/cli_singletons/close.py b/curator/cli_singletons/close.py index 9bc82742..bd8e6372 100644 --- a/curator/cli_singletons/close.py +++ b/curator/cli_singletons/close.py @@ -5,13 +5,17 @@ @click.command(context_settings=get_width()) @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context -def close(ctx, delete_aliases, ignore_empty_list, filter_list): +def close(ctx, delete_aliases, ignore_empty_list, allow_ilm_indices, filter_list): """ Close Indices """ - manual_options = { 'delete_aliases': delete_aliases } + manual_options = { + 'delete_aliases': delete_aliases, + 'allow_ilm_indices': allow_ilm_indices, + } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) \ No newline at end of file diff --git a/curator/cli_singletons/delete.py b/curator/cli_singletons/delete.py index 0a9bbbe8..139efa13 100644 --- a/curator/cli_singletons/delete.py +++ b/curator/cli_singletons/delete.py @@ -5,14 +5,15 @@ #### Indices #### @click.command(context_settings=get_width()) @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context -def delete_indices(ctx, ignore_empty_list, filter_list): +def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): """ Delete Indices """ # ctx.info_name is the name of the function or name specified in @click.command decorator - action = cli_action(ctx.info_name, ctx.obj['config']['client'], {}, filter_list, ignore_empty_list) + action = cli_action(ctx.info_name, ctx.obj['config']['client'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) #### Snapshots #### @@ -21,15 +22,17 @@ def delete_indices(ctx, ignore_empty_list, filter_list): @click.option('--retry_count', type=int, help='Number of times to retry (max 3)') @click.option('--retry_interval', type=int, help='Time in seconds between retries') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable snapshots') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting snapshots to act on.', required=True) @click.pass_context -def delete_snapshots(ctx, repository, retry_count, retry_interval, ignore_empty_list, filter_list): +def delete_snapshots(ctx, repository, retry_count, retry_interval, ignore_empty_list, allow_ilm_indices, filter_list): """ Delete Snapshots """ manual_options = { 'retry_count': retry_count, - 'retry_interval': retry_interval + 'retry_interval': retry_interval, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list, repository=repository) diff --git a/curator/cli_singletons/forcemerge.py b/curator/cli_singletons/forcemerge.py index 44388bbb..b1e2e146 100644 --- a/curator/cli_singletons/forcemerge.py +++ b/curator/cli_singletons/forcemerge.py @@ -6,15 +6,17 @@ @click.option('--max_num_segments', type=int, required=True, help='Maximum number of segments per shard (minimum of 1)') @click.option('--delay', type=float, help='Time in seconds to delay between operations. Default 0, maximum 3600') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context -def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, filter_list): +def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, filter_list): """ forceMerge Indices (reduce segment count) """ manual_options = { 'max_num_segments': max_num_segments, 'delay': delay, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) diff --git a/curator/cli_singletons/object_class.py b/curator/cli_singletons/object_class.py index 8dd2b628..7a48a176 100644 --- a/curator/cli_singletons/object_class.py +++ b/curator/cli_singletons/object_class.py @@ -50,6 +50,13 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l except KeyError: self.logger.critical('Action must be one of {0}'.format(list(CLASS_MAP.keys()))) self.check_options(option_dict) + else: + self.options = option_dict + # Extract allow_ilm_indices so it can be handled separately. + if 'allow_ilm_indices' in self.options: + self.allow_ilm = self.options.pop('allow_ilm_indices') + else: + self.allow_ilm = False if action == 'alias': self.alias = { 'name': option_dict['name'], @@ -61,6 +68,8 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l self.alias[k] = {} self.check_filters(kwargs[k], loc='alias singleton', key=k) self.alias[k]['filters'] = self.filters + if self.allow_ilm: + self.alias[k]['filters'].append({'filtertype':'ilm'}) elif action in [ 'cluster_routing', 'create_index', 'rollover']: self.action_kwargs = {} # No filters for these actions @@ -119,6 +128,8 @@ def check_filters(self, filter_dict, loc='singleton', key='filters'): def do_filters(self): self.logger.debug('Running filters and testing for empty list object') + if self.allow_ilm: + self.filters.append({'filtertype':'ilm','exclude':True}) try: self.list_object.iterate_filters({'filters':self.filters}) self.list_object.empty_list_check() @@ -132,7 +143,7 @@ def do_filters(self): sys.exit(1) def get_list_object(self): - if self.action in snapshot_actions(): + if self.action in snapshot_actions() or self.action == 'show_snapshots': self.list_object = SnapshotList(self.client, repository=self.repository) else: self.list_object = IndexList(self.client) @@ -164,6 +175,7 @@ def do_singleton_action(self, dry_run=False): else: self.get_list_object() self.do_filters() + self.logger.debug('OPTIONS = {0}'.format(self.options)) action_obj = self.action_class(self.list_object, **self.options) try: if dry_run: diff --git a/curator/cli_singletons/open_indices.py b/curator/cli_singletons/open_indices.py index cfec7d9e..5ca7056f 100644 --- a/curator/cli_singletons/open_indices.py +++ b/curator/cli_singletons/open_indices.py @@ -4,12 +4,13 @@ @click.command(name='open', context_settings=get_width()) @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context -def open_indices(ctx, ignore_empty_list, filter_list): +def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): """ Open Indices """ # ctx.info_name is the name of the function or name specified in @click.command decorator - action = cli_action(ctx.info_name, ctx.obj['config']['client'], {}, filter_list, ignore_empty_list) + action = cli_action(ctx.info_name, ctx.obj['config']['client'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/replicas.py b/curator/cli_singletons/replicas.py index e5baf5f9..7f100d46 100644 --- a/curator/cli_singletons/replicas.py +++ b/curator/cli_singletons/replicas.py @@ -6,15 +6,17 @@ @click.option('--count', type=int, required=True, help='Number of replicas (max 10)') @click.option('--wait_for_completion/--no-wait_for_completion', default=False, help='Wait for replication to complete', show_default=True) @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context -def replicas(ctx, count, wait_for_completion, ignore_empty_list, filter_list): +def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, filter_list): """ Change Replica Count """ manual_options = { 'count': count, 'wait_for_completion': wait_for_completion, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) diff --git a/curator/cli_singletons/restore.py b/curator/cli_singletons/restore.py index 9291ee06..8dc3948e 100644 --- a/curator/cli_singletons/restore.py +++ b/curator/cli_singletons/restore.py @@ -18,11 +18,12 @@ @click.option('--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion') @click.option('--skip_repo_fs_check', is_flag=True, show_default=True, help='Skip repository filesystem access validation.') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting snapshots to act on.', required=True) @click.pass_context def restore(ctx, repository, name, index, rename_pattern, rename_replacement, extra_settings, include_aliases, ignore_unavailable, include_global_state, partial, wait_for_completion, - wait_interval, max_wait, skip_repo_fs_check, ignore_empty_list, filter_list): + wait_interval, max_wait, skip_repo_fs_check, ignore_empty_list, allow_ilm_indices, filter_list): """ Restore Indices """ @@ -40,6 +41,7 @@ def restore(ctx, repository, name, index, rename_pattern, rename_replacement, ex 'wait_for_completion': wait_for_completion, 'max_wait': max_wait, 'wait_interval': wait_interval, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list, repository=repository) diff --git a/curator/cli_singletons/rollover.py b/curator/cli_singletons/rollover.py index 971791f4..3aed2724 100644 --- a/curator/cli_singletons/rollover.py +++ b/curator/cli_singletons/rollover.py @@ -10,8 +10,9 @@ @click.option('--extra_settings', type=str, help='JSON version of extra_settings (see documentation)', callback=json_to_dict) @click.option('--new_index', type=str, help='Optional new index name (see documentation)') @click.option('--wait_for_active_shards', type=int, default=1, show_default=True, help='Wait for number of shards to be active before returning') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.pass_context -def rollover(ctx, name, max_age, max_docs, max_size, extra_settings, new_index, wait_for_active_shards): +def rollover(ctx, name, max_age, max_docs, max_size, extra_settings, new_index, wait_for_active_shards, allow_ilm_indices): """ Rollover Index associated with Alias """ @@ -22,6 +23,7 @@ def rollover(ctx, name, max_age, max_docs, max_size, extra_settings, new_index, manual_options = { 'name': name, 'conditions': conditions, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action( diff --git a/curator/cli_singletons/show.py b/curator/cli_singletons/show.py index 10401b79..1a860f7b 100644 --- a/curator/cli_singletons/show.py +++ b/curator/cli_singletons/show.py @@ -10,14 +10,15 @@ @click.option('--header', help='Print header if --verbose', is_flag=True, show_default=True) @click.option('--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True) @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.') @click.pass_context -def show_indices(ctx, verbose, header, epoch, ignore_empty_list, filter_list): +def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indices, filter_list): """ Show Indices """ # ctx.info_name is the name of the function or name specified in @click.command decorator - action = cli_action(ctx.info_name, ctx.obj['config']['client'], {}, filter_list, ignore_empty_list) + action = cli_action(ctx.info_name, ctx.obj['config']['client'], {'allow_ilm_indices': allow_ilm_indices}, filter_list, ignore_empty_list) action.get_list_object() action.do_filters() indices = sorted(action.list_object.indices) diff --git a/curator/cli_singletons/shrink.py b/curator/cli_singletons/shrink.py index 3e56a883..bad6c03d 100644 --- a/curator/cli_singletons/shrink.py +++ b/curator/cli_singletons/shrink.py @@ -19,11 +19,12 @@ @click.option('--wait_interval', default=9, type=int, help='Seconds to wait between completion checks.') @click.option('--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context def shrink(ctx, shrink_node, node_filters, number_of_shards, number_of_replicas, shrink_prefix, shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings, wait_for_active_shards, - wait_for_rebalance, wait_for_completion, wait_interval, max_wait, ignore_empty_list, filter_list): + wait_for_rebalance, wait_for_completion, wait_interval, max_wait, ignore_empty_list, allow_ilm_indices, filter_list): """ Shrink Indices to --number_of_shards """ @@ -43,6 +44,7 @@ def shrink(ctx, shrink_node, node_filters, number_of_shards, number_of_replicas, 'wait_for_completion': wait_for_completion, 'wait_interval': wait_interval, 'max_wait': max_wait, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) diff --git a/curator/cli_singletons/snapshot.py b/curator/cli_singletons/snapshot.py index ef3043df..ca3e89bc 100644 --- a/curator/cli_singletons/snapshot.py +++ b/curator/cli_singletons/snapshot.py @@ -13,10 +13,11 @@ @click.option('--max_wait', default=-1, type=int, help='Maximum number of seconds to wait_for_completion') @click.option('--skip_repo_fs_check', is_flag=True, show_default=True, help='Skip repository filesystem access validation.') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable indices') +@click.option('--allow_ilm_indices/--no-allow_ilm_indices', help='Allow Curator to operate on Index Lifecycle Management monitored indices.', default=False, show_default=True) @click.option('--filter_list', callback=validate_filter_json, help='JSON array of filters selecting indices to act on.', required=True) @click.pass_context def snapshot(ctx, repository, name, ignore_unavailable, include_global_state, partial, - skip_repo_fs_check, wait_for_completion, wait_interval, max_wait, ignore_empty_list, filter_list): + skip_repo_fs_check, wait_for_completion, wait_interval, max_wait, ignore_empty_list, allow_ilm_indices, filter_list): """ Snapshot Indices """ @@ -30,6 +31,7 @@ def snapshot(ctx, repository, name, ignore_unavailable, include_global_state, pa 'wait_for_completion': wait_for_completion, 'max_wait': max_wait, 'wait_interval': wait_interval, + 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = cli_action(ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) diff --git a/curator/defaults/filtertypes.py b/curator/defaults/filtertypes.py index 9b68a754..e350132a 100644 --- a/curator/defaults/filtertypes.py +++ b/curator/defaults/filtertypes.py @@ -83,6 +83,9 @@ def forcemerged(action, config): filter_elements.exclude(exclude=True), ] +def ilm(action, config): + return [ filter_elements.exclude(exclude=True) ] + def kibana(action, config): return [ filter_elements.exclude(exclude=True) ] diff --git a/curator/defaults/option_defaults.py b/curator/defaults/option_defaults.py index 3b5a7c32..5eb46f15 100644 --- a/curator/defaults/option_defaults.py +++ b/curator/defaults/option_defaults.py @@ -8,6 +8,9 @@ def allocation_type(): return { Optional('allocation_type', default='require'): All( Any(*string_types), Any('require', 'include', 'exclude')) } +def allow_ilm_indices(): + return { Optional('allow_ilm_indices', default=False): Any(bool, All(Any(*string_types), Boolean())) } + def conditions(): return { Optional('conditions'): { diff --git a/curator/defaults/settings.py b/curator/defaults/settings.py index 1a863c69..f2c35642 100644 --- a/curator/defaults/settings.py +++ b/curator/defaults/settings.py @@ -73,6 +73,7 @@ def index_filtertypes(): 'closed', 'count', 'forcemerged', + 'ilm', 'kibana', 'none', 'opened', @@ -89,6 +90,7 @@ def all_filtertypes(): def default_options(): return { + 'allow_ilm_indices': False, 'continue_if_exception': False, 'disable_action': False, 'ignore_empty_list': False, diff --git a/curator/exceptions.py b/curator/exceptions.py index a75d8539..40ddc8d4 100644 --- a/curator/exceptions.py +++ b/curator/exceptions.py @@ -43,3 +43,13 @@ class ActionTimeout(CuratorException): """ Exception raised when an action fails to complete in the allotted time """ + +class FailedSnapshot(CuratorException): + """ + Exception raised when a snapshot does not complete with state SUCCESS + """ + +class FailedRestore(CuratorException): + """ + Exception raised when a Snapshot Restore does not restore all selected indices + """ \ No newline at end of file diff --git a/curator/indexlist.py b/curator/indexlist.py index 84377a48..a535d5a4 100644 --- a/curator/indexlist.py +++ b/curator/indexlist.py @@ -97,6 +97,7 @@ def __map_method(self, ft): 'closed': self.filter_closed, 'count': self.filter_by_count, 'forcemerged': self.filter_forceMerged, + 'ilm': self.filter_ilm, 'kibana': self.filter_kibana, 'none': self.filter_none, 'opened': self.filter_opened, @@ -1109,6 +1110,29 @@ def filter_period( 'Removing from list.'.format(index)) self.indices.remove(index) + def filter_ilm(self, exclude=True): + """ + Match indices that have the setting `index.lifecycle.name` + + :arg exclude: If `exclude` is `True`, this filter will remove matching + indices from `indices`. If `exclude` is `False`, then only matching + indices will be kept in `indices`. + Default is `True` + """ + self.loggit.debug('Filtering indices with index.lifecycle.name') + index_lists = utils.chunk_index_list(self.indices) + for l in index_lists: + working_list = self.client.indices.get_settings(index=utils.to_csv(l)) + if working_list: + for index in list(working_list.keys()): + try: + has_ilm = 'name' in working_list[index]['settings']['index']['lifecycle'] + msg = '{0} has index.lifecycle.name {1}'.format(index, working_list[index]['settings']['index']['lifecycle']['name']) + except KeyError: + has_ilm = False + msg = 'index.lifecycle.name is not set for index {0}'.format(index) + self.__excludify(has_ilm, exclude, index, msg) + def iterate_filters(self, filter_dict): """ Iterate over the filters defined in `config` and execute them. diff --git a/curator/validators/options.py b/curator/validators/options.py index 474550a5..222ba5af 100644 --- a/curator/validators/options.py +++ b/curator/validators/options.py @@ -133,6 +133,7 @@ def get_schema(action): # "Required" and "Optional" elements are hashes themselves. options = {} defaults = [ + option_defaults.allow_ilm_indices(), option_defaults.continue_if_exception(), option_defaults.disable_action(), option_defaults.ignore_empty_list(), diff --git a/docs/Changelog.rst b/docs/Changelog.rst index 0278abc0..f4520cd0 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,28 @@ Changelog ========= +5.5.3 (21 May 2018) +------------------- + +Short release cycle here specifically to address the Snapshot restore issue +raised in #1192 + +**Changes** + + * By default, filter out indices with ``index.lifecycle.name`` set. This can + be overridden with the option ``allow_ilm_indices`` with the caveat that + you are on your own if there are conflicts. NOTE: The Index Lifecycle + Management feature will not appear in Elasticsearch until 6.4.0 + * Removed some unused files from the repository. + +**Bug Fixes** + + * Fix an ambiguously designed Alias test (untergeek) + * Snapshot action will now raise an exception if the snapshot does not + complete with state ``SUCCESS``. Reported in #1192 (untergeek) + * The show_indices and show_snapshots singletons were not working within the + new framework. They've been fixed now. + 5.5.2 (14 May 2018) ------------------- diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index b9d08624..7937fbc0 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,4 +1,4 @@ -:curator_version: 5.5.2 +:curator_version: 5.5.3 :curator_major: 5 :curator_doc_tree: 5.5 :es_py_version: 6.2.0 diff --git a/requirements.txt b/requirements.txt index fd18a735..21843d76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ voluptuous>=0.9.3 elasticsearch>=5.5.2,!=6.0.0,<7.0.0 +boto3>=1.7.24 +requests_aws4auth>=0.9 click>=6.7 pyyaml>=3.10 certifi>=2018.4.16 diff --git a/setup.cfg b/setup.cfg index 457fe847..86f46e6d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,8 @@ classifiers = install_requires = voluptuous>=0.9.3 elasticsearch>=5.5.2,!=6.0.0,<7.0.0 + boto3>=1.7.24 + requests_aws4auth>=0.9 click>=6.7 pyyaml>=3.10 certifi>=2018.4.16 @@ -30,6 +32,8 @@ install_requires = setup_requires = voluptuous>=0.9.3 elasticsearch>=5.5.2,!=6.0.0,<7.0.0 + boto3>=1.7.24 + requests_aws4auth>=0.9 click>=6.7 pyyaml>=3.10 certifi>=2018.4.16 diff --git a/setup.py b/setup.py index 77545b3a..26eec8fb 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,8 @@ def get_version(): def get_install_requires(): res = ['elasticsearch>=5.5.2,!=6.0.0,<7.0.0' ] + res.append('boto3>=1.7.24') + res.append('requests_aws4auth>=0.9') res.append('click>=6.7') res.append('pyyaml>=3.10') res.append('voluptuous>=0.9.3') diff --git a/test/integration/test_alias.py b/test/integration/test_alias.py index 394f26b3..9bcb47be 100644 --- a/test/integration/test_alias.py +++ b/test/integration/test_alias.py @@ -321,13 +321,14 @@ def test_add_and_remove_sorted(self): ' disable_action: False\n' ' add:\n' ' filters:\n' - ' - filtertype: none\n' + ' - filtertype: pattern\n' + ' kind: prefix\n' + ' value: dum\n' ' remove:\n' ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' ' value: my\n' - ) self.write_config( self.args['configfile'], testvars.client_config.format(host, port)) @@ -344,7 +345,7 @@ def test_add_and_remove_sorted(self): ], ) self.assertEqual( - {'dummy': {'aliases': {alias: {}}}, 'my_index': {'aliases': {alias: {}}}}, + {'dummy':{'aliases':{'testalias':{}}}}, self.client.indices.get_alias(name=alias) ) diff --git a/test/integration/test_delete_indices.py b/test/integration/test_delete_indices.py index 7f82a46a..38723149 100644 --- a/test/integration/test_delete_indices.py +++ b/test/integration/test_delete_indices.py @@ -460,6 +460,60 @@ def test_name_negative_epoch(self): ], ) self.assertEquals(0, len(curator.get_indices(self.client))) + def test_allow_ilm_indices_true(self): + # ILM will not be added until 6.4 + if curator.get_version(self.client) < (6,4,0): + self.assertTrue(True) + else: + self.create_indices(10) + settings = { + 'index': { + 'lifecycle': { + 'name': 'mypolicy' + } + } + } + self.client.indices.put_settings(index='_all', body=settings) + self.write_config( + self.args['configfile'], testvars.client_config.format(host, port)) + self.write_config(self.args['actionfile'], + testvars.ilm_delete_proto.format( + 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'true' + ) + ) + test = clicktest.CliRunner() + _ = test.invoke( + curator.cli, + [ '--config', self.args['configfile'], self.args['actionfile'] ], + ) + self.assertEquals(5, len(curator.get_indices(self.client))) + def test_allow_ilm_indices_false(self): + # ILM will not be added until 6.4 + if curator.get_version(self.client) < (6,4,0): + self.assertTrue(True) + else: + self.create_indices(10) + settings = { + 'index': { + 'lifecycle': { + 'name': 'mypolicy' + } + } + } + self.client.indices.put_settings(index='_all', body=settings) + self.write_config( + self.args['configfile'], testvars.client_config.format(host, port)) + self.write_config(self.args['actionfile'], + testvars.ilm_delete_proto.format( + 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'false' + ) + ) + test = clicktest.CliRunner() + _ = test.invoke( + curator.cli, + [ '--config', self.args['configfile'], self.args['actionfile'] ], + ) + self.assertEquals(10, len(curator.get_indices(self.client))) class TestCLIDeleteIndices(CuratorTestCase): def test_name_older_than_now_cli(self): diff --git a/test/integration/testvars.py b/test/integration/testvars.py index bfdf88cb..8709d1e1 100644 --- a/test/integration/testvars.py +++ b/test/integration/testvars.py @@ -828,4 +828,22 @@ ' filters:\n' ' - filtertype: pattern\n' ' kind: prefix\n' -' value: my\n') \ No newline at end of file +' value: my\n') + +ilm_delete_proto = ('---\n' +'actions:\n' +' 1:\n' +' description: "Delete indices as filtered"\n' +' action: delete_indices\n' +' options:\n' +' alllow_ilm_indices: {9}\n' +' filters:\n' +' - filtertype: {0}\n' +' source: {1}\n' +' direction: {2}\n' +' timestring: {3}\n' +' unit: {4}\n' +' unit_count: {5}\n' +' field: {6}\n' +' stats_result: {7}\n' +' epoch: {8}\n') \ No newline at end of file diff --git a/test/unit/test_action_restore.py b/test/unit/test_action_restore.py index a0dd7667..6f18143d 100644 --- a/test/unit/test_action_restore.py +++ b/test/unit/test_action_restore.py @@ -110,7 +110,7 @@ def test_report_state_not_all(self): slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore( slo, rename_pattern='(.+)', rename_replacement='new_$1') - self.assertIsNone(ro.report_state()) + self.assertRaises(curator.exceptions.FailedRestore, ro.report_state) def test_do_action_success(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } diff --git a/test/unit/test_action_snapshot.py b/test/unit/test_action_snapshot.py index 4f7018d0..7ed2b854 100644 --- a/test/unit/test_action_snapshot.py +++ b/test/unit/test_action_snapshot.py @@ -101,8 +101,7 @@ def test_report_state_other(self): ilo = curator.IndexList(client) so = curator.Snapshot(ilo, repository=testvars.repo_name, name=testvars.snap_name) - so.report_state() - self.assertEqual('IN_PROGRESS', so.state) + self.assertRaises(curator.exceptions.FailedSnapshot, so.report_state) def test_do_dry_run(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } diff --git a/test/unit/test_class_index_list.py b/test/unit/test_class_index_list.py index b4a8558a..68fb29f5 100644 --- a/test/unit/test_class_index_list.py +++ b/test/unit/test_class_index_list.py @@ -1,5 +1,6 @@ from unittest import TestCase from mock import Mock, patch +from copy import deepcopy import elasticsearch import yaml import curator @@ -887,6 +888,30 @@ def test_unknown_filtertype_raises(self): curator.ConfigurationError, ilo.iterate_filters, config ) + def test_ilm_filtertype_exclude(self): + client = Mock() + client.info.return_value = {'version': {'number': '6.3.0'} } + # If we don't deepcopy, then it munges the settings for future references. + with_ilm = deepcopy(testvars.settings_two) + with_ilm['index-2016.03.03']['settings']['index']['lifecycle'] = {'name':'mypolicy'} + client.indices.get_settings.return_value = with_ilm + client.cluster.state.return_value = testvars.clu_state_two + client.indices.stats.return_value = testvars.stats_two + ilo = curator.IndexList(client) + config = {'filters': [{'filtertype':'ilm','exclude':True}]} + ilo.iterate_filters(config) + self.assertEqual(['index-2016.03.04'], ilo.indices) + def test_ilm_filtertype_no_setting(self): + client = Mock() + client.info.return_value = {'version': {'number': '6.3.0'} } + client.indices.get_settings.return_value = testvars.settings_two + client.cluster.state.return_value = testvars.clu_state_two + client.indices.stats.return_value = testvars.stats_two + ilo = curator.IndexList(client) + config = {'filters': [{'filtertype':'ilm','exclude':True}]} + ilo.iterate_filters(config) + self.assertEqual(['index-2016.03.03','index-2016.03.04'], sorted(ilo.indices)) + class TestIndexListFilterAlias(TestCase): def test_raise(self): client = Mock() @@ -976,7 +1001,7 @@ def test_pattern_no_regex_group(self): def test_pattern_multiple_regex_groups(self): self.builder() self.assertRaises(curator.ActionError, self.il.filter_by_count, - count=1, use_age=True, pattern='^(\ )foo(\ )$', source='name', timestring='%Y.%m.%d', + count=1, use_age=True, pattern=r'^(\ )foo(\ )$', source='name', timestring='%Y.%m.%d', ) class TestIndexListPeriodFilterName(TestCase): def test_get_name_based_age_in_range(self): @@ -1017,7 +1042,6 @@ def test_bad_arguments(self): range_to = -3 timestring = '%Y.%m.%d' epoch = 1456963201 - expected = [] client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_two diff --git a/test/unit/testvars.py b/test/unit/testvars.py index 132bf701..236b7e48 100644 --- a/test/unit/testvars.py +++ b/test/unit/testvars.py @@ -1,5 +1,4 @@ import elasticsearch -from voluptuous import * fake_fail = Exception('Simulated Failure') four_oh_one = elasticsearch.TransportError(401, "simulated error") diff --git a/unix_packages/build_official_package.sh b/unix_packages/build_official_package.sh index 3218dbe6..3a2e841d 100755 --- a/unix_packages/build_official_package.sh +++ b/unix_packages/build_official_package.sh @@ -4,7 +4,7 @@ BASEPATH=$(pwd) PKG_TARGET=/curator_packages WORKDIR=/tmp/curator PYVER=3.6 -MINOR=4 +MINOR=5 INPUT_TYPE=python CATEGORY=python VENDOR=Elastic diff --git a/unix_packages/build_package_from_source.sh b/unix_packages/build_package_from_source.sh index 432ef097..808850b2 100755 --- a/unix_packages/build_package_from_source.sh +++ b/unix_packages/build_package_from_source.sh @@ -4,7 +4,7 @@ BASEPATH=$(pwd) PKG_TARGET=/curator_packages WORKDIR=/tmp/curator PYVER=3.6 -MINOR=4 +MINOR=5 INPUT_TYPE=python CATEGORY=python VENDOR=Elastic diff --git a/unix_packages/cx_freeze-5.0.1.dev.tar.gz b/unix_packages/cx_freeze-5.0.1.dev.tar.gz deleted file mode 100644 index 5836d063..00000000 Binary files a/unix_packages/cx_freeze-5.0.1.dev.tar.gz and /dev/null differ