Merge branch 'master' into 4.x

curator/_version.py

@@ -1 +1 @@
-__version__ = '4.1.0'
+__version__ = '4.1.1'

curator/cli.py

@@ -4,23 +4,14 @@
 import click
 from voluptuous import Schema
 from .defaults import settings
-from .validators import SchemaCheck, config_file
+from .validators import SchemaCheck
+from .config_utils import process_config
 from .exceptions import *
 from .utils import *
 from .indexlist import IndexList
 from .snapshotlist import SnapshotList
 from .actions import *
 from ._version import __version__
-from .logtools import LogInfo, Whitelist, Blacklist
-
-try:
-    from logging import NullHandler
-except ImportError:
-    from logging import Handler
-
-    class NullHandler(Handler):
-        def emit(self, record):
-            pass
 
 CLASS_MAP = {
     'alias' :  Alias,
@@ -68,9 +59,6 @@ def process_action(client, config, **kwargs):
     ### Update the defaults with whatever came with opts, minus any Nones
     mykwargs.update(prune_nones(opts))
     logger.debug('Action kwargs: {0}'.format(mykwargs))
-    # This is no longer necessary with the config schema validator
-    # # Verify the args we're going to pass match the action
-    # verify_args(action, mykwargs)
 
     ### Set up the action ###
     if action == 'alias':
@@ -123,35 +111,8 @@ def cli(config, dry_run, action_file):
 
     See http://elastic.co/guide/en/elasticsearch/client/curator/current
     """
-    # Get config from yaml file
-    yaml_config  = get_yaml(config)
-    # if the file is empty, which is still valid yaml, set as an empty dict
-    yaml_config = {} if not yaml_config else prune_nones(yaml_config)
-    # Voluptuous can't verify the schema of a dict if it doesn't have keys,
-    # so make sure the keys are at least there and are dict()
-    for k in ['client', 'logging']:
-        if k not in yaml_config:
-            yaml_config[k] = {}
-        else:
-            yaml_config[k] = prune_nones(yaml_config[k])
-    config_dict = SchemaCheck(yaml_config, config_file.client(),
-        'Client Configuration', 'full configuration dictionary').result()
-    # Set up logging
-    log_opts = config_dict['logging']
-    loginfo = LogInfo(log_opts)
-    logging.root.addHandler(loginfo.handler)
-    logging.root.setLevel(loginfo.numeric_log_level)
-    logger = logging.getLogger('curator.cli')
-    # Set up NullHandler() to handle nested elasticsearch.trace Logger
-    # instance in elasticsearch python client
-    logging.getLogger('elasticsearch.trace').addHandler(NullHandler())
-    if log_opts['blacklist']:
-        for bl_entry in ensure_list(log_opts['blacklist']):
-            for handler in logging.root.handlers:
-                handler.addFilter(Blacklist(bl_entry))
-
-    client_args = config_dict['client']
-    test_client_options(client_args)
+    client_args = process_config(config)
+    logger = logging.getLogger(__name__)
     logger.debug('Client and logging options validated.')
 
     # Extract this and save it for later, in case there's no timeout_override.

curator/config_utils.py

@@ -0,0 +1,49 @@
+from voluptuous import Schema
+# from .defaults import settings
+from .validators import SchemaCheck, config_file
+from .utils import *
+from .logtools import LogInfo, Whitelist, Blacklist
+
+def test_config(config):
+    # Get config from yaml file
+    yaml_config  = get_yaml(config)
+    # if the file is empty, which is still valid yaml, set as an empty dict
+    yaml_config = {} if not yaml_config else prune_nones(yaml_config)
+    # Voluptuous can't verify the schema of a dict if it doesn't have keys,
+    # so make sure the keys are at least there and are dict()
+    for k in ['client', 'logging']:
+        if k not in yaml_config:
+            yaml_config[k] = {}
+        else:
+            yaml_config[k] = prune_nones(yaml_config[k])
+    return SchemaCheck(yaml_config, config_file.client(),
+        'Client Configuration', 'full configuration dictionary').result()
+
+def set_logging(log_opts):
+    try:
+        from logging import NullHandler
+    except ImportError:
+        from logging import Handler
+
+        class NullHandler(Handler):
+            def emit(self, record):
+                pass
+
+    # Set up logging
+    loginfo = LogInfo(log_opts)
+    logging.root.addHandler(loginfo.handler)
+    logging.root.setLevel(loginfo.numeric_log_level)
+    logger = logging.getLogger('curator.cli')
+    # Set up NullHandler() to handle nested elasticsearch.trace Logger
+    # instance in elasticsearch python client
+    logging.getLogger('elasticsearch.trace').addHandler(NullHandler())
+    if log_opts['blacklist']:
+        for bl_entry in ensure_list(log_opts['blacklist']):
+            for handler in logging.root.handlers:
+                handler.addFilter(Blacklist(bl_entry))
+
+def process_config(yaml_file):
+    config = test_config(yaml_file)
+    set_logging(config['logging'])
+    test_client_options(config['client'])
+    return config['client']

curator/repomgrcli.py

@@ -5,21 +5,12 @@
 import logging
 from .defaults import settings
 from .exceptions import *
+from .config_utils import process_config
 from .utils import *
 from ._version import __version__
-from .logtools import LogInfo
 
 logger = logging.getLogger('curator.repomgrcli')
 
-try:
-    from logging import NullHandler
-except ImportError:
-    from logging import Handler
-
-    class NullHandler(Handler):
-        def emit(self, record):
-            pass
-
 def delete_callback(ctx, param, value):
     if not value:
         ctx.abort()
@@ -31,8 +22,15 @@ def show_repos(client):
 
 @click.command(short_help='Filesystem Repository')
 @click.option('--repository', required=True, type=str, help='Repository name')
-@click.option('--location', required=True, type=str,
-            help='Shared file-system location. Must match remote path, & be accessible to all master & data nodes')
+@click.option(
+    '--location',
+    required=True,
+    type=str,
+    help=(
+        'Shared file-system location. '
+        'Must match remote path, & be accessible to all master & data nodes'
+    )
+)
 @click.option('--compression', type=bool, default=True, show_default=True,
             help='Enable/Disable metadata compression.')
 @click.option('--chunk_size', type=str,
@@ -50,7 +48,8 @@ def fs(
     """
     Create a filesystem repository.
     """
-    client = get_client(**ctx.parent.parent.params)
+    logger = logging.getLogger('curator.repomgrcli.fs')
+    client = get_client(**ctx.obj['client_args'])
     try:
         create_repository(client, repo_type='fs', **ctx.params)
     except FailedExecution as e:
@@ -85,7 +84,8 @@ def s3(
     """
     Create an S3 repository.
     """
-    client = get_client(**ctx.parent.parent.params)
+    logger = logging.getLogger('curator.repomgrcli.s3')
+    client = get_client(**ctx.obj['client_args'])
     try:
         create_repository(client, repo_type='s3', **ctx.params)
     except FailedExecution as e:
@@ -95,41 +95,19 @@ def s3(
 
 @click.group()
 @click.option(
-    '--host', help='Elasticsearch host.', default='127.0.0.1')
-@click.option(
-    '--url_prefix', help='Elasticsearch http url prefix.',default='')
-@click.option('--port', help='Elasticsearch port.', default=9200, type=int)
-@click.option('--use_ssl', help='Connect to Elasticsearch through SSL.', is_flag=True)
-@click.option('--certificate', help='Path to certificate to use for SSL validation. (OPTIONAL)', type=str, default=None)
-@click.option('--client-cert', help='Path to file containing SSL certificate for client auth. (OPTIONAL)', type=str, default=None)
-@click.option('--client-key', help='Path to file containing SSL key for client auth. (OPTIONAL)', type=str, default=None)
-@click.option('--ssl-no-validate', help='Do not validate server\'s SSL certificate', is_flag=True)
-@click.option('--http_auth', help='Use Basic Authentication ex: user:pass', default='')
-@click.option('--timeout', help='Connection timeout in seconds.', default=30, type=int)
-@click.option('--master-only', is_flag=True, help='Only operate on elected master node.')
-@click.option('--debug', is_flag=True, help='Debug mode')
-@click.option('--loglevel', help='Log level', default='INFO')
-@click.option('--logfile', help='log file', default=None)
-@click.option('--logformat', help='Log output format [default|logstash].', default='default')
-@click.version_option(version=__version__)
+    '--config',
+    help="Path to configuration file. Default: ~/.curator/curator.yml",
+    type=click.Path(exists=True), default=settings.config_file()
+)
 @click.pass_context
-def repo_mgr_cli(
-        ctx, host, url_prefix, port, use_ssl, certificate, client_cert,
-        client_key, ssl_no_validate, http_auth, timeout, master_only, debug,
-        loglevel, logfile, logformat):
+def repo_mgr_cli(ctx, config):
     """
     Repository manager for Elasticsearch Curator.
     """
-    # Set up logging
-    if debug:
-        loglevel = 'DEBUG'
-    log_opts = {'loglevel':loglevel, 'logfile':logfile, 'logformat':logformat}
-    loginfo = LogInfo(log_opts)
-    logging.root.addHandler(loginfo.handler)
-    logging.root.setLevel(loginfo.numeric_log_level)
-    # Setting up NullHandler to handle nested elasticsearch.trace Logger
-    # instance in elasticsearch python client
-    logging.getLogger('elasticsearch.trace').addHandler(NullHandler())
+    ctx.obj = {}
+    ctx.obj['client_args'] = process_config(config)
+    logger = logging.getLogger(__name__)
+    logger.debug('Client and logging options validated.')
 
 @repo_mgr_cli.group('create')
 @click.pass_context
@@ -144,7 +122,7 @@ def show(ctx):
     """
     Show all repositories
     """
-    client = get_client(**ctx.parent.params)
+    client = get_client(**ctx.obj['client_args'])
     show_repos(client)
 
 @repo_mgr_cli.command('delete')
@@ -155,11 +133,10 @@ def show(ctx):
 @click.pass_context
 def _delete(ctx, repository):
     """Delete an Elasticsearch repository"""
-    client = get_client(**ctx.parent.params)
+    client = get_client(**ctx.obj['client_args'])
     try:
         logger.info('Deleting repository {0}...'.format(repository))
         client.snapshot.delete_repository(repository=repository)
-        # sys.exit(0)
     except elasticsearch.NotFoundError:
         logger.error(
             'Unable to delete repository: {0}  Not Found.'.format(repository))

curator/snapshotlist.py

@@ -85,6 +85,7 @@ def __get_snapshots(self):
     def __map_method(self, ft):
         methods = {
             'age': self.filter_by_age,
+            'count': self.filter_by_count,
             'none': self.filter_none,
             'pattern': self.filter_by_regex,
             'state': self.filter_by_state,

curator/utils.py

@@ -451,6 +451,12 @@ def check_master(client, master_only=False):
 
 def get_client(**kwargs):
     """
+    NOTE: AWS IAM parameters `aws_key`, `aws_secret_key`, and `aws_region` are
+    provided for future compatibility, should AWS ES support the
+    ``/_cluster/state/metadata`` endpoint.  So long as this endpoint does not
+    function in AWS ES, the client will not be able to use
+    :class:`curator.indexlist.IndexList`, which is the backbone of Curator 4
+    
     Return an :class:`elasticsearch.Elasticsearch` client object using the
     provided parameters. Any of the keyword arguments the
     :class:`elasticsearch.Elasticsearch` client object can receive are valid,
@@ -541,7 +547,7 @@ def get_client(**kwargs):
             else kwargs['aws_secret_key']
         kwargs['aws_region'] = False if not 'aws_region' in kwargs \
             else kwargs['aws_region']
-        if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs['region']:
+        if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs['aws_region']:
             if not kwargs['aws_key'] and kwargs['aws_secret_key'] \
                     and kwargs['aws_region']:
                 raise MissingArgument(

curator/validators/config_file.py

@@ -8,20 +8,18 @@ def config_client():
             None, All(Coerce(int), Range(min=1, max=65535))
         ),
         Optional('url_prefix', default=''): Any(None, str),
-        Optional('use_ssl', default=False): All(Any(int, bool), Coerce(bool)),
+        Optional('use_ssl', default=False): Boolean(),
         Optional('certificate', default=None): Any(None, str),
         Optional('client_cert', default=None): Any(None, str),
         Optional('client_key', default=None): Any(None, str),
         Optional('aws_key', default=None): Any(None, str),
         Optional('aws_secret_key', default=None): Any(None, str),
         Optional('aws_region', default=None): Any(None, str),
-        Optional('ssl_no_validate', default=False): All(
-            Any(int, bool), Coerce(bool)),
+        Optional('ssl_no_validate', default=False): Boolean(),
         Optional('http_auth', default=None): Any(None, str),
         Optional('timeout', default=30): All(
             Coerce(int), Range(min=1, max=86400)),
-        Optional('master_only', default=False): All(
-            Any(int, bool), Coerce(bool)),
+        Optional('master_only', default=False): Boolean(),
     }
 
 # Configuration file: logging

curator/validators/filter_elements.py

@@ -33,8 +33,7 @@ def exclude(**kwargs):
         val = True
     else: # False by default
         val = False
-    return { Optional('exclude', default=val): All(
-        Any(bool, int), Coerce(bool)) }
+    return { Optional('exclude', default=val): Boolean() }
 
 def field(**kwargs):
     # This setting is only used with the age filtertype.
@@ -61,18 +60,20 @@ def max_num_segments(**kwargs):
 def reverse(**kwargs):
     # Only used with space filtertype
     # Should be ignored if `use_age` is True
-    return { Optional('reverse', default=True): All(
-        Any(int, bool), Coerce(bool)) }
+    return { Optional('reverse', default=True): Boolean() }
 
 def source(**kwargs):
     # This setting is only used with the age filtertype, or with the space
     # filtertype when use_age is set to True.
     if 'action' in kwargs and kwargs['action'] in settings.snapshot_actions():
-        return { Optional('source'): Any(
-            'name', 'creation_date') }
+        valuelist = Any('name', 'creation_date')
     else:
-        return { Optional('source'): Any(
-            'name', 'creation_date', 'field_stats') }
+        valuelist = Any('name', 'creation_date', 'field_stats')
+
+    if 'required' in kwargs and kwargs['required']:
+        return { Required('source'): valuelist }
+    else:
+        return { Optional('source'): valuelist }
 
 def state(**kwargs):
     # This setting is only used with the state filtertype.
@@ -110,8 +111,7 @@ def unit_count(**kwargs):
 
 def use_age(**kwargs):
     # Use of this setting requires the additional setting, source.
-    return { Optional('use_age', default=False): All(
-        Any(int, bool), Coerce(bool)) }
+    return { Optional('use_age', default=False): Boolean() }
 
 def value(**kwargs):
     # This setting is only used with the pattern filtertype and is a required

curator/validators/filters.py

@@ -37,7 +37,7 @@ def structure():
         Optional('timestring'): Any(str, None),
         Optional('unit'): str,
         Optional('unit_count'): Coerce(int),
-        Optional('use_age'): Any(int, str, bool),
+        Optional('use_age'): Boolean(),
         Optional('value'): Any(int, float, str, bool),
     }
     retval.update(filtertype())