Merge pull request #111 from untergeek/feature/101

Prep for 1.2.0 release

CONTRIBUTORS

@@ -30,3 +30,4 @@ Contributors:
 * Kamil Essekkat (ekamil)
 * (gbutt)
 * Ben Buchacher (bbuchacher)
+* Ehtesh Choudhury (shurane)

VERSION

@@ -1 +1 @@
-1.1.4-dev
+1.2.0-dev

curator/es_repo_mgr.py

@@ -16,7 +16,7 @@ class NullHandler(Handler):
         def emit(self, record):
             pass
 
-__version__ = '1.1.4-dev'
+__version__ = '1.2.0-dev'
 
 # Elasticsearch versions supported
 version_max  = (2, 0, 0)
@@ -34,6 +34,7 @@ def emit(self, record):
     'dry_run': False,
     'debug': False,
     'log_level': 'INFO',
+    'logformat': 'default',
 }
 
 def make_parser():
@@ -57,6 +58,7 @@ def make_parser():
     parser.add_argument('-D', '--debug', dest='debug', action='store_true', help='Debug mode', default=DEFAULT_ARGS['debug'])
     parser.add_argument('--loglevel', dest='log_level', action='store', help='Log level', default=DEFAULT_ARGS['log_level'], type=str)
     parser.add_argument('-l', '--logfile', dest='log_file', help='log file', type=str)
+    parser.add_argument('--logformat', dest='logformat', help='Log output format [default|logstash]. Default: default', default=DEFAULT_ARGS['logformat'], type=str)
 
     # Command sub_parsers
     subparsers = parser.add_subparsers(title='Commands', dest='command', description='Select one of the following commands:',
@@ -116,6 +118,12 @@ def make_parser():
 
     return parser
 
+class Whitelist(logging.Filter):
+    def __init__(self, *whitelist):
+        self.whitelist = [logging.Filter(name) for name in whitelist]
+
+    def filter(self, record):
+        return any(f.filter(record) for f in self.whitelist)
 
 def show(client, **kwargs):
     for repository in sorted(get_repository(client, '_all').keys()):
@@ -150,10 +158,13 @@ def _create_repository(client, dry_run=False, **kwargs):
         try:
             repo_name = kwargs['repository']
             body = create_repo_body(**kwargs)
+            logging.info("Checking if repository {0} already exists...".format(repo_name))
             result = get_repository(client, repo_name)
             if not result:
+                logging.info("Repository {0} not found. Continuing...".format(repo_name))
                 client.snapshot.create_repository(repository=repo_name, body=body)
             elif result is not None and repo_name not in result and not kwargs['dry_run']:
+                logging.info("Repository {0} not found. Continuing...".format(repo_name))
                 client.snapshot.create_repository(repository=repo_name, body=body)
             else:
                 logger.error("Unable to create repository {0}.  A repository with that name already exists.".format(repo_name))
@@ -227,10 +238,24 @@ def main():
         if not isinstance(numeric_log_level, int):
             raise ValueError('Invalid log level: %s' % arguments.log_level)
 
+    if arguments.logformat == 'logstash':
+        os.environ['TZ'] = 'UTC'
+        time.tzset()
+        format_string = '{"@timestamp":"%(asctime)s.%(msecs)03dZ", "loglevel":"%(levelname)s", "function":"%(funcName)s", "linenum":"%(lineno)d", "message":"%(message)s"}'
+        date_string = '%Y-%m-%dT%H:%M:%S'
+    else:
+        format_string = '%(asctime)s %(levelname)-9s %(funcName)22s:%(lineno)-4d %(message)s'
+        date_string = None
     logging.basicConfig(level=numeric_log_level,
-                        format='%(asctime)s.%(msecs)03d %(levelname)-9s %(funcName)22s:%(lineno)-4d %(message)s',
-                        datefmt="%Y-%m-%dT%H:%M:%S",
+                        format=format_string,
+                        datefmt=date_string,
                         stream=open(arguments.log_file, 'a') if arguments.log_file else sys.stderr)
+
+
+    # Filter out logging from Elasticsearch and associated modules by default
+    if not arguments.debug:
+        for handler in logging.root.handlers:
+            handler.addFilter(Whitelist('root', '__main__'))
     logging.info("Job starting...")
 
     # Setting up NullHandler to handle nested elasticsearch.trace Logger instance in elasticsearch python client

test_curator/integration/__init__.py

@@ -83,10 +83,12 @@ def run_curator(self, **kwargs):
     def create_indices(self, count, unit=None):
         now = datetime.utcnow()
         unit = unit if unit else self.args['time_unit']
-        if unit == 'days':
-            format = self.args['separator'].join(('%Y', '%m', '%d'))
-        else:
-            format = self.args['separator'].join(('%Y', '%m', '%d', '%H'))
+        if unit == 'weeks':
+            format = '%Y.%W'
+        elif unit == 'days':
+            format = '%Y.%m.%d'
+        elif unit == 'hours':
+            format = '%Y.%m.%d.%H'
 
         step = timedelta(**{unit: 1})
         for x in range(count):

test_curator/integration/test_snapshots.py

@@ -37,20 +37,30 @@ def test_snapshot_will_not_be_created_twice(self):
     def test_curator_will_create_and_delete_multiple_snapshots(self):
         self.create_indices(10)
         self.create_repository()
-        curator.command_loop(self.client, command='snapshot', older_than=3, delete_older_than=None, most_recent=None, repository=self.args['repository'])
+        curator.command_loop(self.client, command='snapshot', older_than=3, timestring='%Y.%m.%d', delete_older_than=None, most_recent=None, repository=self.args['repository'])
         result = curator.get_snapshot(self.client, self.args['repository'], '_all')
         self.assertEqual(7, len(result['snapshots']))
-        curator.command_loop(self.client, command='snapshot', delete_older_than=6, repository=self.args['repository'])
+        curator.command_loop(self.client, command='snapshot', delete_older_than=6, timestring='%Y.%m.%d', repository=self.args['repository'])
         result = curator.get_snapshot(self.client, self.args['repository'], '_all')
         self.assertEqual(3, len(result['snapshots']))
 
     def test_curator_will_manage_hourly_snapshots(self):
         self.create_indices(10, 'hours')
         self.create_repository()
-        curator.command_loop(self.client, time_unit='hours', command='snapshot', older_than=3, delete_older_than=None, most_recent=None, repository=self.args['repository'])
+        curator.command_loop(self.client, time_unit='hours', command='snapshot', older_than=3, timestring='%Y.%m.%d.%H', delete_older_than=None, most_recent=None, repository=self.args['repository'])
         result = curator.get_snapshot(self.client, self.args['repository'], '_all')
         self.assertEqual(7, len(result['snapshots']))
-        curator.command_loop(self.client, time_unit='hours', command='snapshot', delete_older_than=6, repository=self.args['repository'])
+        curator.command_loop(self.client, time_unit='hours', command='snapshot', timestring='%Y.%m.%d.%H', delete_older_than=6, repository=self.args['repository'])
+        result = curator.get_snapshot(self.client, self.args['repository'], '_all')
+        self.assertEqual(3, len(result['snapshots']))
+
+    def test_curator_will_manage_weekly_snapshots(self):
+        self.create_indices(10, 'weeks')
+        self.create_repository()
+        curator.command_loop(self.client, time_unit='weeks', command='snapshot', older_than=3, timestring='%Y.%W', delete_older_than=None, most_recent=None, repository=self.args['repository'])
+        result = curator.get_snapshot(self.client, self.args['repository'], '_all')
+        self.assertEqual(7, len(result['snapshots']))
+        curator.command_loop(self.client, time_unit='weeks', command='snapshot', timestring='%Y.%W', delete_older_than=6, repository=self.args['repository'])
         result = curator.get_snapshot(self.client, self.args['repository'], '_all')
         self.assertEqual(3, len(result['snapshots']))
 

test_curator/integration/test_time_based.py

@@ -7,13 +7,19 @@
 class TestTimeBasedDeletion(CuratorTestCase):
     def test_curator_will_properly_delete_indices(self):
         self.create_indices(10)
-        curator.command_loop(self.client, command='delete', older_than=3)
+        curator.command_loop(self.client, command='delete', older_than=3, timestring='%Y.%m.%d')
         mtd = self.client.cluster.state(index=self.args['prefix'] + '*', metric='metadata')
         self.assertEquals(3, len(mtd['metadata']['indices'].keys()))
 
     def test_curator_will_properly_delete_hourly_indices(self):
         self.create_indices(10, 'hours')
-        curator.command_loop(self.client, command='delete', time_unit='hours', older_than=3)
+        curator.command_loop(self.client, command='delete', time_unit='hours', older_than=3, timestring='%Y.%m.%d.%H')
+        mtd = self.client.cluster.state(index=self.args['prefix'] + '*', metric='metadata')
+        self.assertEquals(3, len(mtd['metadata']['indices'].keys()))
+
+    def test_curator_will_properly_delete_weely_indices(self):
+        self.create_indices(10, 'weeks')
+        curator.command_loop(self.client, command='delete', time_unit='weeks', older_than=3, timestring='%Y.%W')
         mtd = self.client.cluster.state(index=self.args['prefix'] + '*', metric='metadata')
         self.assertEquals(3, len(mtd['metadata']['indices'].keys()))
 
@@ -25,13 +31,30 @@ def test_find_closed_indices(self):
 
         # all indices should be expired
         index_list = curator.get_object_list(self.client, prefix='l-')
-        expired = list(curator.find_expired_data(self.client, time_unit='days', older_than=1, object_list=index_list,
+        expired = list(curator.find_expired_data(self.client, time_unit='days', older_than=1, timestring='%Y.%m.%d', object_list=index_list,
             utc_now=datetime(2014, 1, 8, 3, 45, 50), prefix='l-'))
 
         self.assertEquals(
             [
-                ('l-2014.01.01', timedelta(7)),
-                ('l-2014.01.03', timedelta(5)),
+                'l-2014.01.01',
+                'l-2014.01.03',
+            ],
+            expired
+        )
+
+    def test_find_closed_weekly_indices(self):
+        self.create_index('l-2014.03')
+        self.client.indices.close(index='l-2014.03')
+        self.create_index('l-2014.04')
+
+        # all indices should be expired
+        index_list = curator.get_object_list(self.client, prefix='l-')
+        expired = list(curator.find_expired_data(self.client, time_unit='weeks', older_than=1, timestring='%Y.%W', object_list=index_list,
+            utc_now=datetime(2014, 2, 4, 0, 0, 0), prefix='l-'))
+
+        self.assertEquals(
+            [ 'l-2014.03',
+              'l-2014.04',
             ],
             expired
         )
@@ -42,9 +65,9 @@ def test_find_indices_ignores_indices_with_different_prefix_or_time_unit(self):
         self.create_index('logstash-2012.01.01.00')
 
         index_list = curator.get_object_list(self.client, prefix=self.args['prefix'])
-        expired = list(curator.find_expired_data(self.client, time_unit='hours', older_than=1, object_list=index_list))
+        expired = list(curator.find_expired_data(self.client, time_unit='hours', older_than=1, timestring='%Y.%m.%d.%H', object_list=index_list))
         self.assertEquals(1, len(expired))
-        self.assertEquals('logstash-2012.01.01.00', expired[0][0])
+        self.assertEquals('logstash-2012.01.01.00', expired[0])
 
     def test_find_reports_correct_time_interval_from_cutoff(self):
         self.create_index('l-2014.01.01')
@@ -53,13 +76,12 @@ def test_find_reports_correct_time_interval_from_cutoff(self):
         self.create_index('l-2014.01.03')
 
         index_list = curator.get_object_list(self.client, prefix='l-')
-        expired = list(curator.find_expired_data(self.client, time_unit='days', older_than=1, object_list=index_list,
+        expired = list(curator.find_expired_data(self.client, time_unit='days', older_than=1, timestring='%Y.%m.%d', object_list=index_list,
             utc_now=datetime(2014, 1, 4, 3, 45, 50), prefix='l-'))
         self.assertEquals(
-            [
-                (u'l-2014.01.01', timedelta(3)),
-                (u'l-2014.01.02', timedelta(2)),
-                (u'l-2014.01.03', timedelta(1)),
+            [ 'l-2014.01.01',
+              'l-2014.01.02',
+              'l-2014.01.03',
             ],
             expired
         )
@@ -70,7 +92,7 @@ def test_curator_will_properly_alias_and_unalias_indices(self):
         self.create_index('dummy')
         self.client.indices.put_alias(index='dummy', name=alias)
         self.create_indices(10)
-        curator.alias_loop(self.client, prefix=self.args['prefix'], alias=alias, alias_older_than=3, unalias_older_than=None)
+        curator.alias_loop(self.client, prefix=self.args['prefix'], alias=alias, timestring='%Y.%m.%d', alias_older_than=3, unalias_older_than=None)
         self.assertEquals(8, len(self.client.indices.get_alias(name=alias)))
-        curator.alias_loop(self.client, prefix=self.args['prefix'], alias=alias, alias_older_than=None, unalias_older_than=3)
+        curator.alias_loop(self.client, prefix=self.args['prefix'], alias=alias, timestring='%Y.%m.%d', alias_older_than=None, unalias_older_than=3)
         self.assertEquals(1, len(self.client.indices.get_alias(name=alias)))

test_curator/test_curator.py

@@ -7,13 +7,14 @@
 
 class TestUtils(TestCase):
     def test_get_index_time(self):
-        for text, sep, dt in [
-            ('2014.01.19', '.', datetime(2014, 1, 19)),
-            ('2014-01-19', '-', datetime(2014, 1, 19)),
-            ('2010-12-29', '-', datetime(2010, 12, 29)),
-            ('2010.12.29.12', '.', datetime(2010, 12, 29, 12)),
+        for text, datestring, dt in [
+            ('2014.01.19', '%Y.%m.%d', datetime(2014, 1, 19)),
+            ('2014-01-19', '%Y-%m-%d', datetime(2014, 1, 19)),
+            ('2010-12-29', '%Y-%m-%d', datetime(2010, 12, 29)),
+            ('2014-28', '%Y-%W', datetime(2014, 7, 14)),
+            ('2010.12.29.12', '%Y.%m.%d.%H', datetime(2010, 12, 29, 12)),
                 ]:
-            self.assertEqual(dt, curator.get_index_time(text, sep))
+            self.assertEqual(dt, curator.get_index_time(text, datestring))
 
 class TestShowIndices(TestCase):
     def test_show_indices(self):
@@ -48,14 +49,14 @@ def test_all_daily_indices_found(self):
             'prefix-2013.01.03.10': True,
         }
         index_list = curator.get_object_list(client, prefix='prefix-')
-        expired = curator.find_expired_data(client, object_list=index_list, time_unit='days', older_than=4, prefix='prefix-', utc_now=datetime(2014, 1, 3))
+        expired = curator.find_expired_data(client, object_list=index_list, time_unit='days', older_than=4, prefix='prefix-', timestring='%Y.%m.%d', utc_now=datetime(2014, 1, 3))
 
         expired = list(expired)
 
         self.assertEquals([
-                ('prefix-2013.01.03', timedelta(days=362)),
-                ('prefix-2013.12.29', timedelta(days=2)),
-                ('prefix-2013.12.30', timedelta(days=1)),
+                'prefix-2013.01.03',
+                'prefix-2013.12.29',
+                'prefix-2013.12.30',
             ],
             expired
         )