Merge branch '8.x' into 8.0

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
-ARG PYVER=3.11.7
-ARG ALPTAG=3.18
+ARG PYVER=3.11.9
+ARG ALPTAG=3.19
 FROM python:${PYVER}-alpine${ALPTAG} as builder
 
 # Add the community repo for access to patchelf binary package

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.14'
+__version__ = '8.0.15'

curator/actions/alias.py

@@ -98,7 +98,7 @@ def remove(self, ilo, warn_if_no_indices=False):
 
             # Re-raise the exceptions.NoIndices so it will behave as before
             raise NoIndices('No indices to remove from alias') from exc
-        aliases = self.client.indices.get_alias()
+        aliases = self.client.indices.get_alias(expand_wildcards=['open', 'closed'])
         for index in ilo.working_list():
             if index in aliases:
                 self.loggit.debug('Index %s in get_aliases output', index)

curator/helpers/date_ops.py

@@ -4,7 +4,7 @@
 import re
 import string
 import time
-from datetime import timedelta, datetime
+from datetime import timedelta, datetime, timezone
 from elasticsearch8.exceptions import NotFoundError
 from curator.exceptions import ConfigurationError
 from curator.defaults.settings import date_regex
@@ -43,6 +43,7 @@ def get_epoch(self, searchme):
                 timestamp = match.group("date")
                 return datetime_to_epoch(get_datetime(timestamp, self.timestring))
 
+
 def absolute_date_range(
         unit, date_from, date_to,
         date_from_format=None, date_to_format=None
@@ -77,7 +78,7 @@ def absolute_date_range(
         raise ConfigurationError('Must provide "date_from_format" and "date_to_format"')
     try:
         start_epoch = datetime_to_epoch(get_datetime(date_from, date_from_format))
-        logger.debug('Start ISO8601 = %s', datetime.utcfromtimestamp(start_epoch).isoformat())
+        logger.debug('Start ISO8601 = %s', epoch2iso(start_epoch))
     except Exception as err:
         raise ConfigurationError(
             f'Unable to parse "date_from" {date_from} and "date_from_format" {date_from_format}. '
@@ -116,7 +117,7 @@ def absolute_date_range(
         end_epoch = get_point_of_reference(
             unit, -1, epoch=datetime_to_epoch(end_date)) -1
 
-    logger.debug('End ISO8601 = %s', datetime.utcfromtimestamp(end_epoch).isoformat())
+    logger.debug('End ISO8601 = %s', epoch2iso(end_epoch))
     return (start_epoch, end_epoch)
 
 def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
@@ -152,7 +153,7 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
     if not epoch:
         epoch = time.time()
     epoch = fix_epoch(epoch)
-    raw_point_of_ref = datetime.utcfromtimestamp(epoch)
+    raw_point_of_ref = datetime.fromtimestamp(epoch, timezone.utc)
     logger.debug('Raw point of Reference = %s', raw_point_of_ref)
     # Reverse the polarity, because -1 as last week makes sense when read by
     # humans, but datetime timedelta math makes -1 in the future.
@@ -208,7 +209,7 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
         start_date = point_of_ref - start_delta
     # By this point, we know our start date and can convert it to epoch time
     start_epoch = datetime_to_epoch(start_date)
-    logger.debug('Start ISO8601 = %s', datetime.utcfromtimestamp(start_epoch).isoformat())
+    logger.debug('Start ISO8601 = %s', epoch2iso(start_epoch))
     # This is the number of units we need to consider.
     count = (range_to - range_from) + 1
     # We have to iterate to one more month, and then subtract a second to get
@@ -234,7 +235,7 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
         # to get hours, days, or weeks, as they don't change
         end_epoch = get_point_of_reference(
             unit, count * -1, epoch=start_epoch) -1
-    logger.debug('End ISO8601 = %s', datetime.utcfromtimestamp(end_epoch).isoformat())
+    logger.debug('End ISO8601 = %s', epoch2iso(end_epoch))
     return (start_epoch, end_epoch)
 
 def datetime_to_epoch(mydate):
@@ -247,9 +248,53 @@ def datetime_to_epoch(mydate):
     :returns: An epoch timestamp based on ``mydate``
     :rtype: int
     """
-    tdelta = (mydate - datetime(1970, 1, 1))
+    tdelta = mydate - datetime(1970, 1, 1)
     return tdelta.seconds + tdelta.days * 24 * 3600
 
+def epoch2iso(epoch: int) -> str:
+    """
+    Return an ISO8601 value for epoch
+
+    :param epoch: An epoch timestamp
+    :type epoch: int
+
+    :returns: An ISO8601 timestamp
+    :rtype: str
+    """
+    # Because Python 3.12 now requires non-naive timezone declarations, we must change.
+    #
+    ### Example:
+    ### epoch == 1491256800
+    ###
+    ### The old way:
+    ###datetime.utcfromtimestamp(epoch)
+    ###   datetime.datetime(2017, 4, 3, 22, 0).isoformat()
+    ###   Result: 2017-04-03T22:00:00
+    ###
+    ### The new way:
+    ###     datetime.fromtimestamp(epoch, timezone.utc)
+    ###     datetime.datetime(2017, 4, 3, 22, 0, tzinfo=datetime.timezone.utc).isoformat()
+    ###     Result: 2017-04-03T22:00:00+00:00
+    ###
+    ### End Example
+    #
+    # Note that the +00:00 is appended now where we affirmatively declare the UTC timezone
+    #
+    # As a result, we will use this function to prune away the timezone if it is +00:00 and replace
+    # it with Z, which is shorter Zulu notation for UTC (which Elasticsearch uses)
+    #
+    # We are MANUALLY, FORCEFULLY declaring timezone.utc, so it should ALWAYS be +00:00, but could
+    # in theory sometime show up as a Z, so we test for that.
+
+    parts = datetime.fromtimestamp(epoch, timezone.utc).isoformat().split('+')
+    if len(parts) == 1:
+        if parts[0][-1] == 'Z':
+            return parts[0] # Our ISO8601 already ends with a Z for Zulu/UTC time
+        return f'{parts[0]}Z' # It doesn't end with a Z so we put one there
+    if parts[1] == '00:00':
+        return f'{parts[0]}Z' # It doesn't end with a Z so we put one there
+    return f'{parts[0]}+{parts[1]}' # Fallback publishes the +TZ, whatever that was
+
 def fix_epoch(epoch):
     """
     Fix value of ``epoch`` to be the count since the epoch in seconds only, which should be 10 or
@@ -537,7 +582,7 @@ def parse_date_pattern(name):
         if char == '%':
             pass
         elif char in date_regex() and prev == '%':
-            rendered += str(datetime.utcnow().strftime(f'%{char}'))
+            rendered += str(datetime.now(timezone.utc).strftime(f'%{char}'))
         else:
             rendered += char
         logger.debug('Partially rendered name: %s', rendered)

curator/indexlist.py

@@ -927,7 +927,6 @@ def filter_by_alias(self, aliases=None, exclude=False):
     def filter_by_count(self, count=None, reverse=True, use_age=False, pattern=None,
             source='creation_date', timestring=None, field=None, stats_result='min_value',
             exclude=True):
-        # pylint: disable=anomalous-backslash-in-string
         """
         Remove indices from the actionable list beyond the number ``count``, sorted
         reverse-alphabetically by default.  If you set ``reverse=False``, it will be sorted
@@ -951,7 +950,7 @@ def filter_by_count(self, count=None, reverse=True, use_age=False, pattern=None,
         :param pattern: Select indices to count from a regular expression pattern. This pattern
             must have one and only one capture group. This can allow a single ``count`` filter
             instance to operate against any number of matching patterns, and keep ``count`` of each
-            index in that group.  For example, given a ``pattern`` of ``'^(.*)-\d{6}$'``, it will
+            index in that group.  For example, given a ``pattern`` of ``'^(.*)-\\d{6}$'``, it will
             match both ``rollover-000001`` and ``index-999990``, but not ``logstash-2017.10.12``.
             Following the same example, if my cluster also had ``rollover-000002`` through
             ``rollover-000010`` and ``index-888888`` through ``index-999999``, it will process both

docker_test/.env

@@ -0,0 +1 @@
+export REMOTE_ES_SERVER="http://172.19.2.14:9201"

docker_test/scripts/create.sh

@@ -127,13 +127,11 @@ echo "Please select one of these environment variables to prepend your 'pytest'
 echo
 
 for IP in $IPLIST; do
-  echo "REMOTE_ES_SERVER=\"http://$IP:${REMOTE_PORT}\""
-  if [ "$AUTO_EXPORT" == "y" ]; then
-    # This puts our curatortestenv file where it can be purged easily by destroy.sh
-    cd $SCRIPTPATH
-    cd ..
-    echo "export REMOTE_ES_SERVER=http://$IP:$REMOTE_PORT" > curatortestenv
-  fi
+  REMOTE="REMOTE_ES_SERVER=\"http://$IP:${REMOTE_PORT}\""
+  echo ${REMOTE}
+  cd $SCRIPTPATH
+  cd ..
+  echo "export ${REMOTE}" > .env
 done
 
 echo

docker_test/scripts/destroy.sh

@@ -26,7 +26,7 @@ UPONE=$(pwd | awk -F\/ '{print $NF}')
 
 if [[ "$UPONE" = "docker_test" ]]; then
   rm -rf $(pwd)/repo/*
-  rm -rf $(pwd)/curatortestenv
+  cp /dev/null $(pwd)/.env
 else
   echo "WARNING: Unable to automatically empty bind mounted repo path."
   echo "Please manually empty the contents of the repo directory!"

docs/Changelog.rst

@@ -3,6 +3,36 @@
 Changelog
 =========
 
+8.0.15 (10 April 2024)
+----------------------
+
+**Announcement**
+
+  * Python 3.12 support becomes official. A few changes were necessary to ``datetime`` calls
+    which were still using naive timestamps. Tests across all minor Python versions from 3.8 - 3.12
+    verify everything is working as expected with regards to those changes. Note that Docker builds
+    are still running Python 3.11 as cx_Freeze still does not officially support Python 3.12.
+  * Added infrastructure to test multiple versions of Python against the code base. This requires
+    you to run:
+        * ``pip install -U hatch hatchling`` -- Install prerequisites
+        * ``hatch run docker:create X.Y.Z`` -- where ``X.Y.Z`` is an ES version on Docker Hub
+        * ``hatch run test:pytest`` -- Run the test suite for each supported version of Python
+        * ``hatch run docker:destroy`` -- Cleanup the Docker containers created in ``docker:create``
+
+**Bugfix**
+
+  * A bug reported in ``es_client`` with Python versions 3.8 and 3.9 has been addressed. Going
+    forward, testing protocol will be to ensure that Curator works with all supported versions of
+    Python, or support will be removed (when 3.8 is EOL, for example).
+
+**Changes**
+
+  * Address deprecation warning in ``get_alias()`` call by limiting indices to only open and
+    closed indices via ``expand_wildcards=['open', 'closed']``.
+  * Address test warnings for an improperly escaped ``\d`` in a docstring in ``indexlist.py``
+  * Updated Python version in Docker build. See Dockerfile for more information.
+  * Docker test scripts updated to make Hatch matrix testing easier (.env file)
+
 8.0.14 (2 April 2024)
 ---------------------
 

docs/asciidoc/index.asciidoc

@@ -1,10 +1,10 @@
-:curator_version: 8.0.14
+:curator_version: 8.0.15
 :curator_major: 8
 :curator_doc_tree: 8.0
 :es_py_version: 8.13.0
 :es_doc_tree: 8.13
 :stack_doc_tree: 8.13
-:pybuild_ver: 3.11.7
+:pybuild_ver: 3.11.9
 :copyright_years: 2011-2024
 :ref:  http://www.elastic.co/guide/en/elasticsearch/reference/{es_doc_tree}
 :esref: http://www.elastic.co/guide/en/elasticsearch/reference/{stack_doc_tree}

docs/conf.py

@@ -72,7 +72,7 @@
 
 intersphinx_mapping = {
 	'python': ('https://docs.python.org/3.11', None),
-    'es_client': ('https://es-client.readthedocs.io/en/v8.13.0', None),
+    'es_client': ('https://es-client.readthedocs.io/en/v8.13.1', None),
 	'elasticsearch8': ('https://elasticsearch-py.readthedocs.io/en/v8.13.0', None),
     'voluptuous': ('http://alecthomas.github.io/voluptuous/docs/_build/html', None),
     'click': ('https://click.palletsprojects.com/en/8.1.x', None),

pyproject.toml

@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 keywords = [
     'elasticsearch',
@@ -28,7 +29,7 @@ keywords = [
     'index-expiry'
 ]
 dependencies = [
-    "es_client==8.13.0"
+    "es_client==8.13.1"
 ]
 
 [project.optional-dependencies]
@@ -74,27 +75,40 @@ exclude = [
     "tests",
 ]
 
+### Docker Environment
+[tool.hatch.envs.docker]
+platforms = ["linux", "macos"]
+
+[tool.hatch.envs.docker.scripts]
+create = "docker_test/scripts/create.sh {args}"
+destroy = "docker_test/scripts/destroy.sh"
+
+### Lint environment
+[tool.hatch.envs.lint.scripts]
+run-pyright = "pyright {args:.}"
+run-black = "black --quiet --check --diff {args:.}"
+run-ruff = "ruff check --quiet {args:.}"
+run-curlylint = "curlylint {args:.}"
+python = ["run-pyright", "run-black", "run-ruff"]
+templates = ["run-curlylint"]
+all = ["python", "templates"]
+
+### Test environment
 [tool.hatch.envs.test]
+platforms = ["linux", "macos"]
 dependencies = [
-    "coverage[toml]",
     "requests",
     "pytest >=7.2.1",
-    "pytest-cov",
+    "pytest-cov"
 ]
 
-[tool.hatch.envs.test.scripts]
-step0 = "$(docker_test/scripts/destroy.sh 2&>1 /dev/null)"
-step1 = "step0 ; echo 'Starting test environment in Docker...' ; $(AUTO_EXPORT=y docker_test/scripts/create.sh 8.12.1 2&>1 /dev/null)"
-step2 = "step1 ; source docker_test/curatortestenv; echo 'Running tests:'"
-step3 = "step2 ; pytest ; EXITCODE=$?"
-step4 = "step3 ; echo 'Tests complete! Destroying Docker test environment...' "
-full = "step4 ; $(docker_test/scripts/destroy.sh 2&>1 /dev/null ) ;  exit $EXITCODE"
-run-coverage = "pytest --cov-config=pyproject.toml --cov=curator --cov=tests"
-run = "run-coverage --no-cov"
-
 [[tool.hatch.envs.test.matrix]]
-python = ["3.9", "3.10", "3.11"]
-version = ["8.0.11"]
+python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
+[tool.hatch.envs.test.scripts]
+pytest = "source docker_test/.env; pytest"
+pytest-cov = "source docker_test/.env; pytest --cov=curator"
+pytest-cov-report = "source docker_test/.env; pytest --cov=curator --cov-report=term-missing"
 
 [tool.pytest.ini_options]
 pythonpath = [".", "curator"]