From 340969638787d4c06ccccb7797a22aed45391a35 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Fri, 2 Feb 2024 16:36:07 +0100 Subject: [PATCH 1/6] Update date math section to use y instead of Y (#1510) elasticsearch has changed to use y instead of Y as it migrated to use java.time --- docs/asciidoc/inc_datemath.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/asciidoc/inc_datemath.asciidoc b/docs/asciidoc/inc_datemath.asciidoc index 4811f90b..913aba95 100644 --- a/docs/asciidoc/inc_datemath.asciidoc +++ b/docs/asciidoc/inc_datemath.asciidoc @@ -11,7 +11,7 @@ A date math name takes the following form: |=== |static_name| is the static text part of the name |date_math_expr| is a dynamic date math expression that computes the date dynamically -|date_format|is the optional format in which the computed date should be rendered. Defaults to `YYYY.MM.dd`. +|date_format|is the optional format in which the computed date should be rendered. Defaults to `yyyy.MM.dd`. |time_zone|is the optional time zone . Defaults to `utc`. |=== @@ -22,7 +22,7 @@ The following example shows different forms of date math names and the final for |Expression| Resolves to || logstash-2024.03.22 || logstash-2024.03.01 -|| logstash-2024.03 -|| logstash-2024.02 -| | logstash-2024.03.23 -|=== \ No newline at end of file +|| logstash-2024.03 +|| logstash-2024.02 +| | logstash-2024.03.23 +|=== From 5130c56d89bf72b38d4ec7dbe8e20fe5562d2aba Mon Sep 17 00:00:00 2001 From: mattsdevop <44505636+mattsdevop@users.noreply.github.com> Date: Fri, 2 Feb 2024 10:40:55 -0500 Subject: [PATCH 2/6] Update period filtertype description (#1550) Update period filtertype description to correctly reflect the possible options for `unit` referenced in https://www.elastic.co/guide/en/elasticsearch/client/curator/current/fe_unit.html as `hours`, `days`, `weeks`, `months`, or `years` only. --- docs/asciidoc/filters.asciidoc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/asciidoc/filters.asciidoc b/docs/asciidoc/filters.asciidoc index e401a9f4..a15f01f7 100644 --- a/docs/asciidoc/filters.asciidoc +++ b/docs/asciidoc/filters.asciidoc @@ -611,15 +611,15 @@ NOTE: Empty values and commented lines will result in the default value, if any, === Relative Periods A relative period will be reckoned relative to execution time, unless an <> timestamp is provided. Reckoning is truncated to the most -recent whole unit, where a <> can be one of `seconds`, `minutes`, -`hours`, `days`, `weeks`, `months`, or `years`. For example, if I selected -`hours` as my `unit`, and I began execution at 02:35, then the point of reckoning -would be 02:00. This is relatively easy with `days`, `months`, and `years`, but -slightly more complicated with `weeks`. Some users may wish to reckon weeks by -the ISO standard, which starts weeks on Monday. Others may wish to use Sunday -as the first day of the week. Both are acceptable options with the `period` -filter. The default behavior for `weeks` is to have Sunday be the start of the -week. This can be overridden with <> as follows: +recent whole unit, where a <> can be one of `hours`, `days`, `weeks`, +`months`, or `years`. For example, if I selected `hours` as my `unit`, and I +began execution at 02:35, then the point of reckoning would be 02:00. This is +relatively easy with `days`, `months`, and `years`, but slightly more complicated +with `weeks`. Some users may wish to reckon weeks by the ISO standard, which +starts weeks on Monday. Others may wish to use Sunday as the first day of the +week. Both are acceptable options with the `period` filter. The default behavior +for `weeks` is to have Sunday be the start of the week. This can be overridden +with <> as follows: [source,yaml] ------------- From c804a456002ad9a23e0db616765040dedc0dda16 Mon Sep 17 00:00:00 2001 From: harduino <37391150+harduino@users.noreply.github.com> Date: Fri, 2 Feb 2024 18:47:51 +0300 Subject: [PATCH 3/6] add .dockerignore to increase build speed (#1604) This change is already in place now, so it's safe to merge. From dfe3d5d03b8bb99bc9fad8b8ed7764db3e8a020a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Krze=C5=9Bniak?= Date: Fri, 2 Feb 2024 16:51:48 +0100 Subject: [PATCH 4/6] [docs] clarification on prefix and suffix kinds (#1558) Clarify that values of `prefix` and `suffix` filter kinds are not taken literally but rather a regexp is constructed from them. This is important difference if index names are using dots (or other special characters). For example, with prefix of `value: project.prod-a.` the index `project.prod-admin-portal.2020.10.15` is matched. By reading the documentation this is unexpected as the impression is that prefix and suffix should be used literally. --- docs/asciidoc/inc_kinds.asciidoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/asciidoc/inc_kinds.asciidoc b/docs/asciidoc/inc_kinds.asciidoc index 188991c1..f9a8fcd6 100644 --- a/docs/asciidoc/inc_kinds.asciidoc +++ b/docs/asciidoc/inc_kinds.asciidoc @@ -21,6 +21,8 @@ To match all indices _except_ those starting with `logstash-`: exclude: True ------------- +Note: Internally _regex_ pattern is constructed from prefix value as `^{0}.*$`. Special characters should be escaped with backslash to match literally. + === suffix To match all indices ending with `-prod`: @@ -42,6 +44,8 @@ To match all indices _except_ those ending with `-prod`: exclude: True ------------- +Note: Internally _regex_ pattern is constructed from suffix value as `^.*{0}$`. Specials character should be escaped with backslash to match literally. + === timestring IMPORTANT: No age calculation takes place here. It is strictly a pattern match. From 1375775cb833a0c4901837fcb8452ee59d74eb99 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Fri, 2 Feb 2024 16:59:25 +0100 Subject: [PATCH 5/6] Use build in unittest.mock (#1695) Since Python 3.3 the unittest module provides mock. --- docs/testing.rst | 5 ++--- pyproject.toml | 2 -- tests/integration/test_es_repo_mgr.py | 2 +- tests/unit/__init__.py | 2 +- tests/unit/test_action_alias.py | 2 +- tests/unit/test_action_allocation.py | 2 +- tests/unit/test_action_close.py | 2 +- tests/unit/test_action_clusterrouting.py | 2 +- tests/unit/test_action_cold2frozen.py | 2 +- tests/unit/test_action_create_index.py | 2 +- tests/unit/test_action_delete_indices.py | 2 +- tests/unit/test_action_delete_snapshots.py | 2 +- tests/unit/test_action_forcemerge.py | 2 +- tests/unit/test_action_indexsettings.py | 2 +- tests/unit/test_action_open.py | 2 +- tests/unit/test_action_reindex.py | 2 +- tests/unit/test_action_replicas.py | 2 +- tests/unit/test_action_restore.py | 2 +- tests/unit/test_action_rollover.py | 2 +- tests/unit/test_action_shrink.py | 2 +- tests/unit/test_action_snapshot.py | 2 +- tests/unit/test_class_index_list.py | 2 +- tests/unit/test_class_snapshot_list.py | 2 +- tests/unit/test_helpers_date_ops.py | 2 +- tests/unit/test_helpers_getters.py | 2 +- tests/unit/test_helpers_testers.py | 2 +- tests/unit/test_helpers_utils.py | 2 +- tests/unit/test_helpers_waiters.py | 2 +- 28 files changed, 28 insertions(+), 31 deletions(-) diff --git a/docs/testing.rst b/docs/testing.rst index f4a16d5b..a7066b80 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -34,7 +34,6 @@ An example is listed below: [project.optional-dependencies] test = [ - "mock", "requests", "pytest >=7.2.1", "pytest-cov", @@ -45,7 +44,7 @@ before running the following: .. code-block:: shell - pip install -U mock requests pytest pytest-cov + pip install -U requests pytest pytest-cov It should be simpler to run the regular method, but if you have some reason to do this manually, those are the steps. @@ -308,4 +307,4 @@ This will test method ``test_method`` of class ``TestClass`` in ``test_file.py`` $ pytest tests/unit/test_file.py::TestClass::test_method . [100%] - 1 passed in 0.31s \ No newline at end of file + 1 passed in 0.31s diff --git a/pyproject.toml b/pyproject.toml index 001c8b26..cf1b5776 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "mock", "requests", "pytest >=7.2.1", "pytest-cov", @@ -78,7 +77,6 @@ exclude = [ [tool.hatch.envs.test] dependencies = [ "coverage[toml]", - "mock", "requests", "pytest >=7.2.1", "pytest-cov", diff --git a/tests/integration/test_es_repo_mgr.py b/tests/integration/test_es_repo_mgr.py index 4acb36f9..0e04cc74 100644 --- a/tests/integration/test_es_repo_mgr.py +++ b/tests/integration/test_es_repo_mgr.py @@ -14,7 +14,7 @@ # class TestLoggingModules(CuratorTestCase): # def test_logger_without_null_handler(self): -# from mock import patch, Mock +# from unittest.mock import patch, Mock # mock = Mock() # modules = {'logger': mock, 'logger.NullHandler': mock.module} # self.write_config( diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index b6043a2c..cee1dd40 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -4,7 +4,7 @@ import random import string from unittest import SkipTest, TestCase -from mock import Mock +from unittest.mock import Mock from .testvars import * class CLITestCase(TestCase): diff --git a/tests/unit/test_action_alias.py b/tests/unit/test_action_alias.py index c80e6d6f..e46fc0bf 100644 --- a/tests/unit/test_action_alias.py +++ b/tests/unit/test_action_alias.py @@ -1,7 +1,7 @@ """Alias unit tests""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator import IndexList from curator.exceptions import ActionError, FailedExecution, MissingArgument, NoIndices from curator.actions.alias import Alias diff --git a/tests/unit/test_action_allocation.py b/tests/unit/test_action_allocation.py index 9e2e1c4a..70651506 100644 --- a/tests/unit/test_action_allocation.py +++ b/tests/unit/test_action_allocation.py @@ -1,7 +1,7 @@ """Alias unit tests""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator import IndexList from curator.exceptions import MissingArgument from curator.actions.allocation import Allocation diff --git a/tests/unit/test_action_close.py b/tests/unit/test_action_close.py index 5a3656db..3b06cfde 100644 --- a/tests/unit/test_action_close.py +++ b/tests/unit/test_action_close.py @@ -1,7 +1,7 @@ """Alias unit tests""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator import IndexList from curator.exceptions import FailedExecution from curator.actions.close import Close diff --git a/tests/unit/test_action_clusterrouting.py b/tests/unit/test_action_clusterrouting.py index bf0ca70a..23f4a630 100644 --- a/tests/unit/test_action_clusterrouting.py +++ b/tests/unit/test_action_clusterrouting.py @@ -1,6 +1,6 @@ """test_action_clusterrouting""" from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import ClusterRouting # Get test variables and constants from a single source from . import testvars diff --git a/tests/unit/test_action_cold2frozen.py b/tests/unit/test_action_cold2frozen.py index a63d103c..becda8c4 100644 --- a/tests/unit/test_action_cold2frozen.py +++ b/tests/unit/test_action_cold2frozen.py @@ -1,7 +1,7 @@ """test_action_cold2frozen""" # pylint: disable=attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock import pytest from curator.actions import Cold2Frozen from curator.exceptions import CuratorException, SearchableSnapshotException diff --git a/tests/unit/test_action_create_index.py b/tests/unit/test_action_create_index.py index 53c51397..1c0b5f45 100644 --- a/tests/unit/test_action_create_index.py +++ b/tests/unit/test_action_create_index.py @@ -1,6 +1,6 @@ """Unit tests for create_index action""" from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import CreateIndex from curator.exceptions import ConfigurationError, FailedExecution # Get test variables and constants from a single source diff --git a/tests/unit/test_action_delete_indices.py b/tests/unit/test_action_delete_indices.py index 988c946d..7d4cef56 100644 --- a/tests/unit/test_action_delete_indices.py +++ b/tests/unit/test_action_delete_indices.py @@ -1,7 +1,7 @@ """test_action_delete_indices""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import DeleteIndices from curator.exceptions import FailedExecution from curator import IndexList diff --git a/tests/unit/test_action_delete_snapshots.py b/tests/unit/test_action_delete_snapshots.py index be137f18..a51077f4 100644 --- a/tests/unit/test_action_delete_snapshots.py +++ b/tests/unit/test_action_delete_snapshots.py @@ -1,6 +1,6 @@ """test_action_delete_snapshots""" from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import DeleteSnapshots from curator.exceptions import FailedExecution from curator import SnapshotList diff --git a/tests/unit/test_action_forcemerge.py b/tests/unit/test_action_forcemerge.py index acfb6d59..ec78bcdf 100644 --- a/tests/unit/test_action_forcemerge.py +++ b/tests/unit/test_action_forcemerge.py @@ -1,7 +1,7 @@ """test_action_forcemerge""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import ForceMerge from curator.exceptions import FailedExecution, MissingArgument from curator import IndexList diff --git a/tests/unit/test_action_indexsettings.py b/tests/unit/test_action_indexsettings.py index bcfcb9cc..53a6a04c 100644 --- a/tests/unit/test_action_indexsettings.py +++ b/tests/unit/test_action_indexsettings.py @@ -1,7 +1,7 @@ """test_action_indexsettings""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import IndexSettings from curator.exceptions import ActionError, ConfigurationError, MissingArgument from curator import IndexList diff --git a/tests/unit/test_action_open.py b/tests/unit/test_action_open.py index 37e4825b..e458d9e1 100644 --- a/tests/unit/test_action_open.py +++ b/tests/unit/test_action_open.py @@ -1,7 +1,7 @@ """test_action_open""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import Open from curator.exceptions import FailedExecution from curator import IndexList diff --git a/tests/unit/test_action_reindex.py b/tests/unit/test_action_reindex.py index 4c457cbf..a783ebe0 100644 --- a/tests/unit/test_action_reindex.py +++ b/tests/unit/test_action_reindex.py @@ -1,7 +1,7 @@ """test_action_reindex""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import Reindex from curator.exceptions import ConfigurationError, CuratorException, FailedExecution, NoIndices from curator import IndexList diff --git a/tests/unit/test_action_replicas.py b/tests/unit/test_action_replicas.py index 6ff4844e..acf4414f 100644 --- a/tests/unit/test_action_replicas.py +++ b/tests/unit/test_action_replicas.py @@ -1,7 +1,7 @@ """test_action_replicas""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import Replicas from curator.exceptions import FailedExecution, MissingArgument from curator import IndexList diff --git a/tests/unit/test_action_restore.py b/tests/unit/test_action_restore.py index aa8092b7..ecd5702f 100644 --- a/tests/unit/test_action_restore.py +++ b/tests/unit/test_action_restore.py @@ -1,7 +1,7 @@ """test_action_restore""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import Restore from curator.exceptions import CuratorException, FailedExecution, FailedRestore, SnapshotInProgress from curator import SnapshotList diff --git a/tests/unit/test_action_rollover.py b/tests/unit/test_action_rollover.py index 48b2957a..37aa6220 100644 --- a/tests/unit/test_action_rollover.py +++ b/tests/unit/test_action_rollover.py @@ -1,7 +1,7 @@ """test_action_rollover""" # pylint: disable=missing-function-docstring, missing-class-docstring, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import Rollover from curator.exceptions import ConfigurationError diff --git a/tests/unit/test_action_shrink.py b/tests/unit/test_action_shrink.py index d8e15283..2f569d7c 100644 --- a/tests/unit/test_action_shrink.py +++ b/tests/unit/test_action_shrink.py @@ -1,7 +1,7 @@ """test_action_shrink""" # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import Shrink from curator.exceptions import ActionError, ConfigurationError from curator import IndexList diff --git a/tests/unit/test_action_snapshot.py b/tests/unit/test_action_snapshot.py index d2316753..83420a3e 100644 --- a/tests/unit/test_action_snapshot.py +++ b/tests/unit/test_action_snapshot.py @@ -1,7 +1,7 @@ """test_action_snapshot""" # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long, protected-access, attribute-defined-outside-init from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from curator.actions import Snapshot from curator.exceptions import ActionError, CuratorException, FailedExecution, FailedSnapshot, MissingArgument, SnapshotInProgress from curator import IndexList diff --git a/tests/unit/test_class_index_list.py b/tests/unit/test_class_index_list.py index c2624b72..77e79a85 100644 --- a/tests/unit/test_class_index_list.py +++ b/tests/unit/test_class_index_list.py @@ -2,7 +2,7 @@ # pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long, attribute-defined-outside-init, protected-access from unittest import TestCase from copy import deepcopy -from mock import Mock +from unittest.mock import Mock import yaml from es_client.exceptions import FailedValidation from curator.exceptions import ActionError, ConfigurationError, FailedExecution, MissingArgument, NoIndices diff --git a/tests/unit/test_class_snapshot_list.py b/tests/unit/test_class_snapshot_list.py index 79758aed..d7065379 100644 --- a/tests/unit/test_class_snapshot_list.py +++ b/tests/unit/test_class_snapshot_list.py @@ -1,6 +1,6 @@ """test_class_snapshot_list""" from unittest import TestCase -from mock import Mock +from unittest.mock import Mock import yaml from es_client.exceptions import FailedValidation from curator import SnapshotList diff --git a/tests/unit/test_helpers_date_ops.py b/tests/unit/test_helpers_date_ops.py index 86825ef6..1b126828 100644 --- a/tests/unit/test_helpers_date_ops.py +++ b/tests/unit/test_helpers_date_ops.py @@ -2,7 +2,7 @@ from unittest import TestCase from datetime import datetime import pytest -from mock import Mock +from unittest.mock import Mock from elasticsearch8 import NotFoundError from elastic_transport import ApiResponseMeta from curator.exceptions import ConfigurationError diff --git a/tests/unit/test_helpers_getters.py b/tests/unit/test_helpers_getters.py index a6f4ae30..9c3efb19 100644 --- a/tests/unit/test_helpers_getters.py +++ b/tests/unit/test_helpers_getters.py @@ -1,6 +1,6 @@ """Unit testing for helpers.creators functions""" from unittest import TestCase -from mock import Mock +from unittest.mock import Mock import pytest from elastic_transport import ApiResponseMeta from elasticsearch8 import NotFoundError, TransportError diff --git a/tests/unit/test_helpers_testers.py b/tests/unit/test_helpers_testers.py index 9dc984f4..f4cbb295 100644 --- a/tests/unit/test_helpers_testers.py +++ b/tests/unit/test_helpers_testers.py @@ -1,7 +1,7 @@ """Unit tests for utils""" from unittest import TestCase import pytest -from mock import Mock +from unittest.mock import Mock from elastic_transport import ApiResponseMeta from elasticsearch8 import Elasticsearch from elasticsearch8.exceptions import AuthenticationException, NotFoundError diff --git a/tests/unit/test_helpers_utils.py b/tests/unit/test_helpers_utils.py index 613058ee..c1204b7b 100644 --- a/tests/unit/test_helpers_utils.py +++ b/tests/unit/test_helpers_utils.py @@ -1,7 +1,7 @@ """Unit tests for utils""" from unittest import TestCase # import pytest -from mock import Mock +from unittest.mock import Mock # from curator.exceptions import MissingArgument from curator.indexlist import IndexList from curator.helpers.utils import chunk_index_list, show_dry_run, to_csv diff --git a/tests/unit/test_helpers_waiters.py b/tests/unit/test_helpers_waiters.py index 732f049f..ea7aebde 100644 --- a/tests/unit/test_helpers_waiters.py +++ b/tests/unit/test_helpers_waiters.py @@ -1,7 +1,7 @@ """Unit tests for utils""" from unittest import TestCase import pytest -from mock import Mock +from unittest.mock import Mock from curator.exceptions import ActionTimeout, ConfigurationError, CuratorException, MissingArgument from curator.helpers.waiters import ( health_check, restore_check, snapshot_check,task_check, wait_for_it) From 7c571030fcc6f48fbd66b6f938f4588c51a19675 Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Wed, 20 Mar 2024 19:16:10 -0600 Subject: [PATCH 6/6] Release prep for 8.0.11 (#1706) --- curator/_version.py | 2 +- curator/actions/cold2frozen.py | 107 ++++++++++++++--------- curator/actions/create_index.py | 10 ++- curator/cli.py | 69 +++++---------- curator/cli_singletons/alias.py | 5 +- curator/cli_singletons/allocation.py | 5 +- curator/cli_singletons/close.py | 5 +- curator/cli_singletons/delete.py | 9 +- curator/cli_singletons/forcemerge.py | 5 +- curator/cli_singletons/object_class.py | 20 ++--- curator/cli_singletons/open_indices.py | 5 +- curator/cli_singletons/replicas.py | 5 +- curator/cli_singletons/restore.py | 5 +- curator/cli_singletons/rollover.py | 5 +- curator/cli_singletons/show.py | 9 +- curator/cli_singletons/shrink.py | 5 +- curator/cli_singletons/snapshot.py | 5 +- curator/helpers/testers.py | 3 +- curator/repomgrcli.py | 20 ++--- curator/singletons.py | 49 +++-------- docs/Changelog.rst | 38 ++++++++ docs/asciidoc/inc_kinds.asciidoc | 6 +- docs/asciidoc/index.asciidoc | 4 +- pyproject.toml | 4 +- tests/integration/test_alias.py | 9 +- tests/integration/test_create_index.py | 22 +++++ tests/integration/test_delete_indices.py | 2 +- tests/integration/testvars.py | 4 + 28 files changed, 228 insertions(+), 209 deletions(-) diff --git a/curator/_version.py b/curator/_version.py index 434b2f25..9d236b0a 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1,2 @@ """Curator Version""" -__version__ = '8.0.10' +__version__ = '8.0.11' diff --git a/curator/actions/cold2frozen.py b/curator/actions/cold2frozen.py index e4512e5f..20daaf90 100644 --- a/curator/actions/cold2frozen.py +++ b/curator/actions/cold2frozen.py @@ -127,56 +127,85 @@ def do_dry_run(self): ) self.loggit.info(msg) - - def do_action(self): + def mount_index(self, newidx, kwargs): """ Call :py:meth:`~.elasticsearch.client.SearchableSnapshotsClient.mount` to mount the indices in :py:attr:`ilo` in the Frozen tier. + """ + try: + self.loggit.debug('Mounting new index %s in frozen tier...', newidx) + self.client.searchable_snapshots.mount(**kwargs) + # pylint: disable=broad-except + except Exception as err: + report_failure(err) - Verify index looks good - + def verify_mount(self, newidx): + """ + Verify that newidx is a mounted index + """ + self.loggit.debug('Verifying new index %s is mounted properly...', newidx) + idx_settings = self.client.indices.get(index=newidx)[newidx] + if is_idx_partial(idx_settings['settings']['index']): + self.loggit.info('Index %s is mounted for frozen tier', newidx) + else: + report_failure(SearchableSnapshotException( + f'Index {newidx} not a mounted searchable snapshot')) + + def update_aliases(self, current_idx, newidx, aliases): + """ Call :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` to update each new frozen index with the aliases from the old cold-tier index. Verify aliases look good. - + """ + alias_names = aliases.keys() + if not alias_names: + self.loggit.warning('No aliases associated with index %s', current_idx) + else: + self.loggit.debug('Transferring aliases to new index %s', newidx) + self.client.indices.update_aliases( + actions=get_alias_actions(current_idx, newidx, aliases)) + verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys() + if alias_names != verify: + self.loggit.error( + 'Alias names do not match! %s does not match: %s', alias_names, verify) + report_failure(FailedExecution('Aliases failed to transfer to new index')) + + def cleanup(self, current_idx, newidx): + """ Call :py:meth:`~.elasticsearch.client.IndicesClient.delete` to delete the cold tier index. """ + self.loggit.debug('Deleting old index: %s', current_idx) try: - for kwargs in self.action_generator(): - aliases = kwargs.pop('aliases') - current_idx = kwargs.pop('current_idx') - newidx = kwargs['renamed_index'] - # Actually do the mount - self.loggit.debug('Mounting new index %s in frozen tier...', newidx) - self.client.searchable_snapshots.mount(**kwargs) - # Verify it's mounted as a partial now: - self.loggit.debug('Verifying new index %s is mounted properly...', newidx) - idx_settings = self.client.indices.get(index=newidx)[newidx] - if is_idx_partial(idx_settings['settings']['index']): - self.loggit.info('Index %s is mounted for frozen tier', newidx) - else: - raise SearchableSnapshotException( - f'Index {newidx} not a mounted searchable snapshot') - # Update Aliases - alias_names = aliases.keys() - if not alias_names: - self.loggit.warning('No aliases associated with index %s', current_idx) - else: - self.loggit.debug('Transferring aliases to new index %s', newidx) - self.client.indices.update_aliases( - actions=get_alias_actions(current_idx, newidx, aliases)) - verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys() - if alias_names != verify: - self.loggit.error( - 'Alias names do not match! %s does not match: %s', alias_names, verify) - raise FailedExecution('Aliases failed to transfer to new index') - # Clean up old index - self.loggit.debug('Deleting old index: %s', current_idx) - self.client.indices.delete(index=current_idx) - self.loggit.info( - 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx) - + self.client.indices.delete(index=current_idx) # pylint: disable=broad-except except Exception as err: report_failure(err) + self.loggit.info( + 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx) + + def do_action(self): + """ + Do the actions outlined: + + Mount + Verify + Update Aliases + Cleanup + """ + for kwargs in self.action_generator(): + aliases = kwargs.pop('aliases') + current_idx = kwargs.pop('current_idx') + newidx = kwargs['renamed_index'] + + # Mount the index + self.mount_index(newidx, kwargs) + + # Verify it's mounted as a partial now: + self.verify_mount(newidx) + + # Update Aliases + self.update_aliases(current_idx, newidx, aliases) + + # Clean up old index + self.cleanup(current_idx, newidx) diff --git a/curator/actions/create_index.py b/curator/actions/create_index.py index b6ff6240..1ef0563e 100644 --- a/curator/actions/create_index.py +++ b/curator/actions/create_index.py @@ -74,10 +74,14 @@ def do_action(self): # Most likely error is a 400, `resource_already_exists_exception` except RequestError as err: match_list = ["index_already_exists_exception", "resource_already_exists_exception"] - if err.error in match_list and self.ignore_existing: - self.loggit.warning('Index %s already exists.', self.name) + if err.error in match_list: + if self.ignore_existing: + self.loggit.warning('Index %s already exists.', self.name) + else: + raise FailedExecution(f'Index {self.name} already exists.') from err else: - raise FailedExecution(f'Index {self.name} already exists.') from err + msg = f'Unable to create index "{self.name}". Error: {err.error}' + raise FailedExecution(msg) from err # pylint: disable=broad-except except Exception as err: report_failure(err) diff --git a/curator/cli.py b/curator/cli.py index 95ab05dd..b7c7f244 100644 --- a/curator/cli.py +++ b/curator/cli.py @@ -2,8 +2,9 @@ import sys import logging import click -from es_client.defaults import LOGGING_SETTINGS -from es_client.helpers.config import cli_opts, context_settings, get_args, get_client, get_config +from es_client.defaults import OPTION_DEFAULTS +from es_client.helpers.config import ( + cli_opts, context_settings, generate_configdict, get_client, get_config, options_from_dict) from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper, prune_nones from curator.exceptions import ClientException @@ -129,24 +130,17 @@ def process_action(client, action_def, dry_run=False): logger.debug('Doing the action here.') action_def.action_cls.do_action() -def run(client_args, other_args, action_file, dry_run=False): +def run(ctx: click.Context) -> None: """ - Called by :py:func:`cli` to execute what was collected at the command-line + :param ctx: The Click command context - :param client_args: The ClientArgs arguments object - :param other_args: The OtherArgs arguments object - :param action_file: The action configuration file - :param dry_run: Do not perform any changes + :type ctx: :py:class:`Context ` - :type client_args: :py:class:`~.es_client.ClientArgs` - :type other_args: :py:class:`~.es_client.OtherArgs` - :type action_file: str - :type dry_run: bool + Called by :py:func:`cli` to execute what was collected at the command-line """ logger = logging.getLogger(__name__) - - logger.debug('action_file: %s', action_file) - all_actions = ActionsFile(action_file) + logger.debug('action_file: %s', ctx.params['action_file']) + all_actions = ActionsFile(ctx.params['action_file']) for idx in sorted(list(all_actions.actions.keys())): action_def = all_actions.actions[idx] ### Skip to next action if 'disabled' @@ -160,7 +154,7 @@ def run(client_args, other_args, action_file, dry_run=False): # Override the timeout, if specified, otherwise use the default. if action_def.timeout_override: - client_args.request_timeout = action_def.timeout_override + ctx.obj['client_args'].request_timeout = action_def.timeout_override # Create a client object for each action... logger.info('Creating client object and testing connection') @@ -168,8 +162,8 @@ def run(client_args, other_args, action_file, dry_run=False): try: client = get_client(configdict={ 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) + 'client': prune_nones(ctx.obj['client_args'].asdict()), + 'other_settings': prune_nones(ctx.obj['other_args'].asdict()) } }) except ClientException as exc: @@ -187,7 +181,7 @@ def run(client_args, other_args, action_file, dry_run=False): msg = f'Trying Action ID: {idx}, "{action_def.action}": {action_def.description}' try: logger.info(msg) - process_action(client, action_def, dry_run=dry_run) + process_action(client, action_def, dry_run=ctx.params['dry_run']) # pylint: disable=broad-except except Exception as err: exception_handler(action_def, err) @@ -196,31 +190,8 @@ def run(client_args, other_args, action_file, dry_run=False): # pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals, line-too-long @click.command(context_settings=context_settings(), epilog=footer(__version__, tail='command-line.html')) -@click_opt_wrap(*cli_opts('config')) -@click_opt_wrap(*cli_opts('hosts')) -@click_opt_wrap(*cli_opts('cloud_id')) -@click_opt_wrap(*cli_opts('api_token')) -@click_opt_wrap(*cli_opts('id')) -@click_opt_wrap(*cli_opts('api_key')) -@click_opt_wrap(*cli_opts('username')) -@click_opt_wrap(*cli_opts('password')) -@click_opt_wrap(*cli_opts('bearer_auth')) -@click_opt_wrap(*cli_opts('opaque_id')) -@click_opt_wrap(*cli_opts('request_timeout')) -@click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('ca_certs')) -@click_opt_wrap(*cli_opts('client_cert')) -@click_opt_wrap(*cli_opts('client_key')) -@click_opt_wrap(*cli_opts('ssl_assert_hostname')) -@click_opt_wrap(*cli_opts('ssl_assert_fingerprint')) -@click_opt_wrap(*cli_opts('ssl_version')) -@click_opt_wrap(*cli_opts('master-only', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF)) +@options_from_dict(OPTION_DEFAULTS) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) -@click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS)) @click.argument('action_file', type=click.Path(exists=True), nargs=1) @click.version_option(__version__, '-v', '--version', prog_name="curator") @click.pass_context @@ -228,7 +199,7 @@ def cli( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, - dry_run, loglevel, logfile, logformat, action_file + loglevel, logfile, logformat, blacklist, dry_run, action_file ): """ Curator for Elasticsearch indices @@ -244,8 +215,8 @@ def cli( curator_cli -h """ ctx.obj = {} - ctx.obj['dry_run'] = dry_run - cfg = get_config(ctx.params, default_config_file()) - configure_logging(cfg, ctx.params) - client_args, other_args = get_args(ctx.params, cfg) - run(client_args, other_args, action_file, dry_run) + ctx.obj['default_config'] = default_config_file() + get_config(ctx) + configure_logging(ctx) + generate_configdict(ctx) + run(ctx) diff --git a/curator/cli_singletons/alias.py b/curator/cli_singletons/alias.py index f6e77046..803ab99c 100644 --- a/curator/cli_singletons/alias.py +++ b/curator/cli_singletons/alias.py @@ -1,11 +1,10 @@ """Alias Singleton""" import logging import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--name', type=str, help='Alias name', required=True) @click.option( '--add', @@ -51,7 +50,7 @@ def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_ ignore_empty_list = warn_if_no_indices action = CLIAction( ctx.info_name, - ctx.obj['config'], + ctx.obj['configdict'], manual_options, [], # filter_list is empty in our case ignore_empty_list, diff --git a/curator/cli_singletons/allocation.py b/curator/cli_singletons/allocation.py index 1ede28bc..260d8a65 100644 --- a/curator/cli_singletons/allocation.py +++ b/curator/cli_singletons/allocation.py @@ -1,10 +1,9 @@ """Allocation Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--key', type=str, required=True, help='Node identification tag') @click.option('--value', type=str, default=None, help='Value associated with --key') @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude'])) @@ -72,5 +71,5 @@ def allocation( } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/close.py b/curator/cli_singletons/close.py index 8039b50c..c6d11d3f 100644 --- a/curator/cli_singletons/close.py +++ b/curator/cli_singletons/close.py @@ -1,10 +1,9 @@ """Close Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed') @click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed') @click.option( @@ -36,5 +35,5 @@ def close(ctx, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/delete.py b/curator/cli_singletons/delete.py index 57deedd9..e43546d3 100644 --- a/curator/cli_singletons/delete.py +++ b/curator/cli_singletons/delete.py @@ -1,11 +1,10 @@ """Delete Index and Delete Snapshot Singletons""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json #### Indices #### -@click.command(context_settings=context_settings()) +@click.command() @click.option( '--ignore_empty_list', is_flag=True, @@ -31,7 +30,7 @@ def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'delete_indices', - ctx.obj['config'], + ctx.obj['configdict'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list @@ -39,7 +38,7 @@ def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): action.do_singleton_action(dry_run=ctx.obj['dry_run']) #### Snapshots #### -@click.command(context_settings=context_settings()) +@click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository name') @click.option('--retry_count', type=int, help='Number of times to retry (max 3)') @click.option('--retry_interval', type=int, help='Time in seconds between retries') @@ -76,7 +75,7 @@ def delete_snapshots( # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'delete_snapshots', - ctx.obj['config'], + ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, diff --git a/curator/cli_singletons/forcemerge.py b/curator/cli_singletons/forcemerge.py index 95ff1211..ed6cb5bd 100644 --- a/curator/cli_singletons/forcemerge.py +++ b/curator/cli_singletons/forcemerge.py @@ -1,10 +1,9 @@ """ForceMerge Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option( '--max_num_segments', type=int, @@ -44,5 +43,5 @@ def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indice } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/object_class.py b/curator/cli_singletons/object_class.py index 622fd7a2..900282e3 100644 --- a/curator/cli_singletons/object_class.py +++ b/curator/cli_singletons/object_class.py @@ -45,10 +45,10 @@ class CLIAction(): """ Unified class for all CLI singleton actions """ - def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_list, **kwargs): + def __init__(self, action, configdict, option_dict, filter_list, ignore_empty_list, **kwargs): """Class setup :param action: The action name. - :param client_args: ``dict`` containing everything needed for + :param configdict: ``dict`` containing everything needed for :py:class:`~.es_client.builder.Builder` to build an :py:class:`~.elasticsearch.Elasticsearch` client object. :param option_dict: Options for ``action``. @@ -57,7 +57,7 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l :param kwargs: Other keyword args to pass to ``action``. :type action: str - :type client_args: dict + :type configdict: dict :type option_dict: dict :type filter_list: list :type ignore_empty_list: bool @@ -102,9 +102,8 @@ def __init__(self, action, client_args, option_dict, filter_list, ignore_empty_l self.logger.debug('rollover option_dict = %s', option_dict) else: self.check_filters(filter_list) - try: - builder = Builder(configdict=client_args) + builder = Builder(configdict=configdict) builder.connect() # pylint: disable=broad-except except Exception as exc: @@ -212,13 +211,10 @@ def do_singleton_action(self, dry_run=False): self.do_filters() self.logger.debug('OPTIONS = %s', self.options) action_obj = self.action_class(self.list_object, **self.options) - try: - if dry_run: - action_obj.do_dry_run() - else: - action_obj.do_action() - except Exception as exc: - raise Exception from exc # pass it on? + if dry_run: + action_obj.do_dry_run() + else: + action_obj.do_action() # pylint: disable=broad-except except Exception as exc: self.logger.critical( diff --git a/curator/cli_singletons/open_indices.py b/curator/cli_singletons/open_indices.py index 4ba94750..61faa31d 100644 --- a/curator/cli_singletons/open_indices.py +++ b/curator/cli_singletons/open_indices.py @@ -1,10 +1,9 @@ """Open (closed) Index Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(name='open', context_settings=context_settings()) +@click.command(name='open') @click.option( '--ignore_empty_list', is_flag=True, @@ -30,7 +29,7 @@ def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( ctx.info_name, - ctx.obj['config'], + ctx.obj['configdict'], {'allow_ilm_indices':allow_ilm_indices}, filter_list, ignore_empty_list diff --git a/curator/cli_singletons/replicas.py b/curator/cli_singletons/replicas.py index 9ae792d2..70d039f0 100644 --- a/curator/cli_singletons/replicas.py +++ b/curator/cli_singletons/replicas.py @@ -1,10 +1,9 @@ """Change Replica Count Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json -@click.command(context_settings=context_settings()) +@click.command() @click.option('--count', type=int, required=True, help='Number of replicas (max 10)') @click.option( '--wait_for_completion/--no-wait_for_completion', @@ -41,5 +40,5 @@ def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indic } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/restore.py b/curator/cli_singletons/restore.py index 8ff4921b..313fb5db 100644 --- a/curator/cli_singletons/restore.py +++ b/curator/cli_singletons/restore.py @@ -1,11 +1,10 @@ """Snapshot Restore Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository') @click.option('--name', type=str, help='Snapshot name', required=False, default=None) @click.option('--index', multiple=True, help='Index name to restore. (Can invoke repeatedly for multiple indices)') @@ -53,7 +52,7 @@ def restore( # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( ctx.info_name, - ctx.obj['config'], + ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list, diff --git a/curator/cli_singletons/rollover.py b/curator/cli_singletons/rollover.py index dd1cc94e..7c6539b6 100644 --- a/curator/cli_singletons/rollover.py +++ b/curator/cli_singletons/rollover.py @@ -1,12 +1,11 @@ """Index Rollover Singleton""" import click -from es_client.helpers.config import context_settings from es_client.helpers.utils import prune_nones from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--name', type=str, help='Alias name', required=True) @click.option('--max_age', type=str, help='max_age condition value (see documentation)') @click.option('--max_docs', type=str, help='max_docs condition value (see documentation)') @@ -35,7 +34,7 @@ def rollover( } # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( - ctx.info_name, ctx.obj['config'], manual_options, [], True, + ctx.info_name, ctx.obj['configdict'], manual_options, [], True, extra_settings=extra_settings, new_index=new_index, wait_for_active_shards=wait_for_active_shards diff --git a/curator/cli_singletons/show.py b/curator/cli_singletons/show.py index 51b5e72a..3751b0f7 100644 --- a/curator/cli_singletons/show.py +++ b/curator/cli_singletons/show.py @@ -1,7 +1,6 @@ """Show Index/Snapshot Singletons""" from datetime import datetime import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json from curator.helpers.getters import byte_size @@ -11,7 +10,7 @@ #### Indices #### # pylint: disable=line-too-long -@click.command(context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) +@click.command(epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) @click.option('--verbose', help='Show verbose output.', is_flag=True, show_default=True) @click.option('--header', help='Print header if --verbose', is_flag=True, show_default=True) @click.option('--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True) @@ -26,7 +25,7 @@ def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indic # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'show_indices', - ctx.obj['config'], + ctx.obj['configdict'], {'allow_ilm_indices': allow_ilm_indices}, filter_list, ignore_empty_list @@ -83,7 +82,7 @@ def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indic #### Snapshots #### # pylint: disable=line-too-long -@click.command(context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) +@click.command(epilog=footer(__version__, tail='singleton-cli.html#_show_indicessnapshots')) @click.option('--repository', type=str, required=True, help='Snapshot repository name') @click.option('--ignore_empty_list', is_flag=True, help='Do not raise exception if there are no actionable snapshots') @click.option('--filter_list', callback=validate_filter_json, default='{"filtertype":"none"}', help='JSON string representing an array of filters.') @@ -95,7 +94,7 @@ def show_snapshots(ctx, repository, ignore_empty_list, filter_list): # ctx.info_name is the name of the function or name specified in @click.command decorator action = CLIAction( 'show_snapshots', - ctx.obj['config'], + ctx.obj['configdict'], {}, filter_list, ignore_empty_list, diff --git a/curator/cli_singletons/shrink.py b/curator/cli_singletons/shrink.py index 0b209800..79653143 100644 --- a/curator/cli_singletons/shrink.py +++ b/curator/cli_singletons/shrink.py @@ -1,11 +1,10 @@ """Shrink Index Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import json_to_dict, validate_filter_json # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--shrink_node', default='DETERMINISTIC', type=str, help='Named node, or DETERMINISTIC', show_default=True) @click.option('--node_filters', help='JSON version of node_filters (see documentation)', callback=json_to_dict) @click.option('--number_of_shards', default=1, type=int, help='Shrink to this many shards per index') @@ -53,5 +52,5 @@ def shrink( 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator - action = CLIAction(ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + action = CLIAction(ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/cli_singletons/snapshot.py b/curator/cli_singletons/snapshot.py index 7ed26b1a..ad0e69fc 100644 --- a/curator/cli_singletons/snapshot.py +++ b/curator/cli_singletons/snapshot.py @@ -1,11 +1,10 @@ """Snapshot Singleton""" import click -from es_client.helpers.config import context_settings from curator.cli_singletons.object_class import CLIAction from curator.cli_singletons.utils import validate_filter_json # pylint: disable=line-too-long -@click.command(context_settings=context_settings()) +@click.command() @click.option('--repository', type=str, required=True, help='Snapshot repository') @click.option('--name', type=str, help='Snapshot name', show_default=True, default='curator-%Y%m%d%H%M%S') @click.option('--ignore_unavailable', is_flag=True, show_default=True, help='Ignore unavailable shards/indices.') @@ -40,5 +39,5 @@ def snapshot( 'allow_ilm_indices': allow_ilm_indices, } # ctx.info_name is the name of the function or name specified in @click.command decorator - action = CLIAction(ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list) + action = CLIAction(ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list) action.do_singleton_action(dry_run=ctx.obj['dry_run']) diff --git a/curator/helpers/testers.py b/curator/helpers/testers.py index 9147ea36..7109b8cf 100644 --- a/curator/helpers/testers.py +++ b/curator/helpers/testers.py @@ -301,8 +301,7 @@ def verify_client_object(test): """ logger = logging.getLogger(__name__) # Ignore mock type for testing - if str(type(test)) == "" or \ - str(type(test)) == "": + if str(type(test)) == "": pass elif not isinstance(test, Elasticsearch): msg = f'Not a valid client object. Type: {type(test)} was passed' diff --git a/curator/repomgrcli.py b/curator/repomgrcli.py index bc320b44..c2c12f36 100644 --- a/curator/repomgrcli.py +++ b/curator/repomgrcli.py @@ -6,7 +6,7 @@ from elasticsearch8 import ApiError, NotFoundError from es_client.defaults import LOGGING_SETTINGS, SHOW_OPTION from es_client.builder import Builder -from es_client.helpers.config import cli_opts, context_settings, get_config, get_args +from es_client.helpers.config import cli_opts, context_settings, generate_configdict, get_config from es_client.helpers.logging import configure_logging from es_client.helpers.utils import option_wrapper, prune_nones from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer @@ -57,7 +57,7 @@ def get_client(ctx): :returns: A client connection object :rtype: :py:class:`~.elasticsearch.Elasticsearch` """ - builder = Builder(configdict=ctx.obj['esconfig']) + builder = Builder(configdict=ctx.obj['configdict']) try: builder.connect() # pylint: disable=broad-except @@ -350,21 +350,13 @@ def repo_mgr_cli( es_repo_mgr show-all-options """ ctx.obj = {} - # Ensure a passable ctx object - ctx.ensure_object(dict) ctx.obj['dry_run'] = dry_run - cfg = get_config(ctx.params, default_config_file()) - configure_logging(cfg, ctx.params) + ctx.obj['default_config'] = default_config_file() + get_config(ctx) + configure_logging(ctx) logger = logging.getLogger('curator.repomgrcli') - client_args, other_args = get_args(ctx.params, cfg) - ctx.obj['esconfig'] = { - 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) - } - } + generate_configdict(ctx) logger.debug('Exiting initial command function...') - @repo_mgr_cli.command( 'show-all-options', diff --git a/curator/singletons.py b/curator/singletons.py index 4828deae..a531f7be 100644 --- a/curator/singletons.py +++ b/curator/singletons.py @@ -1,9 +1,10 @@ """CLI module for curator_cli""" import click -from es_client.defaults import LOGGING_SETTINGS, SHOW_OPTION -from es_client.helpers.config import cli_opts, context_settings, get_config, get_args +from es_client.defaults import SHOW_EVERYTHING +from es_client.helpers.config import ( + cli_opts, context_settings, generate_configdict, get_config, options_from_dict) from es_client.helpers.logging import configure_logging -from es_client.helpers.utils import option_wrapper, prune_nones +from es_client.helpers.utils import option_wrapper from curator.defaults.settings import CLICK_DRYRUN, default_config_file, footer from curator._version import __version__ from curator.cli_singletons import ( @@ -12,44 +13,20 @@ ) from curator.cli_singletons.show import show_indices, show_snapshots -ONOFF = {'on': '', 'off': 'no-'} click_opt_wrap = option_wrapper() # pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals @click.group( context_settings=context_settings(), epilog=footer(__version__, tail='singleton-cli.html')) -@click_opt_wrap(*cli_opts('config')) -@click_opt_wrap(*cli_opts('hosts')) -@click_opt_wrap(*cli_opts('cloud_id')) -@click_opt_wrap(*cli_opts('api_token')) -@click_opt_wrap(*cli_opts('id')) -@click_opt_wrap(*cli_opts('api_key')) -@click_opt_wrap(*cli_opts('username')) -@click_opt_wrap(*cli_opts('password')) -@click_opt_wrap(*cli_opts('bearer_auth', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('opaque_id', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('request_timeout')) -@click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF, override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF)) -@click_opt_wrap(*cli_opts('ca_certs')) -@click_opt_wrap(*cli_opts('client_cert')) -@click_opt_wrap(*cli_opts('client_key')) -@click_opt_wrap(*cli_opts('ssl_assert_hostname', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('ssl_assert_fingerprint', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('ssl_version', override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('master-only', onoff=ONOFF, override=SHOW_OPTION)) -@click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF, override=SHOW_OPTION)) +@options_from_dict(SHOW_EVERYTHING) @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN)) -@click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS)) -@click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS)) @click.version_option(__version__, '-v', '--version', prog_name='curator_cli') @click.pass_context def curator_cli( ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth, opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key, ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test, - dry_run, loglevel, logfile, logformat + loglevel, logfile, logformat, blacklist, dry_run ): """ Curator CLI (Singleton Tool) @@ -63,16 +40,10 @@ def curator_cli( """ ctx.obj = {} ctx.obj['dry_run'] = dry_run - cfg = get_config(ctx.params, default_config_file()) - configure_logging(cfg, ctx.params) - client_args, other_args = get_args(ctx.params, cfg) - final_config = { - 'elasticsearch': { - 'client': prune_nones(client_args.asdict()), - 'other_settings': prune_nones(other_args.asdict()) - } - } - ctx.obj['config'] = final_config + ctx.obj['default_config'] = default_config_file() + get_config(ctx) + configure_logging(ctx) + generate_configdict(ctx) # Add the subcommands curator_cli.add_command(alias) diff --git a/docs/Changelog.rst b/docs/Changelog.rst index 185bfc44..eb2c6683 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,44 @@ Changelog ========= +8.0.11 (20 March 2024) +---------------------- + +**Announcement** + + * With the advent of ``es_client==8.12.5``, environment variables can now be used to automatically + populate command-line options. The ``ESCLIENT_`` prefix just needs to prepend the capitalized + option name, and any hyphens need to be replaced by underscores. ``--http-compress True`` is + automatically settable by having ``ESCLIENT_HTTP_COMPRESS=1``. Boolean values are 1, 0, True, + or False (case-insensitive). Options like ``hosts`` which can have multiple values just need to + have whitespace between the values, e.g. + ``ESCLIENT_HOSTS='http://127.0.0.1:9200 http://localhost:9200'``. It splits perfectly. This is + tremendous news for the containerization/k8s community. You won't have to have all of the + options spelled out any more. Just have the environment variables assigned. + * Also, log blacklisting has made it to the command-line as well. It similarly can be set via + environment variable, e.g. ``ESCLIENT_BLACKLIST='elastic_transport urllib3'``, or by multiple + ``--blacklist`` entries at the command line. + * ``es_client`` has simplified things such that I can clean up arg sprawl in the command line + scripts. + +**Changes** + +Lots of pending pull requests have been merged. Thank you to the community +members who took the time to contribute to Curator's code. + + * DOCFIX - Update date math section to use ``y`` instead of ``Y`` (#1510) + * DOCFIX - Update period filtertype description (#1550) + * add .dockerignore to increase build speed (#1604) + * DOCFIX - clarification on prefix and suffix kinds (#1558) + The provided documentation was adapted and edited. + * Use builtin unittest.mock (#1695) + * Had to also update ``helpers.testers.verify_client_object``. + * Display proper error when mapping incorrect (#1526) - @namreg + Also assisting with this is @alexhornblake in #1537 + Apologies for needing to adapt the code manually since it's been so long. + * Version bumps: + * ``es_client==8.12.6`` + 8.0.10 (1 February 2024) ------------------------ diff --git a/docs/asciidoc/inc_kinds.asciidoc b/docs/asciidoc/inc_kinds.asciidoc index f9a8fcd6..7e7bcb9d 100644 --- a/docs/asciidoc/inc_kinds.asciidoc +++ b/docs/asciidoc/inc_kinds.asciidoc @@ -21,7 +21,8 @@ To match all indices _except_ those starting with `logstash-`: exclude: True ------------- -Note: Internally _regex_ pattern is constructed from prefix value as `^{0}.*$`. Special characters should be escaped with backslash to match literally. +NOTE: Internally, the `prefix` value is used to create a _regex_ pattern: `^{0}.*$`. Any special +characters should be escaped with a backslash to match literally. === suffix @@ -44,7 +45,8 @@ To match all indices _except_ those ending with `-prod`: exclude: True ------------- -Note: Internally _regex_ pattern is constructed from suffix value as `^.*{0}$`. Specials character should be escaped with backslash to match literally. +NOTE: Internally, the `suffix` value is used to create a _regex_ pattern: `^.*{0}$`. Any special +characters should be escaped with a backslash to match literally. === timestring diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index 75913364..4cf1819f 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,7 +1,7 @@ -:curator_version: 8.0.9 +:curator_version: 8.0.11 :curator_major: 8 :curator_doc_tree: 8.0 -:es_py_version: 8.12.0 +:es_py_version: 8.12.1 :es_doc_tree: 8.12 :stack_doc_tree: 8.12 :pybuild_ver: 3.11.7 diff --git a/pyproject.toml b/pyproject.toml index cf1b5776..6b7b7f43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ keywords = [ 'index-expiry' ] dependencies = [ - "es_client==8.12.4" + "es_client==8.12.6" ] [project.optional-dependencies] @@ -94,7 +94,7 @@ run = "run-coverage --no-cov" [[tool.hatch.envs.test.matrix]] python = ["3.9", "3.10", "3.11"] -version = ["8.0.9"] +version = ["8.0.11"] [tool.pytest.ini_options] pythonpath = [".", "curator"] diff --git a/tests/integration/test_alias.py b/tests/integration/test_alias.py index 915aeeed..ecf6468b 100644 --- a/tests/integration/test_alias.py +++ b/tests/integration/test_alias.py @@ -1,11 +1,13 @@ """Test Alias action""" # pylint: disable=missing-function-docstring, missing-class-docstring, invalid-name, line-too-long import os +import logging from datetime import datetime, timedelta import pytest from elasticsearch8.exceptions import NotFoundError from . import CuratorTestCase from . import testvars +LOGGER = logging.getLogger('test_alias') HOST = os.environ.get('TEST_ES_SERVER', 'http://127.0.0.1:9200') @@ -231,9 +233,10 @@ def test_warn_if_no_indices(self): '--remove', '{"filtertype":"pattern","kind":"prefix","value":"my"}', '--warn_if_no_indices' ] - assert 0 == self.run_subprocess(args, logname='TestCLIAlias.test_warn_if_no_indices') + LOGGER.debug('ARGS = %s', args) + assert 0 == self.run_subprocess(args) expected = {idx1: {'aliases': {alias: {}}}, idx2: {'aliases': {alias: {}}}} - assert expected == self.client.indices.get_alias(name=alias) + assert expected == dict(self.client.indices.get_alias(name=alias)) def test_exit_1_on_empty_list(self): """test_exit_1_on_empty_list""" alias = 'testalias' @@ -246,4 +249,4 @@ def test_exit_1_on_empty_list(self): '--add', '{"filtertype":"pattern","kind":"prefix","value":"dum","exclude":false}', '--remove', '{"filtertype":"pattern","kind":"prefix","value":"my","exclude":false}', ] - assert 1 == self.run_subprocess(args, logname='TestCLIAlias.test_warn_if_no_indices') \ No newline at end of file + assert 1 == self.run_subprocess(args, logname='TestCLIAlias.test_exit_1_on_empty_list') \ No newline at end of file diff --git a/tests/integration/test_create_index.py b/tests/integration/test_create_index.py index 25875c75..51456570 100644 --- a/tests/integration/test_create_index.py +++ b/tests/integration/test_create_index.py @@ -85,3 +85,25 @@ def test_already_existing_pass(self): self.invoke_runner() assert [idx] == get_indices(self.client) assert 0 == self.result.exit_code + def test_incorrect_mapping_fail_with_propper_error(self): + config = ( + '---\n' + 'actions:\n' + ' 1:\n' + ' description: "Create index as named"\n' + ' action: create_index\n' + ' options:\n' + ' name: {0}\n' + ' extra_settings:\n' + ' mappings:\n' + ' properties:\n' + ' name: ["test"]\n' + ) + idx = 'testing' + self.write_config( + self.args['configfile'], testvars.none_logging_config.format(HOST)) + self.write_config(self.args['actionfile'], config.format(idx)) + self.invoke_runner() + assert not get_indices(self.client) + assert 'mapper_parsing_exception' in self.result.stdout + assert 1 == self.result.exit_code diff --git a/tests/integration/test_delete_indices.py b/tests/integration/test_delete_indices.py index 22c3541d..bbf1775e 100644 --- a/tests/integration/test_delete_indices.py +++ b/tests/integration/test_delete_indices.py @@ -332,4 +332,4 @@ def test_name_older_than_now_cli(self): '--filter_list', '{"filtertype":"age","source":"name","direction":"older","timestring":"%Y.%m.%d","unit":"days","unit_count":5}', ] self.assertEqual(0, self.run_subprocess(args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli')) - self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) \ No newline at end of file + self.assertEqual(5, len(exclude_ilm_history(get_indices(self.client)))) diff --git a/tests/integration/testvars.py b/tests/integration/testvars.py index dea4d11b..c7c7b0e9 100644 --- a/tests/integration/testvars.py +++ b/tests/integration/testvars.py @@ -453,6 +453,10 @@ ' continue_if_exception: False\n' ' disable_action: False\n' ' filters:\n' +' - filtertype: pattern\n' +' kind: prefix\n' +' value: ilm-history-\n' +' exclude: True\n' ' - filtertype: {0}\n' ' source: {1}\n' ' range_from: {2}\n'