From ddd15968753123644b055cd7e5471fbe78c1facc Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Tue, 10 Dec 2019 09:42:54 +0100 Subject: [PATCH] Add track parameter for query index pattern With this commit we introduce the track parameter `query_index_prefix` which determines the prefix of indices which are used in query benchmarks and set its default to the former value `elasticlogs_q`. Relates #56 --- README.md | 5 ++--- .../combined-indexing-and-querying.json | 2 +- .../challenges/elasticlogs-1bn-load.json | 6 +++--- ...lasticlogs-continuous-index-and-query.json | 10 +++++----- .../challenges/elasticlogs-querying.json | 2 +- .../challenges/max-indexing-querying.json | 6 +++--- eventdata/elasticlogs-index-template.json | 2 +- eventdata/operations/indexing.json | 18 ++++++++--------- eventdata/operations/querying.json | 20 +++++++++---------- eventdata/operations/stats.json | 6 +++--- eventdata/track.json | 6 +++++- 11 files changed, 42 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 0025de5..b038ccc 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Note: In general, track parameters are only defined for a subset of the challeng | Parameter | Explanation | Type | Default Value | | --------- | ----------- | ---- | ------------- | | `record_raw_event_size` | Adds a new field `_raw_event_size` to the index which contains the size of the raw logging event in bytes. | `bool` | `False` | +| `query_index_prefix` | Start of the index name(s) used in queries for this track. | `str` | `elasticlogs_q` | Note: It is recommended to store any track parameters in a json file and pass them to Rally using `--track-params=./params-file.json`. @@ -126,7 +127,6 @@ The table below shows the track parameters that can be adjusted along with defau | `p2_bulk_indexing_clients` | Number of [clients](https://esrally.readthedocs.io/en/stable/track.html?highlight=number%20of%20clients#schedule) used to index during phase 2 | `int` | `16` | | `p2_bulk_size` | The [build-size](https://esrally.readthedocs.io/en/stable/track.html?highlight=number%20of%20clients#bulk) for the autogenerated events during phase 2 | `int` | `1000` | | `p2_ops` | Number of bulk indexing ops/s for phase 2. A value of `10` with `p2_bulk_size=10` throttles indexing to 10000 docs/s | `int` | `10` | -| `index_alias` | Specifies default index alias. | `str` | `elasticlogs_q_write` | | `rollover_max_size` | Max index size condition for [rollover API](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html#indices-rollover-index) | `str` | `30gb` | | `rollover_max_age` | Max age condition for [rollover API](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html#indices-rollover-index) | `str` | `1d` | | `p2_query1_target_interval` | Frequency of execution (every N sec) of Kibana query: `kibana-traffic-country-dashboard_60m` | `int` | `30` | @@ -134,10 +134,9 @@ The table below shows the track parameters that can be adjusted along with defau | `p2_query3_target_interval` | Frequency of execution (every N sec) of Kibana query: `kibana-traffic-dashboard_30m` | `int` | `30` | | `p2_query4_target_interval` | Frequency of execution (every N sec) of Kibana query: `kibana-content_issues-dashboard_30m"` | `int` | `30` | | `max_rolledover_indices` | Max amount of recently rolled over indices to retain | `int` | `20` | -| `indices_delete_pattern` | pattern to use for matching and deleting old rolled over indices. See also suffix_separator. | `str` | `elasticlogs_q-*` | | `rolledover_indices_suffix_separator` | Separator for extracting suffix to help determining which rolled-over indices to delete | `str` | `-` | -The indices use the alias `elasticlogs_q_write` and start with `elasticlogs_q-000001`. As an example, for a cluster with rolled over indices: `elasticlogs-000001`, `elasticlogs-000002`, ... `000010` a value of `max_rolledover_indices=8` results to the removal of `elasticlogs-000001` and `elasticlogs-000002`. +The indices use the alias `elasticlogs_q_write` (assuming the default value for `p_query_index_prefix`) and start with `elasticlogs_q-000001`. As an example, for a cluster with rolled over indices: `elasticlogs-000001`, `elasticlogs-000002`, ... `000010` a value of `max_rolledover_indices=8` results to the removal of `elasticlogs-000001` and `elasticlogs-000002`. A value of `max_rolledover_indices=20` on a three node bare-metal cluster with the following specifications: diff --git a/eventdata/challenges/combined-indexing-and-querying.json b/eventdata/challenges/combined-indexing-and-querying.json index a2627e5..fb92ac9 100644 --- a/eventdata/challenges/combined-indexing-and-querying.json +++ b/eventdata/challenges/combined-indexing-and-querying.json @@ -8,7 +8,7 @@ { "name": "combined-indexing-and-querying", - "description": "This challenge simulates a set of Kibana queries against historical data (elasticlogs_q-* indices) as well as against the most recent data currently being indexed. It combined this with rate-limited indexing at varying levels. It assumes one of the challenges creating elasticlogs_q-* indices has been run.", + "description": "This challenge simulates a set of Kibana queries against historical data ({{p_query_index_pattern}} indices) as well as against the most recent data currently being indexed. It combined this with rate-limited indexing at varying levels. It assumes one of the challenges creating {{p_query_index_pattern}} indices has been run.", "meta": { "benchmark_type": "indexing/querying", "target_kibana_queries_per_minute": 7 diff --git a/eventdata/challenges/elasticlogs-1bn-load.json b/eventdata/challenges/elasticlogs-1bn-load.json index 0e76440..eeb2d87 100644 --- a/eventdata/challenges/elasticlogs-1bn-load.json +++ b/eventdata/challenges/elasticlogs-1bn-load.json @@ -4,7 +4,7 @@ { "name": "elasticlogs-1bn-load", - "description": "Indexes 1bn (default) documents into elasticlogs_q-* indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.", + "description": "Indexes 1bn (default) documents into {{p_query_index_pattern}} indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.", "default": true, "meta": { "client_count": {{ p_bulk_indexing_clients }}, @@ -23,10 +23,10 @@ { "operation": { "operation-type": "create-index", - "index": "elasticlogs_q-000001", + "index": "{{p_query_index_prefix}}-000001", "body": { "aliases" : { - "elasticlogs_q_write" : {} + "{{p_query_index_write_alias}}" : {} } } } diff --git a/eventdata/challenges/elasticlogs-continuous-index-and-query.json b/eventdata/challenges/elasticlogs-continuous-index-and-query.json index ece6e26..c8d4994 100644 --- a/eventdata/challenges/elasticlogs-continuous-index-and-query.json +++ b/eventdata/challenges/elasticlogs-continuous-index-and-query.json @@ -8,7 +8,7 @@ {% set p2_rate = (p2_ops * (p2_bulk_size | default(1000))) %} { "name": "elasticlogs-continuous-index-and-query", - "description": "Indexes 1bn (default) documents into elasticlogs_q-* indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.", + "description": "Indexes 1bn (default) documents into {{p_query_index_pattern}} indices. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.", "meta": { "benchmark_type": "indexing" }, @@ -26,10 +26,10 @@ "operation": { "name": "create_elasticlogs_q_write", "operation-type": "create-index", - "index": "elasticlogs_q-000001", + "index": "{{p_query_index_prefix}}-000001", "body": { "aliases" : { - "elasticlogs_q_write" : {} + "{{p_query_index_write_alias}}" : {} } } } @@ -43,7 +43,7 @@ "name": "index-append-elasticlogs_q_write-phase1", "operation": { "operation-type": "bulk", - "index": "elasticlogs_q_write", + "index": "{{p_query_index_write_alias}}", "param-source": "elasticlogs_bulk", "bulk-size": {{ p1_bulk_size | default(1000) | int }}, "record_raw_event_size": {{p_record_raw_event_size}} @@ -79,7 +79,7 @@ "name": "index-append-elasticlogs_q_write-phase2", "operation": { "operation-type": "bulk", - "index": "elasticlogs_q_write", + "index": "{{p_query_index_write_alias}}", "param-source": "elasticlogs_bulk", "bulk-size": {{ p2_bulk_size | default(1000) | int }}, "record_raw_event_size": {{p_record_raw_event_size}} diff --git a/eventdata/challenges/elasticlogs-querying.json b/eventdata/challenges/elasticlogs-querying.json index f480e1a..182e27c 100644 --- a/eventdata/challenges/elasticlogs-querying.json +++ b/eventdata/challenges/elasticlogs-querying.json @@ -1,6 +1,6 @@ { "name": "elasticlogs-querying", - "description": "This challenge simulates a set of Kibana queries against historical data (elasticlogs_q-* indices) without any indexing taking place for a period of 30 minutes. It assumes one of the challenges creating elasticlogs_q-* indices has been run.", + "description": "This challenge simulates a set of Kibana queries against historical data ({{p_query_index_pattern}} indices) without any indexing taking place for a period of 30 minutes. It assumes one of the challenges creating {{p_query_index_pattern}} indices has been run.", "meta": { "benchmark_type": "querying", "target_kibana_queries_per_minute": 5 diff --git a/eventdata/challenges/max-indexing-querying.json b/eventdata/challenges/max-indexing-querying.json index 29a82e2..e33042e 100644 --- a/eventdata/challenges/max-indexing-querying.json +++ b/eventdata/challenges/max-indexing-querying.json @@ -3,7 +3,7 @@ { "name": "max-indexing-querying", - "description": "Indexes documents as quickly as possible into elasticlogs_q-* indices, initially for 2 hiours without concurrent querying and then for 2 hours with querying. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.", + "description": "Indexes documents as quickly as possible into {{p_query_index_pattern}} indices, initially for 2 hours without concurrent querying and then for 2 hours with querying. IDs are autogenerated by Elasticsearch, meaning there are no conflicts.", "meta": { "client_count": {{ p_bulk_indexing_clients }}, "benchmark_type": "indexing" @@ -21,10 +21,10 @@ { "operation": { "operation-type": "create-index", - "index": "elasticlogs_q-000001", + "index": "{{p_query_index_prefix}}-000001", "body": { "aliases" : { - "elasticlogs_q_write" : {} + "{{p_query_index_write_alias}}" : {} } } } diff --git a/eventdata/elasticlogs-index-template.json b/eventdata/elasticlogs-index-template.json index 63adcd0..4b15716 100644 --- a/eventdata/elasticlogs-index-template.json +++ b/eventdata/elasticlogs-index-template.json @@ -1,7 +1,7 @@ {% set p_disk_type = disk_type | default('ssd') | lower %} {% set p_translog_sync = translog_sync | default('request') | lower %} { - "index_patterns": ["elasticlogs*"], + "index_patterns": ["{{p_index_prefix}}*"], "settings": { "index.refresh_interval": "5s", "index.codec": "best_compression", diff --git a/eventdata/operations/indexing.json b/eventdata/operations/indexing.json index 80b7e6d..6d59b92 100644 --- a/eventdata/operations/indexing.json +++ b/eventdata/operations/indexing.json @@ -1,8 +1,6 @@ {# The following track-params are supported: -index_alias: used by the `rollover_custom_alias` operation. Defaults to `elasticlogs_q_write`. rollover_max_age: used by the `rollover_custom_alias` operation. Defaults to `1d`. rollover_max_size: used by the `rollover_custom_alias` operation. Defaults to `30gb`. -indices_delete_pattern: used by the `delete_rolledover_index_pattern` operation. Defaults to `elasticlogs_q-*`. max_rolledover_indices: used by the `delete_rolledover_index_pattern` operation. Defaults to `20`. rolledover_indices_suffix_separator: used by the `delete_rolledover_index_pattern` operation. Defaults to `-`. #} @@ -33,7 +31,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter "name": "index-append-1000-elasticlogs_q_write", "operation-type": "bulk", "param-source": "elasticlogs_bulk", - "index": "elasticlogs_q_write", + "index": "{{p_query_index_write_alias}}", "bulk-size": 1000, "record_raw_event_size": {{p_record_raw_event_size}} }, @@ -48,7 +46,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter { "name": "rollover_elasticlogs_q_write_100M", "operation-type": "rollover", - "alias": "elasticlogs_q_write", + "alias": "{{p_query_index_write_alias}}", "body": { "conditions": { "max_age": "1d", @@ -70,7 +68,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter { "name": "rollover_elasticlogs_q_write_50M", "operation-type": "rollover", - "alias": "elasticlogs_q_write", + "alias": "{{p_query_index_write_alias}}", "body": { "conditions": { "max_age": "1d", @@ -92,7 +90,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter { "name": "rollover_elasticlogs_q_write_10M", "operation-type": "rollover", - "alias": "elasticlogs_q_write", + "alias": "{{p_query_index_write_alias}}", "body": { "conditions": { "max_age": "1d", @@ -114,7 +112,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter { "name": "rollover_elasticlogs_q_write_50gb", "operation-type": "rollover", - "alias": "elasticlogs_q_write", + "alias": "{{p_query_index_write_alias}}", "body": { "conditions": { "max_age": "1d", @@ -125,7 +123,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter { "name": "rollover_custom_alias", "operation-type": "rollover", - "alias": {{ index_alias | default("elasticlogs_q_write") | tojson }}, + "alias": "{{p_query_index_write_alias}}", "body": { "conditions": { "max_age": {{ rollover_max_age | default("1d") | tojson }}, @@ -141,7 +139,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter { "name": "deleteindex_elasticlogs_q-*", "operation-type": "delete-index", - "index": "elasticlogs_q-*" + "index": "{{p_query_index_pattern}}" }, { "name": "deleteindex_elasticlogs", @@ -151,7 +149,7 @@ rolledover_indices_suffix_separator: used by the `delete_rolledover_index_patter { "name": "delete_rolledover_index_pattern", "operation-type": "delete_indices", - "index_pattern": {{ indices_delete_pattern | default("elasticlogs_q-*") | tojson }}, + "index_pattern": "{{ p_query_index_pattern }}", "max_indices": {{ max_rolledover_indices | default(20) | int | tojson }}, "suffix_separator": {{ rolledover_indices_suffix_separator | default("-") | tojson }} } diff --git a/eventdata/operations/querying.json b/eventdata/operations/querying.json index f71beed..916a62c 100644 --- a/eventdata/operations/querying.json +++ b/eventdata/operations/querying.json @@ -103,7 +103,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "traffic", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+25%,END", "window_length": "25%" @@ -113,7 +113,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "content_issues", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+25%,END", "window_length": "25%" @@ -123,7 +123,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "traffic", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+50%,END", "window_length": "50%" @@ -133,7 +133,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "content_issues", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+50%,END", "window_length": "50%" @@ -143,7 +143,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "traffic", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+75%,END", "window_length": "75%" @@ -153,7 +153,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "content_issues", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+75%,END", "window_length": "75%" @@ -163,7 +163,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "traffic", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": ["*"], "window_end": "now", "window_length": "30m" @@ -173,7 +173,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "discover", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+50%,END", "window_length": "50%" @@ -183,7 +183,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "discover", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/uncommon_ip_query_strings.json", "window_end": "START+50%,END", "window_length": "50%" @@ -193,7 +193,7 @@ "operation-type": "kibana", "param-source": "elasticlogs_kibana", "dashboard": "discover", - "index_pattern": "elasticlogs_q-*", + "index_pattern": "{{p_query_index_pattern}}", "query_string": "query_string_lists/country_code_query_strings.json", "window_end": "START+50%,END", "window_length": "50%" diff --git a/eventdata/operations/stats.json b/eventdata/operations/stats.json index 780c388..4a5c4b8 100644 --- a/eventdata/operations/stats.json +++ b/eventdata/operations/stats.json @@ -1,7 +1,7 @@ { "name": "fieldstats_elasticlogs_q-*", "operation-type": "fieldstats", - "index_pattern": "elasticlogs_q-*" + "index_pattern": "{{p_query_index_pattern}}" }, { "name": "indicesstats_elasticlogs", @@ -11,7 +11,7 @@ { "name": "indicesstats_elasticlogs_q-*", "operation-type": "indicesstats", - "index_pattern": "elasticlogs_q-*" + "index_pattern": "{{p_query_index_pattern}}" }, { "name": "indicesstats_elasticlogs_i-*", @@ -21,7 +21,7 @@ { "name": "indicesstats_elasticlogs_q_write", "operation-type": "indicesstats", - "index_pattern": "elasticlogs_q_write" + "index_pattern": "{{p_query_index_write_alias}}" }, { "name": "indicesstats_elasticlogs_i_write", diff --git a/eventdata/track.json b/eventdata/track.json index 0758f9b..b6a6995 100644 --- a/eventdata/track.json +++ b/eventdata/track.json @@ -2,6 +2,10 @@ {% set p_bulk_indexing_clients = (bulk_indexing_clients | default(8)) %} {% set p_record_raw_event_size = record_raw_event_size | default(False) | tojson %} +{% set p_index_prefix = index_prefix | default("elasticlogs") %} +{% set p_query_index_prefix = query_index_prefix | default(p_index_prefix ~ "_q") %} +{% set p_query_index_pattern = p_query_index_prefix ~ "-*" %} +{% set p_query_index_write_alias = p_query_index_prefix ~ "_write" %} { "version": 2, @@ -9,7 +13,7 @@ "templates": [ { "name": "elasticlogs-index-template", - "index-pattern": "elasticlogs*", + "index-pattern": "{{p_index_prefix}}*", "delete-matching-indices": false, "template": "elasticlogs-index-template.json" }