From 14cfd7a3f4681247670e15be52f7cac78f96ac83 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 4 Jan 2019 16:12:16 -0800 Subject: [PATCH 001/119] Added daemonset, new fluentd plugin, less permissive rbac, doc fixes (#46) LGTM - Thank You --- README.md | 5 +- .../splunk-kubernetes-metrics/README.md | 4 +- .../templates/_helpers.tpl | 52 ------------- .../templates/clusterRole.yaml | 17 ++++ .../templates/clusterRoleBinding.yaml | 2 +- .../templates/configMap.yaml | 65 ++++++++++------ .../templates/deployment.yaml | 77 ++++++------------- .../splunk-kubernetes-metrics/values.yaml | 27 +++---- 8 files changed, 98 insertions(+), 151 deletions(-) create mode 100644 helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml diff --git a/README.md b/README.md index 3c9b4b5b..dd665caf 100644 --- a/README.md +++ b/README.md @@ -99,10 +99,9 @@ Splunk Connect for Kubernetes collects Kubernetes objects that can help users ac ## Metrics -Splunk Connect for Kubernetes deploys code on the Kubernetes cluster. This deployment has exactly one pod, which runs two containers: +Splunk Connect for Kubernetes deploys daemonsets on the Kubernetes cluster. These daemonsets have exactly one pod, which runs one container: -* [Heapster](https://github.com/kubernetes/heapster) collects metrics and sends them to the Fluentd sidecar via UDP in `statsd` format. -* Fluentd, which receives metrics from Heapster using [in_udp](https://docs.fluentd.org/v1.0/articles/in_udp) and transforms the metrics using filter_jq_transformer. filter_jq_transformer formats the data for Splunk ingestion: It makea sure the metrics have proper metric_name, dimensions, etc., and then sends the metrics to Splunk using out_splunk_hec. +* [Fluentd metrics plugin](https://github.com/splunk/fluent-plugin-kubernetes-metrics) collects the metrics, formats the metrics for Splunk ingestion by assuring the metrics have proper metric_name, dimensions, etc., and then sends the metrics to Splunk using out_splunk_hec using Fluentd engine. Make sure your Splunk configuration has a metrics index that is able to receive the data. See [Get started with metrics](http://docs.splunk.com/Documentation/Splunk/7.1.0/Metrics/GetStarted) in the Splunk Enterprise documentaiton. diff --git a/helm-chart/splunk-kubernetes-metrics/README.md b/helm-chart/splunk-kubernetes-metrics/README.md index 58636354..b23fa7d9 100644 --- a/helm-chart/splunk-kubernetes-metrics/README.md +++ b/helm-chart/splunk-kubernetes-metrics/README.md @@ -2,7 +2,7 @@ `splunk-kubernetes-metrics` is a [Helm](https://github.com/kubernetes/helm) chart that creates a kubernetes deployment along with other kubernetes objects in a kubernetes cluster to collect the cluster's metrics and send them to [splunk](https://www.splunk.com/). -The deployment runs a pod that has two containers. One is [heapster](https://github.com/kubernetes/heapster), it collects the kubernetes metrics, and sends them to [fluend](https://www.fluentd.org/), which is the other container running in the same pod, over UDP, and finally the metrics will be sent to a splunk instance with the [Splunk HEC output plugin](https://github.com/splunk/fluent-plugin-splunk-hec) over [Splunk HEC](http://docs.splunk.com/Documentation/Splunk/7.1.0/Data/AboutHEC). +The deployment runs a daemonset which runs a pod that has one container on each node. [Fluentd metrics plugin](https://github.com/splunk/fluent-plugin-kubernetes-metrics) collects the metrics, formats the metrics for Splunk ingestion by assuring the metrics have proper metric_name, dimensions, etc., and then sends the metrics to a splunk instance with the [Splunk HEC output plugin](https://github.com/splunk/fluent-plugin-splunk-hec) over [Splunk HEC](http://docs.splunk.com/Documentation/Splunk/7.1.0/Data/AboutHEC). Although it works well by itself, this chart is a part of [`splunk-connect-for-kubernetes`](https://github.com/splunk/splunk-connect-for-kubernetes). If you want a full Splunk monitoring solution for kubernetes, check it out. @@ -40,7 +40,7 @@ The following table lists all components (i.e. kubernetes objects) of this chart Component | Description | Template --- | --- | --- -`Deployment` | deploys one pod that runs heapster and fluentd to collect metrics. | [deployment.yaml](templates/deployment.yaml) +`Deployment` | deploys one pod that runs fluentd plugins and fluentd to collect metrics. | [deployment.yaml](templates/deployment.yaml) `ConfigMap` | contains configuration files for fluentd. | [configmap.yaml](templates/configmap.yaml) `Secret` | stores credentials like the Splunk HEC token, and SSL certs and keys for HTTPS connection, etc. | [secret.yaml](templates/secret.yaml) `ServiceAccount` | a service account to run the daemonset. | [serviceaccount.yaml](templates/serviceaccount.yaml) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl index 942294f9..461fe48e 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl @@ -41,55 +41,3 @@ Create the name of the service account to use {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} - - -{{/* -The jq filter used to transform Heapster metrics which are formatted in statd format. -Define it as a template here so there we don't need to escape the double quotes `` " ''. -*/}} -{{- define "splunk-kubernetes-metrics.jq_filter" -}} -def extract_labels: - . as $labels | reduce range(length) as $n ({}; if $n % 2 == 0 then .["label." + $labels[$n]] = $labels[$n + 1] else . end); - -def extract_metric: - if length % 2 == 0 - then (.[:-2] | extract_labels) + {metric: (.[-2] | gsub("/"; ".")), resource_id: .[-1]} - else (.[:-1] | extract_labels) + {metric: (.[-1] | gsub("/"; "."))} - end; - -def extract_container: - split(".") | {container_type: "pod", node: .[1], namespace: .[3], pod: .[5], container: .[7]} + (.[8:] | extract_metric) | .metric = "kube.container." + .metric | . ; - -def extract_syscontainer: - split(".") | {container_type: "sys", node: .[1], container: .[3]} + (.[4:] | extract_metric) | .metric = "kube.container." + .metric | . ; - -def extract_pod: - split(".") | {node: .[1], namespace: .[3], pod: .[5]} + (.[6:] | extract_metric) | .metric = "kube.pod." + .metric | . ; - -def extract_namespace: - split(".") | {namespace: .[1]} + (.[2:] | extract_metric) | .metric = "kube.namespace." + .metric | . ; - -def extract_node: - split(".") | {node: .[1]} + (.[2:] | extract_metric) | .metric = "kube.node." + .metric | . ; - -def extract_cluster: - split(".") | .[1:] | extract_metric | .metric = "kube.cluster." + .metric | . ; - -def extract: - if contains(".container.") - then extract_container - elif contains(".sys-container.") - then extract_syscontainer - elif contains(".pod.") - then extract_pod - elif startswith("namespace.") - then extract_namespace - elif startswith("node.") - then extract_node - elif startswith("cluster.") - then extract_cluster - else {} - end; - - "heapster/namespace:\(env.MY_NAMESPACE)/pod:\(env.MY_POD_NAME)" as $source | .record | to_entries | map({value, source: $source} + (.key | extract)) | . -{{- end -}} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml new file mode 100644 index 00000000..c50268bc --- /dev/null +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml @@ -0,0 +1,17 @@ +# This role allows read access to the kubelet summary API +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubelet-summary-api-read + labels: + app: {{ template "splunk-kubernetes-metrics.name" . }} + chart: {{ template "splunk-kubernetes-metrics.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: +- apiGroups: + - "" + resources: + - "nodes/stats" + verbs: + - "get" diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBinding.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBinding.yaml index 57cb8784..50d0932a 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBinding.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBinding.yaml @@ -11,7 +11,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: "system:heapster" + name: "kubelet-summary-api-read" subjects: - kind: ServiceAccount name: {{ template "splunk-kubernetes-metrics.serviceAccountName" . }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index 58da3b3e..3fe123d1 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -9,32 +9,49 @@ metadata: heritage: {{ .Release.Service }} data: fluent.conf: | - # system wide configurations + # system wide configurations log_level {{ or .Values.logLevel .Values.global.logLevel | default "info" }} - - @type udp - tag raw.metrics.udp - port 9001 - message_length_limit 100m - - @type jq - jq 'split("\n") | reduce .[] as $item ({}; ($item | rindex(":")) as $i | .[$item[:$i]] = ($item[$i+1:-2] | tonumber))' - + @type kubernetes_metrics + tag kube.* + node_name "#{ENV['SPLUNK_HEC_HOST']}" - - - @type jq - jq {{ include "splunk-kubernetes-metrics.jq_filter" . | quote }} - remove_tag_prefix raw - - - + + @type record_modifier + + metric_name ${tag} + + + + @type record_modifier + + source ${record['node']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['container-name']} + + + @type splunk_hec data_type metric - metric_name_key metric + metric_name_key metric_name metric_value_key value protocol {{ or .Values.splunk.hec.protocol .Values.global.splunk.hec.protocol }} {{- with or .Values.splunk.hec.host .Values.global.splunk.hec.host }} @@ -59,14 +76,14 @@ data: {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} + interval 15s @type memory - {{- $limit := .Values.resources.sidecar.limit }} - chunk_limit_size {{ if $limit.memory }}{{ template "splunk-kubernetes-logging.convert-memory" $limit.memory }}{{ else }}{{ "500m" }}{{ end }} - chunk_limit_records 100000 - flush_interval 5s + total_limit_size 100m + chunk_limit_size 10m + flush_interval 15s flush_thread_count 1 overflow_action block retry_max_times 3 - + \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index b36a6e3f..f01e7a9d 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -1,5 +1,5 @@ -apiVersion: apps/v1beta1 -kind: Deployment +apiVersion: apps/v1 +kind: DaemonSet metadata: name: {{ template "splunk-kubernetes-metrics.fullname" . }} labels: @@ -7,20 +7,21 @@ metadata: chart: {{ template "splunk-kubernetes-metrics.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - k8s-app: heapster-fluentd + engine: fluentd spec: - replicas: 1 selector: matchLabels: - app: {{ template "splunk-kubernetes-metrics.name" . }} - release: {{ .Release.Name }} - k8s-app: heapster + name: {{ template "splunk-kubernetes-metrics.fullname" . }} template: metadata: + name: {{ template "splunk-kubernetes-metrics.fullname" . }} labels: + name: {{ template "splunk-kubernetes-metrics.fullname" . }} app: {{ template "splunk-kubernetes-metrics.name" . }} + chart: {{ template "splunk-kubernetes-metrics.chart" . }} release: {{ .Release.Name }} - k8s-app: heapster + heritage: {{ .Release.Service }} + engine: fluentd annotations: scheduler.alpha.kubernetes.io/critical-pod: '' checksum/config: {{ include (print $.Template.BasePath "/configMap.yaml") . | sha256sum }} @@ -39,50 +40,22 @@ spec: {{ toYaml . | indent 8 }} {{- end }} containers: - - image: k8s.gcr.io/heapster-amd64:{{ .Values.image.heapsterTag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - name: splunk-heapster - command: - - /heapster - # - --source=kubernetes.summary_api:'' - - --source=kubernetes - - --sink=statsd:udp://127.0.0.1:9001 - livenessProbe: - httpGet: - path: /healthz - port: 8082 - scheme: HTTP - initialDelaySeconds: 180 - timeoutSeconds: 5 - resources: -{{ toYaml .Values.resources.heapster | indent 12 }} - - image: splunk/fluentd-hec:{{ .Values.image.sidecarTag }} - name: splunk-fluentd-heapster - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - -c - - /fluentd/etc/fluent.conf - env: - - name: MY_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: SPLUNK_HEC_HOST - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: SPLUNK_HEC_TOKEN - valueFrom: - secretKeyRef: - name: {{ template "splunk-kubernetes-metrics.fullname" . }} - key: splunk_hec_token - resources: -{{ toYaml .Values.resources.sidecar | indent 12 }} - volumeMounts: + - name: fluentd + image: {{ .Values.image.name }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPLUNK_HEC_HOST + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: {{ template "splunk-kubernetes-metrics.fullname" . }} + key: splunk_hec_token + resources: +{{ toYaml .Values.resources.fluent | indent 12 }} + volumeMounts: - name: conf-configmap mountPath: /fluentd/etc - name: secrets diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index d41d69bc..c9b5378e 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -74,25 +74,17 @@ splunk: image: - heapsterTag: v1.5.1 - sidecarTag: 1.0.1 - pullPolicy: IfNotPresent + name: dummy-user/dummy-repo:dummy-tag + pullPolicy: Always resources: - heapster: + fluent: limits: - cpu: 100m + cpu: 200m memory: 300Mi requests: - cpu: 100m - memory: 300Mi - sidecar: - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m + cpu: 200m memory: 300Mi @@ -100,10 +92,11 @@ nodeSelector: {} # kubernetes.io/role: master -tolerations: [] - # - key: "node-role.kubernetes.io/master" - # operator: "Exists" - # effect: "NoSchedule" +# This default tolerations allow the daemonset to be deployed on master nodes, +# so that we can also collect metrics from those nodes. +tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule affinity: {} From 368a8de530704ce872f6612c62dce082ac39a1ba Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 4 Jan 2019 16:14:53 -0800 Subject: [PATCH 002/119] Metrics aggregator update (#51) * Added aggregator support --- .../templates/clusterRoleAggregator.yaml | 21 ++++ .../clusterRoleBindingAggregator.yaml | 19 ++++ .../templates/configMapMetricsAggregator.yaml | 97 +++++++++++++++++++ .../deploymentMetricsAggregator.yaml | 62 ++++++++++++ .../splunk-kubernetes-metrics/values.yaml | 7 +- 5 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml create mode 100644 helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBindingAggregator.yaml create mode 100644 helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml create mode 100644 helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml new file mode 100644 index 00000000..916dbec1 --- /dev/null +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml @@ -0,0 +1,21 @@ +# This role allows read access to the kubelet summary API +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-api-aggregator + labels: + app: {{ template "splunk-kubernetes-metrics.name" . }} + chart: {{ template "splunk-kubernetes-metrics.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/stats + - nodes/proxy + - pods + verbs: + - get + - list diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBindingAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBindingAggregator.yaml new file mode 100644 index 00000000..5471721f --- /dev/null +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleBindingAggregator.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "splunk-kubernetes-metrics.fullname" . }}-aggregator + labels: + app: {{ template "splunk-kubernetes-metrics.name" . }} + chart: {{ template "splunk-kubernetes-metrics.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "kube-api-aggregator" +subjects: +- kind: ServiceAccount + name: {{ template "splunk-kubernetes-metrics.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml new file mode 100644 index 00000000..c20113af --- /dev/null +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "splunk-kubernetes-metrics.fullname" . }}-aggregator + labels: + app: {{ template "splunk-kubernetes-metrics.name" . }} + chart: {{ template "splunk-kubernetes-metrics.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + fluent.conf: | + # system wide configurations + + log_level {{ or .Values.logLevel .Values.global.logLevel | default "info" }} + + + @type kubernetes_metrics_aggregator + tag kube.* + + + @type record_modifier + + metric_name ${tag} + + + + @type record_modifier + + source ${record['name']} + + + + @type record_modifier + + source ${record['name']} + + + + @type record_modifier + + source ${record['node']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['container-name']} + + + + @type splunk_hec + data_type metric + metric_name_key metric_name + metric_value_key value + protocol {{ or .Values.splunk.hec.protocol .Values.global.splunk.hec.protocol }} + {{- with or .Values.splunk.hec.host .Values.global.splunk.hec.host }} + hec_host {{ . }} + {{- end }} + {{- with or .Values.splunk.hec.port .Values.global.splunk.hec.port }} + hec_port {{ . }} + {{- end }} + hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" + host "#{ENV['MY_NODE_NAME']}" + source source + insecure_ssl {{ or .Values.splunk.hec.insecureSSL .Values.global.splunk.hec.insecureSSL | default false }} + {{- if or .Values.splunk.hec.clientCert .Values.global.splunk.hec.clientCert }} + client_cert /fluentd/etc/splunk/hec_client_cert + {{- end }} + {{- if or .Values.splunk.hec.clientKey .Values.global.splunk.hec.clientKey }} + client_key /fluentd/etc/splunk/hec_client_key + {{- end }} + {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} + ca_file /fluentd/etc/splunk/hec_ca_file + {{- end }} + interval 15s + + @type memory + total_limit_size 100m + chunk_limit_size 10m + flush_interval 15s + flush_thread_count 1 + overflow_action block + retry_max_times 3 + + diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml new file mode 100644 index 00000000..c18a5ad4 --- /dev/null +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: {{ template "splunk-kubernetes-metrics.fullname" . }} + labels: + app: {{ template "splunk-kubernetes-metrics.name" . }} + chart: {{ template "splunk-kubernetes-metrics.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "splunk-kubernetes-metrics.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "splunk-kubernetes-metrics.name" . }} + release: {{ .Release.Name }} + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + checksum/config: {{ include (print $.Template.BasePath "/configMapMetricsAggregator.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ template "splunk-kubernetes-metrics.serviceAccountName" . }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: fluentd + image: {{ .Values.imageAgg.name }} + imagePullPolicy: {{ .Values.imageAgg.pullPolicy }} + env: + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: {{ template "splunk-kubernetes-metrics.fullname" . }} + key: splunk_hec_token + resources: +{{ toYaml .Values.resources.fluent | indent 12 }} + volumeMounts: + - name: conf-configmap + mountPath: /fluentd/etc + - name: secrets + mountPath: /fluentd/etc/splunk + readOnly: true + volumes: + - name: conf-configmap + configMap: + name: {{ template "splunk-kubernetes-metrics.fullname" . }}-aggregator + - name: secrets + secret: + secretName: {{ template "splunk-kubernetes-metrics.fullname" . }} diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index c9b5378e..89fb2855 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -8,7 +8,7 @@ # Global values have lower priority. # Values defined here are the default values. global: - logLevel: info + logLevel: debug splunk: hec: port: 8088 @@ -78,6 +78,11 @@ image: pullPolicy: Always +imageAgg: + name: dummy-user/dummy-repo:dummy-tag + pullPolicy: Always + + resources: fluent: limits: From 73b6d954f086156dc8a61dc96f1bac456e9071e6 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 7 Jan 2019 14:33:12 -0800 Subject: [PATCH 003/119] Beta release --- helm-chart/splunk-kubernetes-metrics/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 89fb2855..31256df5 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -8,7 +8,7 @@ # Global values have lower priority. # Values defined here are the default values. global: - logLevel: debug + logLevel: info splunk: hec: port: 8088 @@ -74,12 +74,12 @@ splunk: image: - name: dummy-user/dummy-repo:dummy-tag + name: splunk/k8s-metrics:1.1.0.Alpha pullPolicy: Always imageAgg: - name: dummy-user/dummy-repo:dummy-tag + name: splunk/k8s-metrics-aggr:1.1.0.Alpha pullPolicy: Always From c98ebb3b23f4d763024f00361ab3f09e0bde54d1 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 8 Jan 2019 15:13:52 -0800 Subject: [PATCH 004/119] Basic CI which builds and pushes SCK artifacts (#66) --- .circleci/build_sck_artifacts.sh | 39 ++++++++++++++++++++++++++++++++ .circleci/config.yml | 23 +++++++++++++++++++ .circleci/push_sck_artifacts.sh | 10 ++++++++ README.md | 26 +++++++++++---------- 4 files changed, 86 insertions(+), 12 deletions(-) create mode 100755 .circleci/build_sck_artifacts.sh create mode 100644 .circleci/config.yml create mode 100755 .circleci/push_sck_artifacts.sh diff --git a/.circleci/build_sck_artifacts.sh b/.circleci/build_sck_artifacts.sh new file mode 100755 index 00000000..683f9eec --- /dev/null +++ b/.circleci/build_sck_artifacts.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -e +TAG="1.1.0.Alpha-${CIRCLE_SHA1}" + +function replace_generic_version () +{ + file="$1" + _line=`awk '/version:/{print NR;exit}' $file` + replacement="version: $TAG" + # Escape backslash, forward slash and ampersand for use as a sed replacement. + replacement_escaped=$( echo "$replacement" | sed -e 's/[\/&]/\\&/g' ) + sed -i "${_line}s/.*/$replacement_escaped/" "$file" +} + +repos_array=( "helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) +sub_repos_array=( "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) + +for repo in "${repos_array[@]}" +do + filename="${repo}/Chart.yaml" + replace_generic_version $filename +done + +mkdir helm-artifacts +mkdir helm-chart/splunk-connect-for-kubernetes/charts + +for sub_repo in "${sub_repos_array[@]}" +do + cp -rp $sub_repo helm-chart/splunk-connect-for-kubernetes/charts +done + +for repo in "${repos_array[@]}" +do + helm package -d helm-artifacts $repo +done + +rm -rf helm-chart/splunk-connect-for-kubernetes/charts diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..06f0e97f --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,23 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/ruby:2.4.1-node-browsers + + working_directory: ~/repo + + steps: + - checkout + - run: + name: Install dependencies + command: | + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash + helm init -c + - run: + name: Build SCK artifacts + command: | + .circleci/build_sck_artifacts.sh + - run: + name: Push SCK artifacts to s3 + command: | + .circleci/push_sck_artifacts.sh diff --git a/.circleci/push_sck_artifacts.sh b/.circleci/push_sck_artifacts.sh new file mode 100755 index 00000000..516c7d80 --- /dev/null +++ b/.circleci/push_sck_artifacts.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e +sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 +echo "Installing aws cli..." +sudo pip install awscli > /dev/null 2>&1 +echo "Pushing SCK artifacts to s3..." +aws s3 cp helm-artifacts/splunk-kubernetes-logging*.tgz s3://k8s-ci-artifacts/ +aws s3 cp helm-artifacts/splunk-kubernetes-metrics*.tgz s3://k8s-ci-artifacts/ +aws s3 cp helm-artifacts/splunk-kubernetes-objects*.tgz s3://k8s-ci-artifacts/ +aws s3 cp helm-artifacts/splunk-connect-for-kubernetes*.tgz s3://k8s-ci-artifacts/ diff --git a/README.md b/README.md index dd665caf..712d851a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + +[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/splunk-connect-for-kubernetes) # What does Splunk Connect for Kubernetes do? Splunk Connect for Kubernetes provides a way to import and search your Kubernetes logging, object, and metrics data in Splunk. Splunk is a proud contributor to Cloud Native Computing Foundation (CNCF) and Splunk Connect for Kubernetes utilizes and supports multiple CNCF components in the development of these tools to get data into Splunk. @@ -10,18 +12,18 @@ Splunk Connect for Kubernetes provides a way to import and search your Kubernete * http://docs.splunk.com/Documentation/Splunk/7.0.3/Data/UsetheHTTPEventCollector * http://docs.splunk.com/Documentation/Splunk/7.0.3/Data/ScaleHTTPEventCollector * You should be familiar with your Kubernetes configuration and know where your log info is collected in Kubernetes. -* You must have administrator access to your Kubernetes cluster. +* You must have administrator access to your Kubernetes cluster. * To install using Helm (recommended), make sure you are running Helm in your Kubernetes configuration. See https://github.com/kubernetes/helm * Have a minimum of two Splunk indexes ready to collect the log data, one for both logs and Kubernetes objects, and one for metrics. You can also create separate indexes for logs and objects, in which case you will need three Splunk indexes. ## Before you begin -Splunk Connect for Kubernetes supports installation using Helm. Ensure that you thoroughly read the Prerequisites and Installation and Deployment documentation before you start your deployment of Splunk Connect for Kubernetes. +Splunk Connect for Kubernetes supports installation using Helm. Ensure that you thoroughly read the Prerequisites and Installation and Deployment documentation before you start your deployment of Splunk Connect for Kubernetes. Make sure you do the following before you install: -1. Create a minimum of two Splunk indexes: +1. Create a minimum of two Splunk indexes: * one events index, which will handle logs and objects (you may also create two separate indexes for logs and objects). -* one metrics index. +* one metrics index. If you do not configure these indexes, Kubernetes Connect for Splunk uses the defaults created in your HEC token. 2. Create a HEC token if you do not already have one. If you are installing the connector on Splunk Cloud, file a ticket with Splunk Customer Service and they will deploy the indexes for your environment and generate your HEC token. @@ -36,13 +38,13 @@ To install and configure defaults with Helm: $ helm install --name my-release -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-connect-for-kubernetes-1.0.1.tgz ``` -To learn more about using and modifying charts, see: +To learn more about using and modifying charts, see: * https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart * https://docs.helm.sh/using_helm/#using-helm. ## Configuration variables for Helm -To learn more about using and modifying charts, see: +To learn more about using and modifying charts, see: * [The values file for logging](https://github.com/splunk/splunk-connect-for-kubernetes/blob/master/helm-chart/splunk-kubernetes-logging/values.yaml) * [The values file for metrics](https://github.com/splunk/splunk-connect-for-kubernetes/blob/master/helm-chart/splunk-kubernetes-metrics/values.yaml) * [The values file for objects](https://github.com/splunk/splunk-connect-for-kubernetes/blob/master/helm-chart/splunk-kubernetes-objects/values.yaml) @@ -51,7 +53,7 @@ To learn more about using and modifying charts, see: You can grab the manifest YAML files and use them to create the Kubernetes objects needed to deploy Splunk Connect for Kubernetes. Please note that installation and debugging for Splunk Connect for Kubernetes through YAML is community-supported only. -When you use YAML to deploy Splunk Connect for Kubernetes, the installation does not create the default configuration that is created when you install using Helm. To deploy the connector using YAML, you must know how to configure your Kubernetes variables to work with the connector. If you are not familiar with this process, we recommend that you use the Helm installation method. +When you use YAML to deploy Splunk Connect for Kubernetes, the installation does not create the default configuration that is created when you install using Helm. To deploy the connector using YAML, you must know how to configure your Kubernetes variables to work with the connector. If you are not familiar with this process, we recommend that you use the Helm installation method. To configure the Splunk Connector for Kubernetes using YAML files: @@ -76,7 +78,7 @@ To collect the data, Splunk leverages: * [Fluentd](https://www.fluentd.org/) * [JQ plugin](https://rubygems.org/gems/fluent-plugin-jq) for transforming data * [Splunk HEC output plug-in](https://github.com/splunk/fluent-plugin-splunk-hec): The [HTTP Event Collector](http://dev.splunk.com/view/event-collector/SP-CAAAE6M) collects all data sent to Splunk for indexing. -* For Splunk Connect for Kubernetes, Splunk uses the [node logging agent](https://kubernetes.io/docs/concepts/cluster-administration/logging/#using-a-node-logging-agent) method. See the [Kubernetes Logging Architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/) for an overview of the types of Kubernetes logs from which you may wish to collect data as well as information on how to set up those logs. +* For Splunk Connect for Kubernetes, Splunk uses the [node logging agent](https://kubernetes.io/docs/concepts/cluster-administration/logging/#using-a-node-logging-agent) method. See the [Kubernetes Logging Architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/) for an overview of the types of Kubernetes logs from which you may wish to collect data as well as information on how to set up those logs. ## Logs @@ -84,7 +86,7 @@ Splunk Connect for Kubernetes uses the Kubernetes [node logging agent](https://k * [in_systemd](https://rubygems.org/gems/fluent-plugin-systemd) reads logs from systemd journal if systemd is available on the host. * [in_tail](https://docs.fluentd.org/v1.0/articles/in_tail) reads logs from file system. -* [filter_jq_transformer](https://rubygems.org/gems/fluent-plugin-jq) transforms the raw events to a Splunk-friendly format and generates source and sourcetypes. +* [filter_jq_transformer](https://rubygems.org/gems/fluent-plugin-jq) transforms the raw events to a Splunk-friendly format and generates source and sourcetypes. * [out_splunk_hec](https://github.com/splunk/fluent-plugin-splunk-hec) sends the translated logs to Splunk indexes through the HTTP Event Collector input (HEC). ## Kubernetes Objects @@ -94,7 +96,7 @@ Splunk Connect for Kubernetes collects Kubernetes objects that can help users ac * [in_kubernetes_objects](https://github.com/splunk/fluent-plugin-kubernetes-objects) collects object data by calling the Kubernetes API (by https://github.com/abonas/kubeclient). in-kubernetes-objects supports two modes: * watch mode: the Kubernetes API sends new changes to the plugin. In this mode, only the changed data is collected. * pull mode: the plugin queries the Kubernetes API periodically. In this mode, all data is collected. -* [filter_jq_transformer](https://rubygems.org/gems/fluent-plugin-jq) transforms the raw data into a Splunk-friendly format and generates sources and sourcetypes. +* [filter_jq_transformer](https://rubygems.org/gems/fluent-plugin-jq) transforms the raw data into a Splunk-friendly format and generates sources and sourcetypes. * [out_splunk_hec](https://github.com/splunk/fluent-plugin-splunk-hec) sends the data to Splunk via HTTP Event Collector input (HEC). ## Metrics @@ -111,14 +113,14 @@ If you want to learn more about which metrics are collected and labels used with # Performance -Some parameters used with Splunk Connect for Kubernetes can have an impact on overall performance of log ingestion, objects, or metrics. In general, the more filters that are added to one of the streams, the greater the preformance impact. +Some parameters used with Splunk Connect for Kubernetes can have an impact on overall performance of log ingestion, objects, or metrics. In general, the more filters that are added to one of the streams, the greater the preformance impact. Splunk Connect for Kubernetes can exceed the default throughput of HEC. To best address capacity needs, Splunk recommends that you monitor the HEC throughput and back pressure on Splunk Connect for Kubernetes deployments and be prepared to add additional nodes as needed. # Processing Multi-Line Logs -One possible filter option is to enable the processing of multi-line events. This feature is currently experimental and considered to be community supported. +One possible filter option is to enable the processing of multi-line events. This feature is currently experimental and considered to be community supported. # License From 92ddd108cc170d6233991e73f3157378921f6c14 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 9 Jan 2019 13:58:57 -0800 Subject: [PATCH 005/119] Make metrics interval configurable (#67) --- .../splunk-kubernetes-metrics/templates/configMap.yaml | 6 +++--- .../templates/configMapMetricsAggregator.yaml | 2 +- helm-chart/splunk-kubernetes-metrics/values.yaml | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index 3fe123d1..3c00c0dc 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -9,7 +9,7 @@ metadata: heritage: {{ .Release.Service }} data: fluent.conf: | - # system wide configurations + # system wide configurations log_level {{ or .Values.logLevel .Values.global.logLevel | default "info" }} @@ -76,7 +76,7 @@ data: {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} - interval 15s + interval {{ or .Values.metricsInterval .Values.global.metricsInterval }} @type memory total_limit_size 100m @@ -86,4 +86,4 @@ data: overflow_action block retry_max_times 3 - \ No newline at end of file + diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index c20113af..634cfa57 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -84,7 +84,7 @@ data: {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} - interval 15s + interval {{ or .Values.metricsInterval .Values.global.metricsInterval }} @type memory total_limit_size 100m diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 31256df5..ac7e63c6 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -93,6 +93,10 @@ resources: memory: 300Mi +# Configure how often SCK pulls metrics for its kubenetes sources. 15s is the default where 's' is seconds. +metricsInterval: 15s + + nodeSelector: {} # kubernetes.io/role: master From c697f80069300542482b16566658de91e7ba77b6 Mon Sep 17 00:00:00 2001 From: gp510 Date: Mon, 14 Jan 2019 13:01:00 -0800 Subject: [PATCH 006/119] Updates to support metrics SSL (#68) * Updates to support metcirs SSL * Updates based on review * Clean up commit log messages * Further revisison --- .../templates/clusterRole.yaml | 2 ++ .../templates/configMap.yaml | 20 +++++++++++++++++-- .../splunk-kubernetes-metrics/values.yaml | 15 ++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml index c50268bc..32ed5d0b 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml @@ -12,6 +12,8 @@ rules: - apiGroups: - "" resources: + - "nodes" - "nodes/stats" + - "nodes/metrics" verbs: - "get" diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index 3c00c0dc..754979cf 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -9,7 +9,7 @@ metadata: heritage: {{ .Release.Service }} data: fluent.conf: | - # system wide configurations + # system wide configurations log_level {{ or .Values.logLevel .Values.global.logLevel | default "info" }} @@ -17,6 +17,22 @@ data: @type kubernetes_metrics tag kube.* node_name "#{ENV['SPLUNK_HEC_HOST']}" + {{- with .Values.kubernetes.kubeletPort }} + kubelet_port {{ . }} + {{- end }} + use_rest_client_ssl {{ .Values.kubernetes.useRestClientSSL }} + {{- with .Values.kubernetes.insecureSSL }} + insecure_ssl {{ . }} + {{- end }} + {{- if .Values.kubernetes.caFile }} + ca_file /fluentd/etc/splunk/k8s_ca_file + {{- end }} + {{- with .Values.kubernetes.bearerTokenFile }} + bearer_token_file {{ . }} + {{- end }} + {{- with .Values.kubernetes.secretDir }} + secret_dir {{ . }} + {{- end }} @type record_modifier @@ -86,4 +102,4 @@ data: overflow_action block retry_max_times 3 - + \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index ac7e63c6..9965e1bf 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -109,3 +109,18 @@ tolerations: affinity: {} + +# = Kubernetes Connection Configs = +kubernetes: + # The port that kubelet is listening to. Default is 10250 + kubeletPort: + # This option is used to get the metrics from summary api on each kubelet using ssl + useRestClientSSL: true + # if insecureSSL is set to true, insecure HTTPS API call is allowed, default false + insecureSSL: false + # Path to the CA file. + caFile: + # Path to the file contains the API token. By default it reads from the file "token" in the `secret_dir`. + bearerTokenFile: + # Path of the location where pod's service account's credentials are stored. Usually you don't need to care about this config, the default value should work in most cases. + secretDir: From ed0d503bf71bd477d0e27ce90b49c0228d43e143 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 14 Jan 2019 14:38:18 -0800 Subject: [PATCH 007/119] Make objects docker image configurable (#69) --- .../splunk-kubernetes-objects/templates/deployment.yaml | 2 +- helm-chart/splunk-kubernetes-objects/values.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index ebb707f4..95a5a89e 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -39,7 +39,7 @@ spec: {{- end }} containers: - name: splunk-fluentd-k8s-objects - image: splunk/kube-objects:{{ .Values.image.tag }} + image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - -c diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 0db71def..12de1c51 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -62,7 +62,7 @@ kubernetes: # Path of the location where pod's service account's credentials are stored. Usually you don't need to care about this config, the default value should work in most cases. secretDir: # a list of kubernetes objects to pull periodically - + # = Object Lists = # NOTE: at least one object must be provided. @@ -164,7 +164,7 @@ splunk: image: - tag: 1.0.1 + name: splunk/kube-objects:1.0.1 pullPolicy: IfNotPresent From f6d7cf0ea888a59b7e6b59f662c861e52d8c4ee8 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 16 Jan 2019 14:03:58 -0800 Subject: [PATCH 008/119] Fixes from feedback (#75) --- .../templates/configMapMetricsAggregator.yaml | 3 +++ .../splunk-kubernetes-metrics/templates/deployment.yaml | 2 +- .../templates/deploymentMetricsAggregator.yaml | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index 634cfa57..fb4e7a37 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -16,6 +16,9 @@ data: @type kubernetes_metrics_aggregator tag kube.* + {{- with .Values.kubernetes.kubeletPort }} + kubelet_port {{ . }} + {{- end }} @type record_modifier diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index f01e7a9d..104ac1d1 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -40,7 +40,7 @@ spec: {{ toYaml . | indent 8 }} {{- end }} containers: - - name: fluentd + - name: splunk-fluentd-k8s-metrics image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index c18a5ad4..005227c1 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1beta1 kind: Deployment metadata: - name: {{ template "splunk-kubernetes-metrics.fullname" . }} + name: {{ template "splunk-kubernetes-metrics.fullname" . }}-agg labels: app: {{ template "splunk-kubernetes-metrics.name" . }} chart: {{ template "splunk-kubernetes-metrics.chart" . }} @@ -36,7 +36,7 @@ spec: {{ toYaml . | indent 8 }} {{- end }} containers: - - name: fluentd + - name: splunk-fluentd-k8s-metrics-agg image: {{ .Values.imageAgg.name }} imagePullPolicy: {{ .Values.imageAgg.pullPolicy }} env: From 71ef9876bda18ff8051f1ed84287945d30ebda43 Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Fri, 18 Jan 2019 13:20:13 -0800 Subject: [PATCH 009/119] added ci deployment pipeline (#78) --- .circleci/config.yml | 17 ++++++++++++++ .circleci/deploy_connector.sh | 15 ++++++++++++ .circleci/kubeconfig.gpg | Bin 0 -> 3921 bytes .circleci/push_sck_artifacts.sh | 3 --- .circleci/sck_values.yml | 40 ++++++++++++++++++++++++++++++++ .circleci/update_sck_config.sh | 28 ++++++++++++++++++++++ 6 files changed, 100 insertions(+), 3 deletions(-) mode change 100644 => 100755 .circleci/config.yml create mode 100755 .circleci/deploy_connector.sh create mode 100755 .circleci/kubeconfig.gpg create mode 100755 .circleci/sck_values.yml create mode 100755 .circleci/update_sck_config.sh diff --git a/.circleci/config.yml b/.circleci/config.yml old mode 100644 new mode 100755 index 06f0e97f..458ddf48 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,16 +8,33 @@ jobs: steps: - checkout + - setup_remote_docker: + reusable: true - run: name: Install dependencies command: | curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash helm init -c + sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 + echo "Installing aws cli..." + sudo pip install awscli > /dev/null 2>&1 + + - run: + name: Update configurables to build artifacts + command: | + .circleci/update_sck_config.sh + - run: name: Build SCK artifacts command: | .circleci/build_sck_artifacts.sh + - run: name: Push SCK artifacts to s3 command: | .circleci/push_sck_artifacts.sh + + - run: + name: Deploy k8s connector + command: | + .circleci/deploy_connector.sh diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh new file mode 100755 index 00000000..851731ec --- /dev/null +++ b/.circleci/deploy_connector.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e +echo "Setup kube client..." +curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +sudo apt-get -y install gnupg +sudo mkdir ~/.kube +echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt .circleci/kubeconfig.gpg +sudo mv config ~/.kube/config +helm install --name=ci-$CIRCLE_SHA1 -f .circleci/sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz +#Todo-need to add test cases and run them before cleaning the deployment +sleep 60 +echo "clean up k8s-connect deployment..." +helm delete --purge ci-$CIRCLE_SHA1 \ No newline at end of file diff --git a/.circleci/kubeconfig.gpg b/.circleci/kubeconfig.gpg new file mode 100755 index 0000000000000000000000000000000000000000..7ba69a9fa786ae0272528dbae9d392d00aabde3d GIT binary patch literal 3921 zcmV-X53cZx4Fm@R0v&N$>Nysi7Uk0G0X1QyElG^(PY$g;+IvD(;8Qf~qP+)6kUfO?#Lt+@D?I(&qmdsnbq_qL zhaJ#b7q-a2ldy9<%T5#V@;f`X60|pcz|&94R1sY=Pu^Na1FPrTN((;=Yo|gtRLMD2 z#QC4lNVcyVQvs1E6O2qw+XmI^yd17U;Kekr%cDy9Bcmo+yw3wBo(jxenL$^<@C29T zSAR4Yg_tPCY68$T@y32(oX-Axvm0<0^2X^escedabJtjOhN|HP(RE|Lk^`tv4+#;TF$e--q z`4NWnl3b+1=C7BGS(B-_vuN0`)dTmb?X{H}@jNQ}$ySYa7`vrKF4?6DhS1V`{^FR; z5+QL~mQL>|DNSKbeUJQ`%_+uEP8|@F*nn+}$}WH&_3={R*PWzPc*n`tb#ns&W1=Ll zC=Cp~vyATg)1Sfwtbzqgsch1&swl4D9V>n}l}@H2oT@Z1;c~A*F-$J~V(%eApd)N9 znFgR0u6E$%&SmoZ(qse?VBJEoMgH*4qL||fulV3VDSjO=_JXj$#1_aAW#B_fd8&?~ zm!c$n&Cd5$_m97F+U+7?Vqhxkq2RfXwoeBG0}g}3Ir-n%*3II*Vs7EqW&0shn*`_V z7EouL>=!H$V1TRb1fD(cB{w6jEfCE4>!zo1SuhPBGEm;R^4RjT=H9?BmIX`*d1|Fr z?O9*oy*M%$L~b-C>bYCYVcZK_Xz)=Y|b9=Za{pTKDYG{vo&(1_c{+IquC4O*Lgzb;sk+Z<#5u7mc?c z3gm0qm~bpHc%(sQUm{$&#l4S%A(J4v5;_ETV^u*UAho|4m9N zMy}i5w9Y=u^(?U59*H0hGLu2LVaJgBSrQe`v8wDz?br9`^LjA3 zMu1wIRQplf*WmxbxS9|%%=?yO9PCYYde@^KiL8mx_6YT2$Uktrd z!^333Edi1QVASX`d%Sn_fcVkIbVu%XABk@bCtizJwHc^Zx2P5QMwyX4Ce7nt;`2+7 zZk}*4ZGeIQz=1^0{xkN~HjsFaG3AzN4C)D~sX%Z{J44FkiW-yAm5odIKjJ8=<YjWG650N&xj z$UDXkLblfuQ2-UNg%}K&l!a!svsE68Eu#lutABPtJ5`u;2o|w{b06zory1B9C*)ql zMsL`v7Z#&N^DVt5SC{QUg)li`@L4SaRc-iu=6!<))+IB11)8kzu{ZeslI_Zhi{LA|ECE;FF} z`&rA_jEV^1;amw(KX=c5Ay^q+d3W`-oGZRd`Ib2jkdXnD0u`t2OC@+704(-BFsWWH z##Ekf<^|+PA0w5wV0=pRQi)+hpvPw~Iw%a1kUym;&kSV{F{N_0fv`qmr0OoS=nAfB z!zGNp_&i@^--ZSIcBx~cuZ?{+TAPa`GM{h$U!8ePJ*^yi`KnV)CzO{jti~{p$zd#Q z3AG6+Lz}SXz*swuepL*JL+ip7qP6`YMI~e3>s#V9c*6{r8LlP!N^G_jUHLz4if>@~ zAY|2m;q;m??}ywkNbg^@EpJ}MfS+sI1#e5se@|-kc8WShlGDqepl5=3RA} zB)jN8#TqM>Nq-EUus8X97CQ&v&la+!-I+z)QvmNkbhGRin&6eJJL<|E)9fsw!vU~? zCP{p~9~@K1K}Ia>6-i^Bc$td@=ZaF~``4Bbz1L4nC*)e--Q|UE>W+gf*rY^W7ze2F zW)!i_`3cQEQ-_Rp$N|+0FH38WL@R6hPhml@ZlS)d*(It>M^rN`=&I|H>Rk$4T;UU} zab!2xt^5f5#Ww4(kWickkO>+IMye~(%`dta#y%h4T*>G$1bgDmm@bhcXW~OmIHCKy zFKbo60osOZ=&8bo&d-e znDjq-eS%XG-Y63D_8X_VKXePG&g}|ho2%{34c0+cZa|K)cX6e^3u4dd4AA9j6KZt0 z@kbYEjjZ7isRfZ{0wwx{{{V|ZXXu`0ey?BzJAn|WiJ07!B7b?hMPp9C#yZHlzQNxv zd?u{rh~`%>&!`q#?V%N3IA_v|BLZ(iqn6sXuQmt~9gow|pt%n){|7en^uh4ps zv~v=<-YxE_Ww3H5-9C# zv?WtYzM+_4fm;~#HO|v!8MYt24^m4>(Ysd#W z;^AzR!3vx?1OZ>OS_M{EVzxgT-($2 zXD#r$V|m0@4#PjeOt1t$nS1PX(>A z0fQVQ>h*bp+1B>*(cvD^%4Lmm{AauN0chfeyMJ0g z?v(_m&)3iIU7CeigAUk4E{PeqhM;zLHmXW2gJ=Rx=U!VJFUr|apS>o7r*_quNSjT< zH9=|^VNLvtN#GJZE4URuf@U^PhJ#~1y)Yd`ul&zX);5YQ^+9^}-1;zKXqU)nUYhRrFBQy2CcK9E)C;X=0AlnMivYL`3GVp5wPA?ck1b9`Cu_g9?HI@=y89l0uD6>u zFY2>g9E+SF40e2F=ucBn9!JQ<8P9Dm%ib8yD`_qqj1t-{)$Y$X3$Q$M(O1s1-xZZ_ zZ^=f#E``q=@CvVytt=^bL}PBFY^oT&B&R`|XR^~4=PLwgdxmSwQo3}*Dw;pB@N8FU zUh=$hVpvO4baVTI2Mllf#p8Dy5?LAg1Fgo73h#!D4U@%-q>JjpPAYW!xEuy(VAFN> z-3-n$00AbSVCjASx=r_GD6+7;)jJK*!Aki{INbD-m~3VyO`0K)50rXDmnA{>2M?^M zlou~QLar2ZoOkZ@3RiV9M7jY$GiSUVyFKX)JvQILiR5dkuWE)W3&k?d^jP60&6oP- z%E1v%bhv~aOGK;pp~1%xrYfYNKqbI@K47@|nDk3_ctEIM-h}r1JIg2dpmth-r#B*FdL?ffLG_LR(;+8bp!ep+Vl1TfiZZfvI2HHnP zMTJWA`GV+Z6gPV78~bM+--3O;3blWTv=VYNhWPa=W3Jd_CXj#rla3OtJ3k!uyXPkA f!$!(aTQNzfjw&)3o{52=`SwxtJR!^H{h=3P{{)qb literal 0 HcmV?d00001 diff --git a/.circleci/push_sck_artifacts.sh b/.circleci/push_sck_artifacts.sh index 516c7d80..e481fc66 100755 --- a/.circleci/push_sck_artifacts.sh +++ b/.circleci/push_sck_artifacts.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash set -e -sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 -echo "Installing aws cli..." -sudo pip install awscli > /dev/null 2>&1 echo "Pushing SCK artifacts to s3..." aws s3 cp helm-artifacts/splunk-kubernetes-logging*.tgz s3://k8s-ci-artifacts/ aws s3 cp helm-artifacts/splunk-kubernetes-metrics*.tgz s3://k8s-ci-artifacts/ diff --git a/.circleci/sck_values.yml b/.circleci/sck_values.yml new file mode 100755 index 00000000..ec2fa8b8 --- /dev/null +++ b/.circleci/sck_values.yml @@ -0,0 +1,40 @@ +global: + logLevel: info + splunk: + hec: + protocol: https + insecureSSL: true + host: $SPLUNK_HEC_HOST + token: $SPLUNK_HEC_TOKEN + +splunk-kubernetes-logging: + kubernetes: + insecureSSL: true + splunk: + hec: + indexName: circleci_events + image: + name: + pullPolicy: Always + +splunk-kubernetes-metrics: + kubernetes: + useRestClientSSL: false + insecureSSL: true + kubeletPort: 10255 + splunk: + hec: + indexName: circleci_metrics + image: + name: + pullPolicy: Always + +splunk-kubernetes-objects: + splunk: + hec: + indexName: circleci_objects + kubernetes: + insecureSSL: true + image: + name: + pullPolicy: Always \ No newline at end of file diff --git a/.circleci/update_sck_config.sh b/.circleci/update_sck_config.sh new file mode 100755 index 00000000..3e0fdd13 --- /dev/null +++ b/.circleci/update_sck_config.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e +# Install yq yaml parser +wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 +sudo chmod +x yq_linux_amd64 +sudo mv yq_linux_amd64 /usr/local/bin/yq + +# Modify splunk environment values +yq w -i .circleci/sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST +yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN + +#Pull docker images locally +aws ecr get-login --region $AWS_REGION --no-include-email | bash +docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-logging:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' +docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-metrics:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' +docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-metrics-agg:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' +docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-objects:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' + +# Modify docker images to be used +image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-logging:latest") +yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name $image_name +image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics:latest") +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name $image_name +image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics-agg:latest") +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name $image_name +image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-objects:latest") +yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name $image_name \ No newline at end of file From 99151ed64fa6acf8b30d7220a5f0a3e8619eedcc Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 28 Jan 2019 11:57:02 -0700 Subject: [PATCH 010/119] Custom buffer implementation (#60) Implementation to add support for file based buffering in fluentd. - Added daemonset, new fluentd plugin, less permissive rbac, doc fixes - Added aggregator support - make fluentd buffer customizable --- .../templates/configMap.yaml | 16 ++++------ .../splunk-kubernetes-logging/values.yaml | 20 +++++++++++- .../templates/configMap.yaml | 15 ++++----- .../templates/configMapMetricsAggregator.yaml | 15 ++++----- .../splunk-kubernetes-metrics/values.yaml | 31 ++++++++++++++++++- .../templates/configMap.yaml | 13 ++++---- .../splunk-kubernetes-objects/values.yaml | 18 +++++++++++ 7 files changed, 91 insertions(+), 37 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 48e4d856..4130f164 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -90,7 +90,7 @@ data: expression /^(?{{ $logDef.timestampExtraction.regexp }}.*)$/ time_key time time_type string - time_format {{ $logDef.timestampExtraction.format }} + time_format {{ $logDef.timestampExtraction.format }} {{- else }} @type none message_key log @@ -229,17 +229,13 @@ data: container_name container_id + {{- with .Values.buffer }} - @type memory - {{- with .Values.resources.limit }} - chunk_limit_size {{ if .memory }}{{ template "splunk-kubernetes-logging.convert-memory" .memory }}{{ else }}{{ "500m" }}{{ end }} - {{- end }} - chunk_limit_records 100000 - flush_interval 5s - flush_thread_count 1 - overflow_action block - retry_max_times 3 + {{- range $parameter, $value := . }} + {{ $parameter }} {{ $value }} + {{- end }} + {{- end }} @type json diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index a4bd9156..23419f5e 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -85,7 +85,7 @@ journalLogPath: /run/log/journal # journald: # unit: docker.service # ``` -# +# # For `file` logs, `path` is required for specifying where is the log files. Log files are expected in `/var/log`, example: # ``` # docker: @@ -227,6 +227,24 @@ resources: memory: 200Mi + # Controls the output buffer for the fluentd daemonset + # Note that, for memory buffer, if `resources.limits.memory` is set, + # the total buffer size should not bigger than the memory limit, it should also + # consider the basic memory usage by fluentd itself. + # All buffer parameters (except Argument) defined in + # https://docs.fluentd.org/v1.0/articles/buffer-section#parameters + # can be configured here. +buffer: + "@type": memory + total_limit_size: 600m + chunk_limit_size: 200m + chunk_limit_records: 100000 + flush_interval: 5s + flush_thread_count: 1 + overflow_action: block + retry_max_times: 3 + + # This default tolerations allow the daemonset to be deployed on master nodes, # so that we can also collect logs from those nodes. tolerations: diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index 754979cf..f270b8a7 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -91,15 +91,12 @@ data: {{- end }} {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file - {{- end }} - interval {{ or .Values.metricsInterval .Values.global.metricsInterval }} + {{- end }} + {{- with .Values.buffer }} - @type memory - total_limit_size 100m - chunk_limit_size 10m - flush_interval 15s - flush_thread_count 1 - overflow_action block - retry_max_times 3 + {{- range $parameter, $value := . }} + {{ $parameter }} {{ $value }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index fb4e7a37..5fa0a420 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -87,14 +87,11 @@ data: {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} - interval {{ or .Values.metricsInterval .Values.global.metricsInterval }} + {{- with .Values.aggregatorBuffer }} - @type memory - total_limit_size 100m - chunk_limit_size 10m - flush_interval 15s - flush_thread_count 1 - overflow_action block - retry_max_times 3 + {{- range $parameter, $value := . }} + {{ $parameter }} {{ $value }} + {{- end }} - + {{- end }} + \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 9965e1bf..37f14fa9 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -8,7 +8,7 @@ # Global values have lower priority. # Values defined here are the default values. global: - logLevel: info + logLevel: debug splunk: hec: port: 8088 @@ -93,6 +93,35 @@ resources: memory: 300Mi + # Controls the output buffer for fluentd. + # Note that, for memory buffer, if `resources.sidecar.limits.memory` is set, + # the total buffer size should not bigger than the memory limit, it should also + # consider the basic memory usage by fluentd itself. + # All buffer parameters (except Argument) defined in + # https://docs.fluentd.org/v1.0/articles/buffer-section#parameters + # can be configured here. +buffer: + "@type": memory + total_limit_size: 400m + chunk_limit_size: 100m + chunk_limit_records: 1000000 + flush_interval: 5s + flush_thread_count: 1 + overflow_action: block + retry_max_times: 3 + + +aggregatorBuffer: + "@type": memory + total_limit_size: 400m + chunk_limit_size: 100m + chunk_limit_records: 1000000 + flush_interval: 5s + flush_thread_count: 1 + overflow_action: block + retry_max_times: 3 + + # Configure how often SCK pulls metrics for its kubenetes sources. 15s is the default where 's' is seconds. metricsInterval: 15s diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index 3b9031ae..8a3cf04a 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -45,7 +45,7 @@ data: {{- if $.Values.checkpointFile.volume }} path /fluentd/var/checkpoints/{{ .Values.checkpointFile.name | default "kubernetes-objects" }} - + {{- end }} {{- range $objects }} {{- if eq (.mode | default "pull" | lower) "pull" }} @@ -108,12 +108,11 @@ data: {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} + {{- with .Values.buffer }} - @type memory - {{- $limit := .Values.resources.limit }} - chunk_limit_size {{ if $limit.memory }}{{ template "splunk-kubernetes-objects.convert-memory" $limit.memory }}{{ else }}{{ "200m" }}{{ end }} - chunk_limit_records 10000 - flush_interval 3s - retry_max_times 3 + {{- range $parameter, $value := . }} + {{ $parameter }} {{ $value }} + {{- end }} + {{- end }} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 12de1c51..6a676a89 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -178,6 +178,24 @@ resources: memory: 200Mi + # Controls the output buffer for the fluentd daemonset + # Note that, for memory buffer, if `resources.limits.memory` is set, + # the total buffer size should not bigger than the memory limit, it should also + # consider the basic memory usage by fluentd itself. + # All buffer parameters (except Argument) defined in + # https://docs.fluentd.org/v1.0/articles/buffer-section#parameters + # can be configured here. +buffer: + "@type": memory + total_limit_size: 600m + chunk_limit_size: 200m + chunk_limit_records: 10000 + flush_interval: 3s + flush_thread_count: 1 + overflow_action: block + retry_max_times: 3 + + nodeSelector: {} From 47d0b6e10f799f9997803de62380a5f702ea9590 Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Mon, 28 Jan 2019 15:08:43 -0800 Subject: [PATCH 011/119] Initial Added functional tests setup (pytest framework) and initial basic test for k8s-logging (#81) --- .circleci/config.yml | 17 ++- .circleci/deploy_connector.sh | 5 +- .circleci/run_functional_tests.sh | 11 ++ test/README.md | 26 ++++ test/__init__.py | 0 test/common.py | 177 +++++++++++++++++++++++++++ test/conftest.py | 40 ++++++ test/k8s_logging_tests/__init__.py | 0 test/k8s_logging_tests/test_index.py | 37 ++++++ test/requirements.txt | 2 + 10 files changed, 311 insertions(+), 4 deletions(-) create mode 100755 .circleci/run_functional_tests.sh create mode 100644 test/README.md create mode 100644 test/__init__.py create mode 100644 test/common.py create mode 100644 test/conftest.py create mode 100644 test/k8s_logging_tests/__init__.py create mode 100644 test/k8s_logging_tests/test_index.py create mode 100644 test/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 458ddf48..b2d2d5bd 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,9 +13,12 @@ jobs: - run: name: Install dependencies command: | + sudo apt-get update + sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1 + sudo apt-get install python3-pip python3-dev > /dev/null 2>&1 + sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash helm init -c - sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 echo "Installing aws cli..." sudo pip install awscli > /dev/null 2>&1 @@ -38,3 +41,15 @@ jobs: name: Deploy k8s connector command: | .circleci/deploy_connector.sh + + - run: + name: Run functional tests + command: | + .circleci/run_functional_tests.sh + + - run: + name: Teardown deployment + command: | + helm delete --purge ci-$CIRCLE_SHA1 + + diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh index 851731ec..e7960f88 100755 --- a/.circleci/deploy_connector.sh +++ b/.circleci/deploy_connector.sh @@ -10,6 +10,5 @@ echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt .circleci/kubeco sudo mv config ~/.kube/config helm install --name=ci-$CIRCLE_SHA1 -f .circleci/sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz #Todo-need to add test cases and run them before cleaning the deployment -sleep 60 -echo "clean up k8s-connect deployment..." -helm delete --purge ci-$CIRCLE_SHA1 \ No newline at end of file +#wait for deployment to finish +sleep 30 \ No newline at end of file diff --git a/.circleci/run_functional_tests.sh b/.circleci/run_functional_tests.sh new file mode 100755 index 00000000..477ea3b2 --- /dev/null +++ b/.circleci/run_functional_tests.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e +cd test +sudo pip3 install -r requirements.txt +#Run pytests +echo "Running functional tests....." +python3 -m pytest \ + --splunkd-url https://$SPLUNK_HEC_HOST:8089 \ + --splunk-user admin \ + --splunk-password $SPLUNK_PASSWORD \ + -p no:warnings \ No newline at end of file diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..19d958e3 --- /dev/null +++ b/test/README.md @@ -0,0 +1,26 @@ + +# Prerequsite +* Python version must be > 3.x + +# Testing Instructions +0. (Optional) Use a virtual environment for the test + `virtualenv --python=python3.6 venv` + `source venv/bin/activate` +1. Install the dependencies + `pip install -r requirements.txt` +2. Start the test with the required options configured + `python -m pytest ` + + **Options are:** + --splunkd-url + * Description: splunkd url used to send test data to. Eg: https://localhost:8089 + * Default: https://localhost:8089 + + --splunk-user + * Description: splunk username + * Default: admin + + --splunk-password + * Description: splunk user password + * Default: changeme + diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/common.py b/test/common.py new file mode 100644 index 00000000..7ce43b92 --- /dev/null +++ b/test/common.py @@ -0,0 +1,177 @@ +""" +Copyright 2018 Splunk, Inc.. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import json +import logging +import time +import requests +import os +import sys +from requests.packages.urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter + + +TIMEROUT = 500 + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s - %(name)s -' + + ' %(levelname)s - %(message)s') +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(formatter) +logger.addHandler(handler) + + +def check_events_from_splunk(index="circleci_events", + start_time="-1h@h", + end_time="now", + url="", + user="", + password=""): + ''' + send a search request to splunk and return the events from the result + ''' + query = _compose_search_query(index) + logger.info("search query = " + str(query)) + events = _collect_events(query, start_time, end_time, url, user, password) + + return events + + +def _compose_search_query(index="circleci_events"): + return "search index={0}".format(index) + + +def _collect_events(query, start_time, end_time, url="", user="", password=""): + ''' + Collect events by running the given search query + @param: query (search query) + @param: start_time (search start time) + @param: end_time (search end time) + returns events + ''' + + search_url = '{0}/services/search/jobs?output_mode=json'.format( + url) + logger.debug('requesting: %s', search_url) + data = { + 'search': query, + 'earliest_time': start_time, + 'latest_time': end_time, + } + + create_job = _requests_retry_session().post( + search_url, + auth=(user, password), + verify=False, data=data) + _check_request_status(create_job) + + json_res = create_job.json() + job_id = json_res['sid'] + events = _wait_for_job_and__get_events(job_id, url, user, password) + + return events + + +def _wait_for_job_and__get_events(job_id, url="", user="", password=""): + ''' + Wait for the search job to finish and collect the result events + @param: job_id + returns events + ''' + events = [] + job_url = '{0}/services/search/jobs/{1}?output_mode=json'.format( + url, str(job_id)) + logger.debug('requesting: %s', job_url) + + for _ in range(TIMEROUT): + res = _requests_retry_session().get( + job_url, + auth=(user, password), + verify=False) + _check_request_status(res) + + job_res = res.json() + dispatch_state = job_res['entry'][0]['content']['dispatchState'] + + if dispatch_state == 'DONE': + events = _get_events(job_id, url, user, password) + break + if dispatch_state == 'FAILED': + raise Exception('Search job: {0} failed'.format(job_url)) + time.sleep(1) + + return events + + +def _get_events(job_id, url="", user="", password=""): + ''' + collect the result events from a search job + @param: job_id + returns events + ''' + event_url = '{0}/services/search/jobs/{1}/events?output_mode=json'.format( + url, str(job_id)) + logger.debug('requesting: %s', event_url) + + event_job = _requests_retry_session().get( + event_url, auth=(user, password), + verify=False) + _check_request_status(event_job) + + event_job_json = event_job.json() + events = event_job_json['results'] + logger.info("Events from get_events method returned %s events", + len(events)) + + return events + + +def _check_request_status(req_obj): + ''' + check if a request is successful + @param: req_obj + returns True/False + ''' + if not req_obj.ok: + raise Exception('status code: {0} \n details: {1}'.format( + str(req_obj.status_code), req_obj.text)) + + +def _requests_retry_session( + retries=10, + backoff_factor=0.1, + status_forcelist=(500, 502, 504)): + ''' + create a retry session for HTTP/HTTPS requests + @param: retries (num of retry time) + @param: backoff_factor + @param: status_forcelist (list of error status code to trigger retry) + @param: session + returns: session + ''' + session = requests.Session() + retry = Retry( + total=int(retries), + backoff_factor=backoff_factor, + method_whitelist=frozenset(['GET', 'POST']), + status_forcelist=status_forcelist, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + + return session diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..d8e50a6a --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,40 @@ +""" +Copyright 2018 Splunk, Inc.. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest +import time + +def pytest_addoption(parser): + parser.addoption("--splunkd-url", + help="splunkd url used to send test data to. \ + Eg: https://localhost:8089", + default="https://localhost:8089") + parser.addoption("--splunk-user", + help="splunk username", + default="admin") + parser.addoption("--splunk-password", + help="splunk user password", + default="password") + +@pytest.fixture(scope="function") +def setup(request): + config = {} + config["splunkd_url"] = request.config.getoption("--splunkd-url") + config["splunk_user"] = request.config.getoption("--splunk-user") + config["splunk_password"] = request.config.getoption("--splunk-password") + + + return config diff --git a/test/k8s_logging_tests/__init__.py b/test/k8s_logging_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/k8s_logging_tests/test_index.py b/test/k8s_logging_tests/test_index.py new file mode 100644 index 00000000..da0cfa5f --- /dev/null +++ b/test/k8s_logging_tests/test_index.py @@ -0,0 +1,37 @@ +import pytest +import time +import os +import logging +import json +from urllib.parse import urlparse +from ..common import check_events_from_splunk + + +@pytest.mark.parametrize("test_input,expected", [ + ("circleci_events", 1) +]) +def test_splunk_index(setup, test_input, expected): + ''' + Test that user specified index can successfully index the + log stream from k8s. If no index is specified, default + index "circleci_events" will be used. + ''' + logging.getLogger().info("testing test_splunk_index input={0} \ + expected={1} event(s)".format(test_input, expected)) + index = test_input if test_input else "circleci_events" + + events = check_events_from_splunk(index=index, + start_time="-34h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received %s events in the last minute", + len(events)) + assert len(events) != "" + + + + + + + diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 00000000..16d3b778 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,2 @@ +pytest +requests \ No newline at end of file From 8bea46ac8ed8d2699d3fa6e18007fedf9f358a64 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 28 Jan 2019 17:00:01 -0700 Subject: [PATCH 012/119] Added cluster name field to logs, metrics and objects and made it configurable (#80) --- .../splunk-kubernetes-logging/templates/_helpers.tpl | 5 ++++- .../splunk-kubernetes-logging/templates/configMap.yaml | 3 +++ helm-chart/splunk-kubernetes-logging/values.yaml | 6 ++++++ .../splunk-kubernetes-metrics/templates/configMap.yaml | 9 +++++++-- .../templates/configMapMetricsAggregator.yaml | 3 +++ helm-chart/splunk-kubernetes-metrics/values.yaml | 2 ++ .../splunk-kubernetes-objects/templates/configMap.yaml | 10 ++++++++++ helm-chart/splunk-kubernetes-objects/values.yaml | 3 ++- 8 files changed, 37 insertions(+), 4 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index cff71740..e8056fa9 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -92,7 +92,10 @@ def extract_container_info: | .namespace = $parts[1] | .container_name = ($cparts[:-1] | join("-")) | .container_id = ($cparts[-1] | rtrimstr(".log")) + {{- if .Values.kubernetes.clusterName }} + | .cluster_name = "{{ .Values.kubernetes.clusterName }}" + {{- end }} | .; - + .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // "kube:container:\(.container_name)") {{- end -}} diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 4130f164..2403df45 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -228,6 +228,9 @@ data: namespace container_name container_id + {{- if .Values.kubernetes.clusterName }} + cluster_name + {{- end }} {{- with .Values.buffer }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 23419f5e..da860d24 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -258,3 +258,9 @@ nodeSelector: {} # Defines node affinity to restrict pod deployment. affinity: {} + + +# = Kubernetes Connection Configs = +kubernetes: + # The cluster name used to tag logs. Default is cluster_name + clusterName: "cluster_name" diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index f270b8a7..be503a92 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -9,7 +9,7 @@ metadata: heritage: {{ .Release.Service }} data: fluent.conf: | - # system wide configurations + # system wide configurations log_level {{ or .Values.logLevel .Values.global.logLevel | default "info" }} @@ -33,11 +33,17 @@ data: {{- with .Values.kubernetes.secretDir }} secret_dir {{ . }} {{- end }} + {{- with .Values.kubernetes.clusterName }} + cluster_name {{ . }} + {{- end }} @type record_modifier metric_name ${tag} + {{- with .Values.kubernetes.clusterName }} + cluster_name {{ . }} + {{- end }} @@ -98,5 +104,4 @@ data: {{ $parameter }} {{ $value }} {{- end }} - {{- end }} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index 5fa0a420..2068935e 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -24,6 +24,9 @@ data: @type record_modifier metric_name ${tag} + {{- with .Values.kubernetes.clusterName }} + cluster_name {{ . }} + {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 37f14fa9..ede1ba63 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -153,3 +153,5 @@ kubernetes: bearerTokenFile: # Path of the location where pod's service account's credentials are stored. Usually you don't need to care about this config, the default value should work in most cases. secretDir: + # The cluster name used to tag cluster metrics from the aggregator. Default is cluster_name + clusterName: "cluster_name" diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index 8a3cf04a..bd036f5e 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -82,6 +82,11 @@ data: jq '.record.source = "namespace:\(env.MY_NAMESPACE)/pod:\(env.MY_POD_NAME)" | .record.sourcetype = (.tag | gsub("\\\\."; ":")) | .record' + + @type jq_transformer + jq '.record.cluster_name = "{{ .Values.kubernetes.clusterName }}" | .record' + + @type splunk_hec protocol {{ or .Values.splunk.hec.protocol .Values.global.splunk.hec.protocol | default "https" }} @@ -108,6 +113,11 @@ data: {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} + + {{- if .Values.kubernetes.clusterName }} + cluster_name + {{- end }} + {{- with .Values.buffer }} {{- range $parameter, $value := . }} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 6a676a89..e9aab0f6 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -61,7 +61,8 @@ kubernetes: bearerTokenFile: # Path of the location where pod's service account's credentials are stored. Usually you don't need to care about this config, the default value should work in most cases. secretDir: - # a list of kubernetes objects to pull periodically + # The cluster name used to tag cluster metrics from the aggregator. Default is cluster_name + clusterName: "cluster_name" # = Object Lists = From dfc186bbe1c0633588450f25d269f78cd10a0c1b Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Mon, 28 Jan 2019 16:27:57 -0800 Subject: [PATCH 013/119] fix configmap for metrics (#82) --- helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index be503a92..fe9285ef 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -104,4 +104,5 @@ data: {{ $parameter }} {{ $value }} {{- end }} + {{- end }} \ No newline at end of file From 96df23c52c4af454d3671b92682f8dd8f8a03acc Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 29 Jan 2019 18:07:45 -0700 Subject: [PATCH 014/119] Index Routing - send logs to an indexer with the same name as namespace (#61) --- .../splunk-kubernetes-logging/templates/_helpers.tpl | 10 +++++++++- .../splunk-kubernetes-logging/templates/configMap.yaml | 4 ++++ helm-chart/splunk-kubernetes-logging/values.yaml | 10 +++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index e8056fa9..cd8d8fc4 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -85,11 +85,19 @@ elif startswith({{ list (or .from.container .name) .from.pod | join "/" | quote else empty end; +def set_namespace(value): +if value == "default" +then +{{- $index := or .Values.splunk.hec.indexRoutingDefaultIndex .Values.global.splunk.hec.indexRoutingDefaultIndex | default "main" | quote}} +{{- printf " %s" $index -}} +else value +end; + def extract_container_info: (.source | ltrimstr("/var/log/containers/") | split("_")) as $parts | ($parts[-1] | split("-")) as $cparts | .pod = $parts[0] - | .namespace = $parts[1] + | .namespace = set_namespace($parts[1]) | .container_name = ($cparts[:-1] | join("-")) | .container_id = ($cparts[-1] | rtrimstr(".log")) {{- if .Values.kubernetes.clusterName }} diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 2403df45..c32f6a93 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -210,9 +210,13 @@ data: host "#{ENV['SPLUNK_HEC_HOST']}" source_key source sourcetype_key sourcetype + {{- if or .Values.splunk.hec.indexRouting .Values.global.splunk.hec.indexRouting }} + index_key namespace + {{- else }} {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} index {{ . }} {{- end }} + {{- end }} insecure_ssl {{ or .Values.splunk.hec.insecureSSL .Values.global.splunk.hec.insecureSSL | default false }} {{- if or .Values.splunk.hec.clientCert .Values.global.splunk.hec.clientCert }} client_cert /fluentd/etc/splunk/hec_client_cert diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index da860d24..75ed7df0 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -14,6 +14,10 @@ global: hec: protocol: https insecureSSL: false + # indexRouting is a boolean, it indicates whether user wants to route logs to the index with name same as the namespace. Default is false. + indexRouting: + # indexRoutingDefaultIndex tells which index to use for the default kubenetes namespace. Used with indexRouting. Default is main. + indexRoutingDefaultIndex: # logLevel is to set log level of the Splunk log collector. Avaiable values are: @@ -39,7 +43,7 @@ splunk: protocol: # indexName tells which index to use, this is optional. If it's not present, will use the default one configured in HEC. indexName: - # insecureSSL is a boolean, it indecates should it allow inscure SSL connection (when protocol is "https"). Default is false. + # insecureSSL is a boolean, it indicates should it allow inscure SSL connection (when protocol is "https"). Default is false. insecureSSL: # The PEM-format CA certificate for this client. # NOTE: The content of the certificate itself should be used here, not the file path. @@ -53,6 +57,10 @@ splunk: # NOTE: The content of the file itself should be used here, not the file path. # The file will be stored as a secret in kubernetes. caFile: + # indexRouting is a boolean, it indicates whether user wants to route logs to the index with name same as the namespace. Default is false. + indexRouting: + # indexRoutingDefaultIndex tells which index to use for the default kubenetes namespace. Used with indexRouting. Default is main. + indexRoutingDefaultIndex: # Directory where to read journald logs. From 8fe628383a6250e5aeb3abf117f5a7c01b4fb0e1 Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Wed, 30 Jan 2019 14:59:49 -0800 Subject: [PATCH 015/119] ADDON-21053:added cluster_name functional test (#83) --- .circleci/sck_values.yml | 1 + test/common.py | 9 ++--- test/conftest.py | 4 +-- test/k8s_logging_tests/test_config.py | 49 +++++++++++++++++++++++++++ test/k8s_logging_tests/test_index.py | 37 -------------------- 5 files changed, 54 insertions(+), 46 deletions(-) create mode 100644 test/k8s_logging_tests/test_config.py delete mode 100644 test/k8s_logging_tests/test_index.py diff --git a/.circleci/sck_values.yml b/.circleci/sck_values.yml index ec2fa8b8..15681ccf 100755 --- a/.circleci/sck_values.yml +++ b/.circleci/sck_values.yml @@ -10,6 +10,7 @@ global: splunk-kubernetes-logging: kubernetes: insecureSSL: true + clusterName: "circleci-k8s-cluster" splunk: hec: indexName: circleci_events diff --git a/test/common.py b/test/common.py index 7ce43b92..664da000 100644 --- a/test/common.py +++ b/test/common.py @@ -1,5 +1,5 @@ """ -Copyright 2018 Splunk, Inc.. +Copyright 2018-2019 Splunk, Inc.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -40,21 +40,16 @@ def check_events_from_splunk(index="circleci_events", end_time="now", url="", user="", + query="", password=""): ''' send a search request to splunk and return the events from the result ''' - query = _compose_search_query(index) logger.info("search query = " + str(query)) events = _collect_events(query, start_time, end_time, url, user, password) return events - -def _compose_search_query(index="circleci_events"): - return "search index={0}".format(index) - - def _collect_events(query, start_time, end_time, url="", user="", password=""): ''' Collect events by running the given search query diff --git a/test/conftest.py b/test/conftest.py index d8e50a6a..0c75f5cc 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,5 @@ """ -Copyright 2018 Splunk, Inc.. +Copyright 2018-2019 Splunk, Inc.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -37,4 +37,4 @@ def setup(request): config["splunk_password"] = request.config.getoption("--splunk-password") - return config + return config \ No newline at end of file diff --git a/test/k8s_logging_tests/test_config.py b/test/k8s_logging_tests/test_config.py new file mode 100644 index 00000000..d3b9d8e0 --- /dev/null +++ b/test/k8s_logging_tests/test_config.py @@ -0,0 +1,49 @@ +import pytest +import time +import os +import logging +import json +from urllib.parse import urlparse +from ..common import check_events_from_splunk + + +@pytest.mark.parametrize("test_input,expected", [ + ("test_data", 1) +]) +def test_splunk_index(setup, test_input, expected): + ''' + Test that user specified index can successfully index the + log stream from k8s. If no index is specified, default + index "circleci_events" will be used. + ''' + logging.getLogger().info("testing test_splunk_index input={0} \ + expected={1} event(s)".format(test_input, expected)) + search_query = "index=circleci_events" + events = check_events_from_splunk(start_time="-34h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search {0}".format(search_query)], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received %s events in the last minute", + len(events)) + assert len(events) >= expected + + +@pytest.mark.parametrize("test_input,expected", [ + ("circleci-k8s-cluster", 1) +]) +def test_cluster_name(setup, test_input, expected): + ''' + Clustername is a configurable parameter, test that user specified cluster-name is attached as a metadata to all the logs + ''' + logging.getLogger().info("testing test_clusterName input={0} \ + expected={1} event(s)".format(test_input, expected)) + search_query = "index=circleci_events cluster_name=circleci-k8s-cluster | head 1" + events = check_events_from_splunk(start_time="-24h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search {0}".format(search_query)], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received %s events in the last minute", + len(events)) + assert len(events) == expected diff --git a/test/k8s_logging_tests/test_index.py b/test/k8s_logging_tests/test_index.py deleted file mode 100644 index da0cfa5f..00000000 --- a/test/k8s_logging_tests/test_index.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest -import time -import os -import logging -import json -from urllib.parse import urlparse -from ..common import check_events_from_splunk - - -@pytest.mark.parametrize("test_input,expected", [ - ("circleci_events", 1) -]) -def test_splunk_index(setup, test_input, expected): - ''' - Test that user specified index can successfully index the - log stream from k8s. If no index is specified, default - index "circleci_events" will be used. - ''' - logging.getLogger().info("testing test_splunk_index input={0} \ - expected={1} event(s)".format(test_input, expected)) - index = test_input if test_input else "circleci_events" - - events = check_events_from_splunk(index=index, - start_time="-34h@h", - url=setup["splunkd_url"], - user=setup["splunk_user"], - password=setup["splunk_password"]) - logging.getLogger().info("Splunk received %s events in the last minute", - len(events)) - assert len(events) != "" - - - - - - - From 4c0c2f6d5538b9e83090e5ad45e13856ddd47130 Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Wed, 6 Feb 2019 13:20:17 -0800 Subject: [PATCH 016/119] updated aggr configmap to make indexName configurable (#85) --- .../templates/configMapMetricsAggregator.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index 2068935e..8af9fab8 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -79,6 +79,9 @@ data: {{- end }} hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" host "#{ENV['MY_NODE_NAME']}" + {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} + index {{ . }} + {{- end }} source source insecure_ssl {{ or .Values.splunk.hec.insecureSSL .Values.global.splunk.hec.insecureSSL | default false }} {{- if or .Values.splunk.hec.clientCert .Values.global.splunk.hec.clientCert }} From be98cd47b6dd5bdd758d79f5230d66468cf355df Mon Sep 17 00:00:00 2001 From: gp510 Date: Wed, 6 Feb 2019 13:51:59 -0800 Subject: [PATCH 017/119] First metrics functional test (#84) Addition of Metrics Functional Tests for all sources --- test/common.py | 44 ++++++++++++++++ test/k8s_metrics_tests/__init__.py | 0 test/k8s_metrics_tests/test_config.py | 76 +++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 test/k8s_metrics_tests/__init__.py create mode 100644 test/k8s_metrics_tests/test_config.py diff --git a/test/common.py b/test/common.py index 664da000..62fcfa5c 100644 --- a/test/common.py +++ b/test/common.py @@ -50,6 +50,21 @@ def check_events_from_splunk(index="circleci_events", return events +def check_metrics_from_splunk(index="circleci_metrics", + start_time="-1h@h", + end_time="now", + url="", + user="", + password="", + metric_name=""): + ''' + send a search api request to splunk to check for values associated with a given metric and dimension + ''' + logger.debug("Calling _collect_metrics ") + events = _collect_metrics(start_time, end_time, url, user, password, index, metric_name) + + return events + def _collect_events(query, start_time, end_time, url="", user="", password=""): ''' Collect events by running the given search query @@ -80,6 +95,35 @@ def _collect_events(query, start_time, end_time, url="", user="", password=""): return events +def _collect_metrics(start_time, end_time, url="", user="", password="", index="", metric_name=""): + ''' + Verify metrics by running the given api query + @param: dimension (metric dimension) + @param: metric_name (metric name) + @param: start_time (search start time) + @param: end_time (search end time) + returns events + ''' + api_url = url + '/services/catalog/metricstore/dimensions/host/values?filter=index%3d' + index + '&metric_name=' + metric_name + '&earliest=' + start_time + '&latest=' + end_time + '&output_mode=json'.format( + url) + + logger.debug('requesting: %s', api_url) + + create_job = _requests_retry_session().get( + api_url, + auth=(user, password), + verify=False + ) + + _check_request_status(create_job) + + json_res = create_job.json() + + events = json_res['entry'] + logger.info('events: %s', events) + + return events + def _wait_for_job_and__get_events(job_id, url="", user="", password=""): ''' diff --git a/test/k8s_metrics_tests/__init__.py b/test/k8s_metrics_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/k8s_metrics_tests/test_config.py b/test/k8s_metrics_tests/test_config.py new file mode 100644 index 00000000..50474d40 --- /dev/null +++ b/test/k8s_metrics_tests/test_config.py @@ -0,0 +1,76 @@ +import pytest +import time +import os +import logging +import json +from urllib.parse import urlparse +from ..common import check_events_from_splunk +from ..common import check_metrics_from_splunk + + +@pytest.mark.parametrize("metric", [ + #Aggregator Metrics + ("kube.container.cpu.limit"), + ("kube.container.memory.request"), + ("kube.pod.cpu.limit"), + ("kube.pod.memory.request"), + ("kube.namespace.cpu.limit"), + ("kube.namespace.memory.request"), + ("kube.cluster.cpu.limit"), + ("kube.cluster.memory.request"), + ("kube.node.cpu.capacity"), + ("kube.node.memory.utilization"), + #Summary Metrics + ("kube.node.cpu.usage"), + ("kube.node.memory.usage"), + ("kube.node.uptime"), + ("kube.node.network.rx_bytes"), + ("kube.node.fs.available_bytes"), + ("kube.node.imagefs.available_bytes"), + ("kube.node.runtime.imagefs.maxpid"), + ("kube.sys-container.cpu.usage"), + ("kube.sys-container.memory.usage_bytes"), + ("kube.sys-container.uptime"), + ("kube.pod.uptime"), + ("kube.pod.cpu.usage"), + ("kube.pod.memory.usage_bytes"), + ("kube.pod.network.rx_bytes"), + ("kube.pod.ephemeral-storage.available_bytes"), + ("kube.pod.volume.available_bytes"), + ("kube.container.uptime"), + ("kube.container.cpu.usage"), + ("kube.container.memory.usage_bytes"), + ("kube.container.rootfs.available_bytes"), + ("kube.container.logs.used_bytes"), + #Stats Metrics + ("kube.node.cpu.cfs.periods"), + ("kube.node.diskio.io_service_bytes.stats.Read"), + ("kube.node.filesystem.available"), + ("kube.node.memory.cache"), + ("kube.node.tasks_stats.nr_running"), + ("kube.node.network.*.rx_bytes"), + #Cadvisor Metrics + ("kube.container.cpu.load.average.10s"), + ("kube.container.fs.inodes.free"), + ("kube.container.last.seen"), + ("kube.container.memory.usage.bytes"), + ("kube.container.network.receive.bytes.total"), + ("kube.container.spec.cpu.period"), + ("kube.container.tasks.state") +]) +def test_metric_name(setup, metric): + ''' + This test covers one metric from each endpoint that the metrics plugin covers + ''' + logging.info("testing for presence of metric={0}".format(metric)) + + events = check_metrics_from_splunk(start_time="-24h@h", + end_time="now", + url=setup["splunkd_url"], + user=setup["splunk_user"], + password=setup["splunk_password"], + index="circleci_metrics", + metric_name=metric) + logging.info("Splunk received %s events in the last minute", + len(events)) + assert len(events) > 0 From 94b13bc968bcfd710d59ad26797847d52a268c46 Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Wed, 6 Feb 2019 16:32:32 -0800 Subject: [PATCH 018/119] added a logic to check and clean previous deployment (#88) --- .circleci/deploy_connector.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh index e7960f88..efbdf80f 100755 --- a/.circleci/deploy_connector.sh +++ b/.circleci/deploy_connector.sh @@ -8,7 +8,15 @@ sudo apt-get -y install gnupg sudo mkdir ~/.kube echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt .circleci/kubeconfig.gpg sudo mv config ~/.kube/config +#Make sure to check and clean previously failed deployment +echo "Checking if previous deployment exist..." +if [ "`helm ls`" == "" ]; then + echo "Nothing to clean, ready for deployment" +else + helm delete --purge $(helm ls --short) +fi +echo "Deploying k8s-connect with latest changes" helm install --name=ci-$CIRCLE_SHA1 -f .circleci/sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz #Todo-need to add test cases and run them before cleaning the deployment -#wait for deployment to finish -sleep 30 \ No newline at end of file +#wait for deployment to finish, sleep longer for metrics-aggr to start sending metrics +sleep 90 \ No newline at end of file From 9bfabe88f30a329e6290048c90ff0bec20f81420 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 11 Feb 2019 18:22:18 -0700 Subject: [PATCH 019/119] Added functional tests for namespace routing feature (#87) --- .circleci/run_functional_tests.sh | 1 + .circleci/sck_values.yml | 3 +- test/common.py | 70 +++++++++ test/k8s_logging_tests/test_config.py | 49 ------ test/k8s_logging_tests/test_config_logging.py | 140 ++++++++++++++++++ ...{test_config.py => test_config_metrics.py} | 0 test/requirements.txt | 3 +- 7 files changed, 215 insertions(+), 51 deletions(-) delete mode 100644 test/k8s_logging_tests/test_config.py create mode 100644 test/k8s_logging_tests/test_config_logging.py rename test/k8s_metrics_tests/{test_config.py => test_config_metrics.py} (100%) diff --git a/.circleci/run_functional_tests.sh b/.circleci/run_functional_tests.sh index 477ea3b2..0b87ceb4 100755 --- a/.circleci/run_functional_tests.sh +++ b/.circleci/run_functional_tests.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -e cd test +sudo pip3 install --upgrade pip sudo pip3 install -r requirements.txt #Run pytests echo "Running functional tests....." diff --git a/.circleci/sck_values.yml b/.circleci/sck_values.yml index 15681ccf..30fec1b1 100755 --- a/.circleci/sck_values.yml +++ b/.circleci/sck_values.yml @@ -13,7 +13,8 @@ splunk-kubernetes-logging: clusterName: "circleci-k8s-cluster" splunk: hec: - indexName: circleci_events + indexRouting: true + indexRoutingDefaultIndex: circleci_events image: name: pullPolicy: Always diff --git a/test/common.py b/test/common.py index 62fcfa5c..e58074fa 100644 --- a/test/common.py +++ b/test/common.py @@ -65,6 +65,76 @@ def check_metrics_from_splunk(index="circleci_metrics", return events + +def create_index_in_splunk(index="", + url="", + user="", + password=""): + ''' + Send a request to a Splunk instance to create an index + @param: index (index to be deleted) + @param: url (splunkd rest api url) + @param: user (splunk username) + @param: password (splunk password) + returns True/False + ''' + + search_url = '{0}/services/data/indexes/{1}?output_mode=json'.format(url, index) + logger.debug('requesting: %s', search_url) + data = { + 'name': index + } + + create_job = _requests_retry_session().post( + search_url, + auth=(user, password), + verify=False, data=data) + + if create_job.status_code == 201: + logger.info('The index: %s successfully created', index) + elif create_job.status_code == 409: + logger.info('The index: %s already exits', index) + else: + logger.info('The index: {0} not created, exit code is {1}'.format(index, create_job.status_code)) + return False + + return True + + +def delete_index_in_splunk(index="", + user="", + url="", + password=""): + ''' + Send a request to a Splunk instance to delete an index + @param: index (index to be deleted) + @param: url (splunkd rest api url) + @param: user (splunk username) + @param: password (splunk password) + returns True/False + ''' + + search_url = '{0}/services/data/indexes/{1}?output_mode=json'.format(url, index) + logger.debug('requesting: %s', search_url) + data = { + 'name': index + } + + create_job = _requests_retry_session().delete( + search_url, + auth=(user, password), + verify=False, data=data) + + if create_job.status_code == 200: + logger.info('The index: %s successfully deleted', index) + elif create_job.status_code == 409: + logger.info('The index: %s already disabled', index) + else: + return False + + return True + + def _collect_events(query, start_time, end_time, url="", user="", password=""): ''' Collect events by running the given search query diff --git a/test/k8s_logging_tests/test_config.py b/test/k8s_logging_tests/test_config.py deleted file mode 100644 index d3b9d8e0..00000000 --- a/test/k8s_logging_tests/test_config.py +++ /dev/null @@ -1,49 +0,0 @@ -import pytest -import time -import os -import logging -import json -from urllib.parse import urlparse -from ..common import check_events_from_splunk - - -@pytest.mark.parametrize("test_input,expected", [ - ("test_data", 1) -]) -def test_splunk_index(setup, test_input, expected): - ''' - Test that user specified index can successfully index the - log stream from k8s. If no index is specified, default - index "circleci_events" will be used. - ''' - logging.getLogger().info("testing test_splunk_index input={0} \ - expected={1} event(s)".format(test_input, expected)) - search_query = "index=circleci_events" - events = check_events_from_splunk(start_time="-34h@h", - url=setup["splunkd_url"], - user=setup["splunk_user"], - query=["search {0}".format(search_query)], - password=setup["splunk_password"]) - logging.getLogger().info("Splunk received %s events in the last minute", - len(events)) - assert len(events) >= expected - - -@pytest.mark.parametrize("test_input,expected", [ - ("circleci-k8s-cluster", 1) -]) -def test_cluster_name(setup, test_input, expected): - ''' - Clustername is a configurable parameter, test that user specified cluster-name is attached as a metadata to all the logs - ''' - logging.getLogger().info("testing test_clusterName input={0} \ - expected={1} event(s)".format(test_input, expected)) - search_query = "index=circleci_events cluster_name=circleci-k8s-cluster | head 1" - events = check_events_from_splunk(start_time="-24h@h", - url=setup["splunkd_url"], - user=setup["splunk_user"], - query=["search {0}".format(search_query)], - password=setup["splunk_password"]) - logging.getLogger().info("Splunk received %s events in the last minute", - len(events)) - assert len(events) == expected diff --git a/test/k8s_logging_tests/test_config_logging.py b/test/k8s_logging_tests/test_config_logging.py new file mode 100644 index 00000000..f404fdeb --- /dev/null +++ b/test/k8s_logging_tests/test_config_logging.py @@ -0,0 +1,140 @@ +import pytest +import time +import os +import logging +import json +from urllib.parse import urlparse +from ..common import check_events_from_splunk, create_index_in_splunk, delete_index_in_splunk +from kubernetes import client, config +from kubernetes.client.rest import ApiException + +@pytest.mark.parametrize("test_input,expected", [ + ("test_data", 1) +]) +def test_splunk_index(setup, test_input, expected): + ''' + Test that user specified index can successfully index the + log stream from k8s. If no index is specified, default + index "circleci_events" will be used. + ''' + logging.getLogger().info("testing test_splunk_index input={0} \ + expected={1} event(s)".format(test_input, expected)) + search_query = "index=circleci_events" + events = check_events_from_splunk(start_time="-1h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search {0}".format(search_query)], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received %s events in the last minute", + len(events)) + assert len(events) >= expected + + +@pytest.mark.parametrize("test_input,expected", [ + ("circleci-k8s-cluster", 1) +]) +def test_cluster_name(setup, test_input, expected): + ''' + Clustername is a configurable parameter, test that user specified cluster-name is attached as a metadata to all the logs + ''' + logging.getLogger().info("testing test_clusterName input={0} \ + expected={1} event(s)".format(test_input, expected)) + search_query = "index=circleci_events cluster_name=circleci-k8s-cluster | head 1" + events = check_events_from_splunk(start_time="-1h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search {0}".format(search_query)], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received %s events in the last minute", + len(events)) + assert len(events) == expected + +@pytest.mark.parametrize("test_input,expected", [ + ("test-namespace-routing", 1), + ("kube-system", 1), + ("kube-public", 1) +]) +def test_namespace_routing(setup, test_input, expected): + ''' + Test namespace routing feature. This test will create an index in Splunk with name as the namespace, start a pod to generate some logs, + then check if the logs are indexed as events in Splunk. The test handles the cleanup of the index and its events. + ''' + # Splunk index and namespace are assumed to be the same + index = test_input + namespace = test_input + + # Handle special cases of default namespaces kube-system and kube-public + if test_input == "kube-system" or test_input == "kube-public": + search_query = "index={0}".format(test_input) + events = check_events_from_splunk(index=index, + start_time="-1h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search {0}".format(search_query)], + password=setup["splunk_password"]) + logging.getLogger().info("Received {0} events in the index {1}".format(len(events), index)) + assert len(events) >= 0 + pytest.skip("Test successful, skipping rest of the test for special cases") + + # Initialize kubernetes python client + config.load_kube_config() + v1 = client.CoreV1Api() + found = False + + # Search for namespace + for ns in v1.list_namespace().items: + if test_input == ns.metadata.name: + found = True + + # Create namespace + if not found: + logging.getLogger().info("creating namespace") + try: + v1.create_namespace(client.V1Namespace(metadata=client.V1ObjectMeta(name=test_input))) + except ApiException as e: + logging.getLogger().info("Exception when calling CoreV1Api create_namespace: {0}".format(e)) + + search_query = "index={0} | delete".format(test_input) + events = check_events_from_splunk(index=index, + start_time="-1h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search {0}".format(search_query)], + password=setup["splunk_password"]) + logging.getLogger().info("Received {0} events in the index {1}".format(len(events), index)) + assert len(events) == 0 + + # Data generator image metadata + image_name = "cp-data-gen" + image_address = "chaitanyaphalak/kafkadatagen:1.0-4-gca7f6d8" + image_pull_policy = "IfNotPresent" + + # Create pod in the test namespace to generate logs + pod = client.V1Pod() + pod.metadata = client.V1ObjectMeta(name=image_name) + + container = client.V1Container(name=image_name, image=image_address, image_pull_policy=image_pull_policy) + + spec = client.V1PodSpec(containers=[container]) + pod.spec = spec + try: + v1.create_namespaced_pod(namespace=namespace, body=pod) + except ApiException as e: + logging.getLogger().info("Exception when calling CoreV1Api create_namespaced_pod: {0}".format(e)) + + logging.getLogger().info("Sleeping for 60 seconds") + time.sleep(60) + + # Check if we have those generated logs from kubernetes in Splunk + v1.delete_namespaced_pod(name=image_name, namespace=namespace, body=pod) + + search_query = "index={0}".format(test_input) + events = check_events_from_splunk(index=index, + start_time="-1h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search {0}".format(search_query)], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received {0} events in the last minute in the index {1}".format(len(events), index)) + + assert len(events) > 0 diff --git a/test/k8s_metrics_tests/test_config.py b/test/k8s_metrics_tests/test_config_metrics.py similarity index 100% rename from test/k8s_metrics_tests/test_config.py rename to test/k8s_metrics_tests/test_config_metrics.py diff --git a/test/requirements.txt b/test/requirements.txt index 16d3b778..a4382b8c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,3 @@ pytest -requests \ No newline at end of file +requests +kubernetes \ No newline at end of file From d795376a0d1114d422c50bd46e276dca9c2ab57d Mon Sep 17 00:00:00 2001 From: gp510 Date: Mon, 11 Feb 2019 18:30:12 -0800 Subject: [PATCH 020/119] Updated logging func test to include sourcetype and source tests (#91) --- test/k8s_logging_tests/test_config_logging.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/k8s_logging_tests/test_config_logging.py b/test/k8s_logging_tests/test_config_logging.py index f404fdeb..06d8595d 100644 --- a/test/k8s_logging_tests/test_config_logging.py +++ b/test/k8s_logging_tests/test_config_logging.py @@ -138,3 +138,52 @@ def test_namespace_routing(setup, test_input, expected): logging.getLogger().info("Splunk received {0} events in the last minute in the index {1}".format(len(events), index)) assert len(events) > 0 + +@pytest.mark.parametrize("test_input,expected", [ + ("kube:kube-apiserver", 1), + ("fluentd:monitor-agent", 1), + ("kube:container:splunk-fluentd-k8s-metrics-agg", 1), + ("kube:container:splunk-fluentd-k8s-metrics", 1), + ("kube:container:splunk-fluentd-k8s-logs", 1), + ("kube:dns-controller", 1), + ("kube:etcd", 1), + ("kube:container:splunk-fluentd-k8s-objects", 1), + ("kube:container:tiller", 1), + ("kube:kube-controller-manager", 1) +]) +def test_sourcetype(setup, test_input, expected): + ''' + Test that known sourcetypes are present in target index + ''' + logging.getLogger().info("testing for presence of sourcetype={0} \ + expected={1} event(s)".format(test_input, expected)) + events = check_events_from_splunk(start_time="-24h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search index=circleci_events sourcetype={0}".format(test_input)], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received %s events in the last minute", + len(events)) + assert len(events) >= expected + +@pytest.mark.parametrize("test_input,expected", [ + ("/var/log/containers/kube-apiserver-ip*", 1), + ("/var/log/containers/ci*", 1), + ("namespace:default/pod:ci*", 1), + ("/var/log/containers/dns-controller*", 1), + ("/var/log/containers/etcd-server-ip*", 1) +]) +def test_source(setup, test_input, expected): + ''' + Test that known sources are present in target index + ''' + logging.getLogger().info("testing for presence of sourcetype={0} \ + expected={1} event(s)".format(test_input, expected)) + events = check_events_from_splunk(start_time="-24h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["search index=circleci_events source={0}".format(test_input)], + password=setup["splunk_password"]) + logging.getLogger().info("Splunk received %s events in the last minute", + len(events)) + assert len(events) >= expected From 0e701c814b65676ad3bb4cba8e674bcc5ed1b274 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 12 Feb 2019 17:30:41 -0800 Subject: [PATCH 021/119] Update to sck_values to specify value for cluster_name (#93) --- .circleci/sck_values.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/sck_values.yml b/.circleci/sck_values.yml index 30fec1b1..4f8eb41e 100755 --- a/.circleci/sck_values.yml +++ b/.circleci/sck_values.yml @@ -24,6 +24,7 @@ splunk-kubernetes-metrics: useRestClientSSL: false insecureSSL: true kubeletPort: 10255 + clusterName: "circleci-k8s-cluster-metrics" splunk: hec: indexName: circleci_metrics From d29c255a6798f4067c611c9726a246d20e8ca0de Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 22 Feb 2019 11:04:31 -0700 Subject: [PATCH 022/119] Perf Test Framework - Added bash/pyhton script to deploy/clean SCK,to deploy/clean kafka data gen, run performance testing scenarios (#95) --- .circleci/config.yml | 55 +++++-- .circleci/deploy_connector.sh | 2 +- .circleci/performance/kubeconfig_perf.gpg | Bin 0 -> 3904 bytes .circleci/performance/perf_deploy_sck.sh | 152 ++++++++++++++++++ .../performance/perf_test_sck_values.yml | 91 +++++++++++ .circleci/performance/run_perf_scenarios.py | 151 +++++++++++++++++ .circleci/update_sck_config.sh | 11 +- 7 files changed, 451 insertions(+), 11 deletions(-) create mode 100644 .circleci/performance/kubeconfig_perf.gpg create mode 100755 .circleci/performance/perf_deploy_sck.sh create mode 100755 .circleci/performance/perf_test_sck_values.yml create mode 100644 .circleci/performance/run_perf_scenarios.py diff --git a/.circleci/config.yml b/.circleci/config.yml index b2d2d5bd..39407615 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,8 @@ version: 2 jobs: build: docker: - - image: circleci/ruby:2.4.1-node-browsers - + - image: circleci/ruby:2.6.1-node-browsers working_directory: ~/repo - steps: - checkout - setup_remote_docker: @@ -21,35 +19,74 @@ jobs: helm init -c echo "Installing aws cli..." sudo pip install awscli > /dev/null 2>&1 - - run: name: Update configurables to build artifacts command: | .circleci/update_sck_config.sh - - run: name: Build SCK artifacts command: | .circleci/build_sck_artifacts.sh - - run: name: Push SCK artifacts to s3 command: | .circleci/push_sck_artifacts.sh - - run: name: Deploy k8s connector command: | .circleci/deploy_connector.sh - - run: name: Run functional tests command: | .circleci/run_functional_tests.sh - - run: name: Teardown deployment command: | helm delete --purge ci-$CIRCLE_SHA1 + perf-test: + docker: + - image: circleci/ruby:2.6.1-node-browsers + working_directory: ~/repo + steps: + - checkout + - setup_remote_docker: + reusable: true + - run: + name: Install dependencies + command: | + sudo apt-get update + sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1 + sudo apt-get install python3-pip python3-dev > /dev/null 2>&1 + sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash + helm init -c + echo "Installing aws cli..." + sudo pip install awscli > /dev/null 2>&1 + - run: + name: Update configurables to build artifacts + command: | + .circleci/update_sck_config.sh + - run: + name: Build SCK artifacts + command: | + .circleci/build_sck_artifacts.sh + - run: + name: Deploy k8s connector for perf-tests + command: | + python .circleci/performance/run_perf_scenarios.py + +# Need to add teardown step +workflows: + version: 2 + build_perf-test: + jobs: + - build: + filters: + branches: + ignore: perf-testing + - perf-test: + filters: + branches: + only: perf-testing \ No newline at end of file diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh index efbdf80f..6649b2b7 100755 --- a/.circleci/deploy_connector.sh +++ b/.circleci/deploy_connector.sh @@ -6,7 +6,7 @@ chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl sudo apt-get -y install gnupg sudo mkdir ~/.kube -echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt .circleci/kubeconfig.gpg +echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg sudo mv config ~/.kube/config #Make sure to check and clean previously failed deployment echo "Checking if previous deployment exist..." diff --git a/.circleci/performance/kubeconfig_perf.gpg b/.circleci/performance/kubeconfig_perf.gpg new file mode 100644 index 0000000000000000000000000000000000000000..3a76ba52a17a467524d8dbac46eb711d1b04c0ec GIT binary patch literal 3904 zcmV-G55Mq?4Fm@R0y*gII(H<9RO8a?0sYN%4|o8i;)u8rz-_VfV&4y-sDu==C^xR< zYTYi8G!`3~tRy9kUbz|DW(nDg=c@}>X(>yjWsBvA?x*Tj5~k;%%~01z%=;``sTH_5 zFA>I{yX=%ur~w0@W2Bwdvm7M>4&RDXx=2aYb7XV{n!FV&eThC%=I^!M5Bc*h%K>WM zI(_@vJm1&|Or<{x$A+Z4SGSTQi0A^(Z~?@=ImI%h%K<@Yz8N50Khe4jKO_T=7$oXE z(yB_0o>t-#I<26>boqikG;_KX|nEaE+FQrUqi7$)%u%kf~ z3?o?)5h@?!EF!3;D7KLcGo8Q=%10TInde~|+zAP>oV76j@o67T6?Y)D%rYVpBhv!P z$|43>AZJiXB7s~*KvFvd*<~0oDH(=jO{s6{ zP(1=9GE^9bC|6S8Hvsr5i9NF-D4i-m;Gic7s#&F*{Vqfg68h7-5aB41+pQw7$lf7(djwNY4^PM2qoaCIv>or3r-D!$TveOPQ**p76AhE^Zf<%A3L&0l7r$ z14Uwm9Ir2?4n|k|GCJgVjp;O%IVi436|+j~~HcmF1j z(ny!fs9t_9cvv~0OeCHMaW;A*W*#4j@MoWyXm{}C z{#-EYjwuiuM`%__v`noRGckse=jXsQKBdp1)ldmqe%&k4Jk5vpQgc-%)6v?${M@W# zQ+(2fe{Mj1Fr+1RwadQBDE~9~eAJb;)(Sye(djwRf(+#%m&87V|JB3?m{z_u5mUuA^{jX$BIh|(}9`Tn`Mg{+Pm8Y-xrqz!0qXJ&eWm@4^y zK8BEc>AWOJvx^K#H3yKvM5E9{FKxp8gWEmm6V!;D z&_Atv)3`Ajm6%?{>z3>eGKJ&Wc!wi7RXWB0dKjP9e_~8{taXn)m+ZBN(89f?L5u0l z^@jo&)DWsRcW5ZCY!$?07pB}`dY+AcQfc-i` z7dHjZgPz5+s&8+Dv-d@2Gf})IfhfYi$&tmqYcZ?wWuzDBs%tenN`=?5EFAz4;GEi_ zl}0a9ZS_v(gO9Nk@r?%nv$LTm&CTrFQ9|Hph;P3NFiZ{fUIPyja?oFn&k=nWE!HV0 z`-DeF8;gLig|Ch__VomZ=uns@*y9SAY~&YhQx!6gd|-Hj{rnLI)~@%9%;%Neb+)gq zXOD@coX_^CM0D>x(h4tKZxs8nSB&5yn@x*FP5$7v>f`v^QP7+ni$jvqkCN(`^+_0gCRsIrO<+~ZX$QwKwcWh%O(HZ!JU(+zcO z*2$Q-ieERF_17&V+>V_Jr|F;8Otuqry+qj0Ve;RlN~hH(xB0!vd)y@@I&2UdS|L8ZzI-1CP%UKJw5>*Fy9ErWS{iJjqnCAJuV~6oPXw=V z@Y#y#*5A*M;+TJj_)1T(HyO8d<9f@*&nXFbD~5u<#qVjdUHpXz$vEwbHl(c!Leue> z*RiX$0^tdp5&t7FpXeLUX2V@;CVxE#C@ODNUEA*2lg)C*!EN-w$UE4bzL6Oo{$@@* zKiBEASh!4_d`8vj@y$V9+L7SQkMc)?W07=8&XLqp2$m8pjTPMHZP3caryzEPtH^{A z%qJsRF}fdbc!Q)EH-e`$m$dM*Y(B{LQS+n4-t8a_DxGs>aR z75TS&Lt3d&w8Ri0O*SsI2NHM?y?5Tv6c$DjG7$3BWE8aOu`Vx9U_tM)Apem6R$^*d zA+D?}mDE^NMxEN6O8`k^#D=Jh!&zO(w^DfaV2iWO4OIqav-Z6wWZzuQVV7Q=Vithd zR>V2ca3yJ|)ATZ*4@2Tz#Ms#byarQW4te86jM`lzDLZ!_8x-M@c7>%C6G*85?V>J! z@-w_-&h%PnfyuDnrIdt&z;@6eaqAeir*o%Ni5wwEY<{Q%jv!BwU{>?+R=m zYByetjGfOPUQv)p73dBqH%eBrMNlaT3!!vjpzuTX)I0d*>{ScDvHke5lmxfC3qY8g zEr`l^A57Tnz|Z?tSI~JXR03?8SLO;o#g$5o|EXCO%~F;AlLv zG6JH=RzHo?dSM51L=w~$a6wz{%fV4o(8DLWBUK%CEc)n`ulkvsvpCL_ha@!HC=ZhE z4d!Cbk;U+HkIK(4jkcr}JnlLUG%VOMr^>J-%1io7d!BdW-sA`xVu0s3;Xw33BeZdT zf+A}bxMe!DQL($1&?lKD~(+Nhs(^js%DtPm8)|I>Hhr;>P={-&xwG> zlzDPhP%${&DOo6u6$t4=gvena-dDrLYy60k61is(K+X7`%nf=H<;V52$K9+7Ic=n=uSfOyOW6NtY1 zhU~G(LrPBE$dvB5>FgU3CTZ1A%S0b4oBOq;m-bOJM9ms-gky&ZH&iwpcJ+mj{e6C~ zy&(4W2_E_nt^+$RK#F-aTgjNu9?ej=u?qE=1t>zuA?g2OmCu3)M0ru7c5Ny=Qo5JK znstab+pt?mx^56gGy<15s8Ju+m&Vy+5K11?h|OL?{IO$AXGL-6i=YC|Atn%EN4_e9 z6RrEM?hA;jPT=Nz0ZA->z;1DNow zqp1mXE;RY_;96UCxA>XX3bv{Yr_gbg*!b4eoToL%m`sbots^w`NH!1NLSPIXm?sk3K@ODy_HeVB69k4Lf}oRfnD)#>=M;oUxcT$R8R z$#r;?jTVaQCP!sW(BdJ=W=uKkanGuSuD`tv9NU%6VdNYx{@M!AaB|NM&5&D3#zGGT+9cq{|2xGFq2-$}CN;UB!>E4D< zU34ZSHCiJ_aHeiR5{+I42x^#WHB#p4!~~BRWyv06xDT=(?ra@o!Ygv!!Z>I7j%)j| zi7i-Mba{jGY6@Td)*AG)caPwJD7X9guyDG+=LDVokQAIzx*=rh3oFo7L#(j6nFE=JD&BC=y@FaJC7dmHj2Tb znKbMpV6Vq>*q%mh Date: Fri, 22 Feb 2019 14:09:26 -0800 Subject: [PATCH 023/119] Finding base test case --- .circleci/performance/perf_deploy_sck.sh | 2 +- .circleci/performance/run_perf_scenarios.py | 179 ++++++++++++++++++-- 2 files changed, 169 insertions(+), 12 deletions(-) diff --git a/.circleci/performance/perf_deploy_sck.sh b/.circleci/performance/perf_deploy_sck.sh index 8abe06d2..e8e0d321 100755 --- a/.circleci/performance/perf_deploy_sck.sh +++ b/.circleci/performance/perf_deploy_sck.sh @@ -27,7 +27,7 @@ function setup_kubeclient () function deploy_sck () { print_msg "Deploying SCK" - setup_kubeclient + setup_kubeclient || true print_msg "Installing the SCK build artifacts on the kubernetes cluster" helm install --name=perf-test -f .circleci/performance/perf_test_sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz } diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 7870c4b8..3cb57493 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -16,12 +16,111 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 3, - 'message_count': 10000, + 'number_of_datagen': 10, + 'message_count': 100000, + 'message_size': 128, + 'eps': 100, + 'sleep_duration': 1800 + }, + { + 'deployment_name': 'perf-test-datagen', + 'namespace': 'default', + 'number_of_datagen': 10, + 'message_count': 100000, 'message_size': 256, - 'eps': 500, - 'sleep_duration': 120 + 'eps': 100, + 'sleep_duration': 1800 + }, + { + 'deployment_name': 'perf-test-datagen', + 'namespace': 'default', + 'number_of_datagen': 10, + 'message_count': 100000, + 'message_size': 512, + 'eps': 100, + 'sleep_duration': 1800 + }, + { + 'deployment_name': 'perf-test-datagen', + 'namespace': 'default', + 'number_of_datagen': 10, + 'message_count': 100000, + 'message_size': 1024, + 'eps': 100, + 'sleep_duration': 1800 }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 128, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 256, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 512, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 1024, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 128, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 256, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 512, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 1024, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, ] # Lookup configurables for fluent and kubernetes @@ -30,18 +129,74 @@ CONNECTOR_PERF_CASES = [ { 'buffer_type': 'memory', - 'total_limit_size': '400m', - 'chunk_limit_size': '500m', - 'chunk_limit_records': 1, - 'flush_interval': '1s', + 'total_limit_size': '600m', + 'chunk_limit_size': '200m', + 'chunk_limit_records': 10000, + 'flush_interval': '5s', 'flush_thread_count': 1, 'overflow_action': 'block', 'retry_max_times': 3, - 'limits_cpu': '1000m', - 'limits_memory': '1000Mi', + 'limits_cpu': '2000m', + 'limits_memory': '2000Mi', 'requests_cpu': '500m', 'requests_memory': '500Mi', }, + # { + # 'buffer_type': 'memory', + # 'total_limit_size': '600m', + # 'chunk_limit_size': '200m', + # 'chunk_limit_records': 10000, + # 'flush_interval': '5s', + # 'flush_thread_count': 1, + # 'overflow_action': 'block', + # 'retry_max_times': 3, + # 'limits_cpu': '2000m', + # 'limits_memory': '2000Mi', + # 'requests_cpu': '1000m', + # 'requests_memory': '1000Mi', + # }, + # { + # 'buffer_type': 'memory', + # 'total_limit_size': '400m', + # 'chunk_limit_size': '100m', + # 'chunk_limit_records': 10000, + # 'flush_interval': '5s', + # 'flush_thread_count': 1, + # 'overflow_action': 'block', + # 'retry_max_times': 3, + # 'limits_cpu': '2000m', + # 'limits_memory': '2000Mi', + # 'requests_cpu': '500m', + # 'requests_memory': '500Mi', + # }, + # { + # 'buffer_type': 'memory', + # 'total_limit_size': '400m', + # 'chunk_limit_size': '100m', + # 'chunk_limit_records': 10000, + # 'flush_interval': '5s', + # 'flush_thread_count': 1, + # 'overflow_action': 'block', + # 'retry_max_times': 3, + # 'limits_cpu': '2000m', + # 'limits_memory': '2000Mi', + # 'requests_cpu': '1000m', + # 'requests_memory': '1000Mi', + # }, + # { + # 'buffer_type': 'file', + # 'total_limit_size': '600m', + # 'chunk_limit_size': '200m', + # 'chunk_limit_records': 10000, + # 'flush_interval': '5s', + # 'flush_thread_count': 1, + # 'overflow_action': 'block', + # 'retry_max_times': 3, + # 'limits_cpu': '2000m', + # 'limits_memory': '2000Mi', + # 'requests_cpu': '500m', + # 'requests_memory': '500Mi', + # }, ] @@ -131,7 +286,9 @@ def teardown_connector_and_datagen(test_case_datagen, test_case_connector): def wait_for_connector_do_data_collection_injection(test_case_datagen): logging.info('Sleeping for data collection') - time.sleep(int(test_case_datagen['sleep_duration'])) + for second in range(int(test_case_datagen['sleep_duration']), 0, -1): + logging.info('Sleeping ' + str(second) + '........') + time.sleep(1) def _do_perf(): From 7140a69beccc8ca6bbacf48f344fb86a119a89bd Mon Sep 17 00:00:00 2001 From: dtregonning Date: Fri, 22 Feb 2019 18:49:56 -0800 Subject: [PATCH 024/119] update sleep message and Second round testing --- .circleci/performance/run_perf_scenarios.py | 144 ++++++++++---------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 3cb57493..7a7bdd7d 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -13,78 +13,78 @@ # Lookup configurables for datagen - https://github.com/dtregonning/kafka-data-gen DATAGEN_PERF_CASES = [ - { - 'deployment_name': 'perf-test-datagen', - 'namespace': 'default', - 'number_of_datagen': 10, - 'message_count': 100000, - 'message_size': 128, - 'eps': 100, - 'sleep_duration': 1800 - }, - { - 'deployment_name': 'perf-test-datagen', - 'namespace': 'default', - 'number_of_datagen': 10, - 'message_count': 100000, - 'message_size': 256, - 'eps': 100, - 'sleep_duration': 1800 - }, - { - 'deployment_name': 'perf-test-datagen', - 'namespace': 'default', - 'number_of_datagen': 10, - 'message_count': 100000, - 'message_size': 512, - 'eps': 100, - 'sleep_duration': 1800 - }, - { - 'deployment_name': 'perf-test-datagen', - 'namespace': 'default', - 'number_of_datagen': 10, - 'message_count': 100000, - 'message_size': 1024, - 'eps': 100, - 'sleep_duration': 1800 - }, - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 50, - # 'message_count': 100000, - # 'message_size': 128, - # 'eps': 100, - # 'sleep_duration': 1800 - # }, - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 50, - # 'message_count': 100000, - # 'message_size': 256, - # 'eps': 100, - # 'sleep_duration': 1800 - # }, - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 50, - # 'message_count': 100000, - # 'message_size': 512, - # 'eps': 100, - # 'sleep_duration': 1800 - # }, - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 50, - # 'message_count': 100000, - # 'message_size': 1024, - # 'eps': 100, - # 'sleep_duration': 1800 + #{ + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 10, + # 'message_count': 100000, + # 'message_size': 128, + # 'eps': 100, + # 'sleep_duration': 1800 + #}, + #{ + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 10, + # 'message_count': 100000, + # 'message_size': 256, + # 'eps': 100, + # 'sleep_duration': 1800 + #}, + #{ + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 10, + # 'message_count': 100000, + # 'message_size': 512, + # 'eps': 100, + # 'sleep_duration': 1800 + #}, + #{ + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 10, + # 'message_count': 100000, + # 'message_size': 1024, + # 'eps': 100, + # 'sleep_duration': 1800 # }, + { + 'deployment_name': 'perf-test-datagen', + 'namespace': 'default', + 'number_of_datagen': 50, + 'message_count': 100000, + 'message_size': 128, + 'eps': 100, + 'sleep_duration': 1800 + }, + { + 'deployment_name': 'perf-test-datagen', + 'namespace': 'default', + 'number_of_datagen': 50, + 'message_count': 100000, + 'message_size': 256, + 'eps': 100, + 'sleep_duration': 1800 + }, + { + 'deployment_name': 'perf-test-datagen', + 'namespace': 'default', + 'number_of_datagen': 50, + 'message_count': 100000, + 'message_size': 512, + 'eps': 100, + 'sleep_duration': 1800 + }, + { + 'deployment_name': 'perf-test-datagen', + 'namespace': 'default', + 'number_of_datagen': 50, + 'message_count': 100000, + 'message_size': 1024, + 'eps': 100, + 'sleep_duration': 1800 + } # { # 'deployment_name': 'perf-test-datagen', # 'namespace': 'default', @@ -287,7 +287,7 @@ def teardown_connector_and_datagen(test_case_datagen, test_case_connector): def wait_for_connector_do_data_collection_injection(test_case_datagen): logging.info('Sleeping for data collection') for second in range(int(test_case_datagen['sleep_duration']), 0, -1): - logging.info('Sleeping ' + str(second) + '........') + if(second % 60 == 0) {logging.info('Sleeping ' + str(second) + '........')} time.sleep(1) From d5ea33ed2d0f13f0985dc54599538ef8ceeedbdd Mon Sep 17 00:00:00 2001 From: dtregonning Date: Fri, 22 Feb 2019 18:54:42 -0800 Subject: [PATCH 025/119] fixed loop --- .circleci/performance/run_perf_scenarios.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 7a7bdd7d..7896f810 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -287,7 +287,8 @@ def teardown_connector_and_datagen(test_case_datagen, test_case_connector): def wait_for_connector_do_data_collection_injection(test_case_datagen): logging.info('Sleeping for data collection') for second in range(int(test_case_datagen['sleep_duration']), 0, -1): - if(second % 60 == 0) {logging.info('Sleeping ' + str(second) + '........')} + if(second % 60 == 0): + logging.info('Sleeping ' + str(second) + '........') time.sleep(1) From 9f17e0e6f44a925c466452cca09c46748c9745db Mon Sep 17 00:00:00 2001 From: dtregonning Date: Fri, 22 Feb 2019 21:26:27 -0800 Subject: [PATCH 026/119] Run 3rd round of tests --- .circleci/performance/run_perf_scenarios.py | 82 ++++++++++----------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 7896f810..8c02e8df 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -49,10 +49,46 @@ # 'eps': 100, # 'sleep_duration': 1800 # }, - { + #{ + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 128, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 256, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 512, + # 'eps': 100, + # 'sleep_duration': 1800 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 50, + # 'message_count': 100000, + # 'message_size': 1024, + # 'eps': 100, + # 'sleep_duration': 1800 + # } + { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 50, + 'number_of_datagen': 100, 'message_count': 100000, 'message_size': 128, 'eps': 100, @@ -61,7 +97,7 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 50, + 'number_of_datagen': 100, 'message_count': 100000, 'message_size': 256, 'eps': 100, @@ -70,7 +106,7 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 50, + 'number_of_datagen': 100, 'message_count': 100000, 'message_size': 512, 'eps': 100, @@ -79,48 +115,12 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 50, + 'number_of_datagen': 100, 'message_count': 100000, 'message_size': 1024, 'eps': 100, 'sleep_duration': 1800 } - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 100, - # 'message_count': 100000, - # 'message_size': 128, - # 'eps': 100, - # 'sleep_duration': 1800 - # }, - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 100, - # 'message_count': 100000, - # 'message_size': 256, - # 'eps': 100, - # 'sleep_duration': 1800 - # }, - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 100, - # 'message_count': 100000, - # 'message_size': 512, - # 'eps': 100, - # 'sleep_duration': 1800 - # }, - # { - # 'deployment_name': 'perf-test-datagen', - # 'namespace': 'default', - # 'number_of_datagen': 100, - # 'message_count': 100000, - # 'message_size': 1024, - # 'eps': 100, - # 'sleep_duration': 1800 - # }, ] # Lookup configurables for fluent and kubernetes From ef521fd8c850d5fb19b6b5aef2889aa090412a5e Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 25 Feb 2019 08:49:12 -0800 Subject: [PATCH 027/119] Run 100 datagens for 1 hour --- .circleci/performance/run_perf_scenarios.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 8c02e8df..22465c9c 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -92,7 +92,7 @@ 'message_count': 100000, 'message_size': 128, 'eps': 100, - 'sleep_duration': 1800 + 'sleep_duration': 3600 }, { 'deployment_name': 'perf-test-datagen', @@ -101,7 +101,7 @@ 'message_count': 100000, 'message_size': 256, 'eps': 100, - 'sleep_duration': 1800 + 'sleep_duration': 3600 }, { 'deployment_name': 'perf-test-datagen', @@ -110,7 +110,7 @@ 'message_count': 100000, 'message_size': 512, 'eps': 100, - 'sleep_duration': 1800 + 'sleep_duration': 3600 }, { 'deployment_name': 'perf-test-datagen', @@ -119,7 +119,7 @@ 'message_count': 100000, 'message_size': 1024, 'eps': 100, - 'sleep_duration': 1800 + 'sleep_duration': 3600 } ] From c753096966d92e55402f2e3821baa6921150cb08 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 25 Feb 2019 16:42:16 -0800 Subject: [PATCH 028/119] Run 200 container testing --- .circleci/performance/run_perf_scenarios.py | 46 ++++++++++++++++++--- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 22465c9c..330f7b0c 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -85,10 +85,46 @@ # 'eps': 100, # 'sleep_duration': 1800 # } - { + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 128, + # 'eps': 100, + # 'sleep_duration': 3600 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 256, + # 'eps': 100, + # 'sleep_duration': 3600 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 512, + # 'eps': 100, + # 'sleep_duration': 3600 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 100000, + # 'message_size': 1024, + # 'eps': 100, + # 'sleep_duration': 3600 + # } + { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 100, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 128, 'eps': 100, @@ -97,7 +133,7 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 100, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 256, 'eps': 100, @@ -106,7 +142,7 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 100, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 512, 'eps': 100, @@ -115,7 +151,7 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 100, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 1024, 'eps': 100, From 391f41c69f09a50d11c32ffd7ccfff57dba733aa Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 25 Feb 2019 22:05:53 -0800 Subject: [PATCH 029/119] Perftest 500 Containers --- .circleci/performance/run_perf_scenarios.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 330f7b0c..456e7dcd 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -124,38 +124,38 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 500, 'message_count': 100000, 'message_size': 128, 'eps': 100, - 'sleep_duration': 3600 + 'sleep_duration': 5400 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 500, 'message_count': 100000, 'message_size': 256, 'eps': 100, - 'sleep_duration': 3600 + 'sleep_duration': 5400 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 500, 'message_count': 100000, 'message_size': 512, 'eps': 100, - 'sleep_duration': 3600 + 'sleep_duration': 5400 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 500, 'message_count': 100000, 'message_size': 1024, 'eps': 100, - 'sleep_duration': 3600 + 'sleep_duration': 5400 } ] From 2f55f63b3fc0277aa61c6fbcbce96732a71ae262 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Wed, 27 Feb 2019 14:35:39 -0800 Subject: [PATCH 030/119] 200 containers - 20 0EPS test --- .circleci/performance/run_perf_scenarios.py | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 456e7dcd..1468b0b7 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -18,7 +18,7 @@ # 'namespace': 'default', # 'number_of_datagen': 10, # 'message_count': 100000, - # 'message_size': 128, + # 'message_size': 128, # 'eps': 100, # 'sleep_duration': 1800 #}, @@ -124,38 +124,38 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 500, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 128, - 'eps': 100, - 'sleep_duration': 5400 + 'eps': 200, + 'sleep_duration': 2700 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 500, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 256, - 'eps': 100, - 'sleep_duration': 5400 + 'eps': 200, + 'sleep_duration': 2700 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 500, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 512, - 'eps': 100, - 'sleep_duration': 5400 + 'eps': 200, + 'sleep_duration': 2700 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 500, + 'number_of_datagen': 200, 'message_count': 100000, 'message_size': 1024, - 'eps': 100, - 'sleep_duration': 5400 + 'eps': 200, + 'sleep_duration': 2700 } ] From 1289bf9b0c5dacd51b66ccecc85aab0878fb3d9b Mon Sep 17 00:00:00 2001 From: dtregonning Date: Wed, 27 Feb 2019 14:56:20 -0800 Subject: [PATCH 031/119] Helm got bumped..broke CI, pinning Helm version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39407615..02d5cd97 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1 sudo apt-get install python3-pip python3-dev > /dev/null 2>&1 sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -s -- --version v2.12.3 helm init -c echo "Installing aws cli..." sudo pip install awscli > /dev/null 2>&1 From a8616cb865b6f1ce5622c458945137180866d5b7 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Wed, 27 Feb 2019 15:00:51 -0800 Subject: [PATCH 032/119] needed updating in 2 places --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 02d5cd97..9fd9e533 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,7 @@ jobs: sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1 sudo apt-get install python3-pip python3-dev > /dev/null 2>&1 sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -s -- --version v2.12.3 helm init -c echo "Installing aws cli..." sudo pip install awscli > /dev/null 2>&1 From ce4ea5153486a12ba31d07fec7208073739fba17 Mon Sep 17 00:00:00 2001 From: gp510 Date: Thu, 28 Feb 2019 15:16:21 -0800 Subject: [PATCH 033/119] Updated index search for logging func tests (#102) --- test/k8s_logging_tests/test_config_logging.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/k8s_logging_tests/test_config_logging.py b/test/k8s_logging_tests/test_config_logging.py index 06d8595d..d403e1b5 100644 --- a/test/k8s_logging_tests/test_config_logging.py +++ b/test/k8s_logging_tests/test_config_logging.py @@ -19,7 +19,7 @@ def test_splunk_index(setup, test_input, expected): ''' logging.getLogger().info("testing test_splunk_index input={0} \ expected={1} event(s)".format(test_input, expected)) - search_query = "index=circleci_events" + search_query = "index=circleci_events OR index=kube-system" events = check_events_from_splunk(start_time="-1h@h", url=setup["splunkd_url"], user=setup["splunk_user"], @@ -39,7 +39,7 @@ def test_cluster_name(setup, test_input, expected): ''' logging.getLogger().info("testing test_clusterName input={0} \ expected={1} event(s)".format(test_input, expected)) - search_query = "index=circleci_events cluster_name=circleci-k8s-cluster | head 1" + search_query = "index=circleci_events OR index=kube-system cluster_name=circleci-k8s-cluster | head 1" events = check_events_from_splunk(start_time="-1h@h", url=setup["splunkd_url"], user=setup["splunk_user"], @@ -160,7 +160,7 @@ def test_sourcetype(setup, test_input, expected): events = check_events_from_splunk(start_time="-24h@h", url=setup["splunkd_url"], user=setup["splunk_user"], - query=["search index=circleci_events sourcetype={0}".format(test_input)], + query=["search index=circleci_events OR index=kube-system sourcetype={0}".format(test_input)], password=setup["splunk_password"]) logging.getLogger().info("Splunk received %s events in the last minute", len(events)) @@ -182,7 +182,7 @@ def test_source(setup, test_input, expected): events = check_events_from_splunk(start_time="-24h@h", url=setup["splunkd_url"], user=setup["splunk_user"], - query=["search index=circleci_events source={0}".format(test_input)], + query=["search index=circleci_events OR index=kube-system source={0}".format(test_input)], password=setup["splunk_password"]) logging.getLogger().info("Splunk received %s events in the last minute", len(events)) From a8c828e7e20d9e8f2c4c2828b89b662d65ded081 Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Thu, 28 Feb 2019 15:49:49 -0800 Subject: [PATCH 034/119] Update helm version on the fly (#101) * update helm version on the fly --- .circleci/deploy_connector.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh index 6649b2b7..951d8103 100755 --- a/.circleci/deploy_connector.sh +++ b/.circleci/deploy_connector.sh @@ -8,6 +8,8 @@ sudo apt-get -y install gnupg sudo mkdir ~/.kube echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg sudo mv config ~/.kube/config +#Update helm server version +helm init --upgrade #Make sure to check and clean previously failed deployment echo "Checking if previous deployment exist..." if [ "`helm ls`" == "" ]; then From ac469e11c0ec011e7d7bd4e69636a9d5154dde34 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Thu, 28 Feb 2019 18:20:40 -0800 Subject: [PATCH 035/119] 200 containers - 200 EPS - rerun --- .circleci/performance/run_perf_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 1468b0b7..3ff523ea 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -37,7 +37,7 @@ # 'number_of_datagen': 10, # 'message_count': 100000, # 'message_size': 512, - # 'eps': 100, + # 'eps': 100, # 'sleep_duration': 1800 #}, #{ From 50195d3126c5d98e607882608505e3524f9b63f1 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 10:46:06 -0800 Subject: [PATCH 036/119] 30con-1000eps-tweaked-fluent --- .circleci/performance/run_perf_scenarios.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 3ff523ea..21f24b59 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -124,25 +124,25 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 30, 'message_count': 100000, 'message_size': 128, - 'eps': 200, + 'eps': 1000, 'sleep_duration': 2700 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 30, 'message_count': 100000, 'message_size': 256, - 'eps': 200, + 'eps': 1000, 'sleep_duration': 2700 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 30, 'message_count': 100000, 'message_size': 512, 'eps': 200, @@ -151,10 +151,10 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, + 'number_of_datagen': 30, 'message_count': 100000, 'message_size': 1024, - 'eps': 200, + 'eps': 1000, 'sleep_duration': 2700 } ] @@ -167,8 +167,8 @@ 'buffer_type': 'memory', 'total_limit_size': '600m', 'chunk_limit_size': '200m', - 'chunk_limit_records': 10000, - 'flush_interval': '5s', + 'chunk_limit_records': 512000, + 'flush_interval': '10s', 'flush_thread_count': 1, 'overflow_action': 'block', 'retry_max_times': 3, From bc72ffa26b40ad97d1b6a78447bb54c633b6e29d Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 11:01:40 -0800 Subject: [PATCH 037/119] 30dg-1000eps-120000mes-1024size --- .circleci/performance/run_perf_scenarios.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 21f24b59..77956081 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -125,37 +125,37 @@ 'deployment_name': 'perf-test-datagen', 'namespace': 'default', 'number_of_datagen': 30, - 'message_count': 100000, - 'message_size': 128, + 'message_count': 120000, + 'message_size': 1024, 'eps': 1000, - 'sleep_duration': 2700 + 'sleep_duration': 60 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', 'number_of_datagen': 30, - 'message_count': 100000, - 'message_size': 256, + 'message_count': 120000, + 'message_size': 1024, 'eps': 1000, - 'sleep_duration': 2700 + 'sleep_duration': 60 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', 'number_of_datagen': 30, - 'message_count': 100000, - 'message_size': 512, - 'eps': 200, - 'sleep_duration': 2700 + 'message_count': 120000, + 'message_size': 1024, + 'eps': 1000, + 'sleep_duration': 60 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', 'number_of_datagen': 30, - 'message_count': 100000, + 'message_count': 120000, 'message_size': 1024, 'eps': 1000, - 'sleep_duration': 2700 + 'sleep_duration': 60 } ] From 73e71bd7b4215f763447a1d450481fd2d5198eff Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 11:16:58 -0800 Subject: [PATCH 038/119] sleeps were too small --- .circleci/performance/run_perf_scenarios.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 77956081..772a08ed 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -128,7 +128,7 @@ 'message_count': 120000, 'message_size': 1024, 'eps': 1000, - 'sleep_duration': 60 + 'sleep_duration': 600 }, { 'deployment_name': 'perf-test-datagen', @@ -137,7 +137,7 @@ 'message_count': 120000, 'message_size': 1024, 'eps': 1000, - 'sleep_duration': 60 + 'sleep_duration': 600 }, { 'deployment_name': 'perf-test-datagen', @@ -146,7 +146,7 @@ 'message_count': 120000, 'message_size': 1024, 'eps': 1000, - 'sleep_duration': 60 + 'sleep_duration': 600 }, { 'deployment_name': 'perf-test-datagen', From f927bf1a8f2f60eed8df2f470ec63457134546ea Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 11:55:22 -0800 Subject: [PATCH 039/119] upgrade-pod-resources --- .circleci/performance/run_perf_scenarios.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 772a08ed..3b775235 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -155,7 +155,7 @@ 'message_count': 120000, 'message_size': 1024, 'eps': 1000, - 'sleep_duration': 60 + 'sleep_duration': 600 } ] @@ -172,10 +172,10 @@ 'flush_thread_count': 1, 'overflow_action': 'block', 'retry_max_times': 3, - 'limits_cpu': '2000m', - 'limits_memory': '2000Mi', - 'requests_cpu': '500m', - 'requests_memory': '500Mi', + 'limits_cpu': '10000m', + 'limits_memory': '10000m', + 'requests_cpu': '2500m', + 'requests_memory': '2500Mi', }, # { # 'buffer_type': 'memory', From 87b45c8523c5cde9fcaa3f9e0de199b8d8d11053 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 12:00:48 -0800 Subject: [PATCH 040/119] fix typo --- .circleci/performance/run_perf_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 3b775235..a6de809d 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -173,7 +173,7 @@ 'overflow_action': 'block', 'retry_max_times': 3, 'limits_cpu': '10000m', - 'limits_memory': '10000m', + 'limits_memory': '10000Mi', 'requests_cpu': '2500m', 'requests_memory': '2500Mi', }, From ae6c24362984a63b34c1fcee94e2494e184111c0 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 13:06:41 -0800 Subject: [PATCH 041/119] 200gen-1kbsize-250eps-1hour --- .circleci/performance/run_perf_scenarios.py | 36 ++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index a6de809d..8e0d3508 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -120,41 +120,41 @@ # 'message_size': 1024, # 'eps': 100, # 'sleep_duration': 3600 - # } - { + # + { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 30, - 'message_count': 120000, + 'number_of_datagen': 200, + 'message_count': 360000 , 'message_size': 1024, - 'eps': 1000, + 'eps': 250, 'sleep_duration': 600 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 30, - 'message_count': 120000, + 'number_of_datagen': 200, + 'message_count': 360000 , 'message_size': 1024, - 'eps': 1000, + 'eps': 250, 'sleep_duration': 600 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 30, - 'message_count': 120000, + 'number_of_datagen': 200, + 'message_count': 360000 , 'message_size': 1024, - 'eps': 1000, + 'eps': 250, 'sleep_duration': 600 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 30, - 'message_count': 120000, + 'number_of_datagen': 200, + 'message_count': 360000 , 'message_size': 1024, - 'eps': 1000, + 'eps': 250, 'sleep_duration': 600 } ] @@ -172,10 +172,10 @@ 'flush_thread_count': 1, 'overflow_action': 'block', 'retry_max_times': 3, - 'limits_cpu': '10000m', - 'limits_memory': '10000Mi', - 'requests_cpu': '2500m', - 'requests_memory': '2500Mi', + 'limits_cpu': '2000m', + 'limits_memory': '2000Mi', + 'requests_cpu': '1000m', + 'requests_memory': '1000Mi', }, # { # 'buffer_type': 'memory', From b6f8d103937ad28957e4010f0a08ea19ed51654a Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 15:39:26 -0800 Subject: [PATCH 042/119] reduce containers to 100 --- .circleci/performance/run_perf_scenarios.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 8e0d3508..0b56b525 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -124,38 +124,38 @@ { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, - 'message_count': 360000 , + 'number_of_datagen': 100, + 'message_count': 225000 , 'message_size': 1024, 'eps': 250, - 'sleep_duration': 600 + 'sleep_duration': 3600 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, - 'message_count': 360000 , + 'number_of_datagen': 100, + 'message_count': 225000 , 'message_size': 1024, 'eps': 250, - 'sleep_duration': 600 + 'sleep_duration': 3600 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, - 'message_count': 360000 , + 'number_of_datagen': 100, + 'message_count': 225000 , 'message_size': 1024, 'eps': 250, - 'sleep_duration': 600 + 'sleep_duration': 3600 }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 200, - 'message_count': 360000 , + 'number_of_datagen': 100, + 'message_count': 225000 , 'message_size': 1024, 'eps': 250, - 'sleep_duration': 600 + 'sleep_duration': 3600 } ] From 489894206bb59674a8e61933d5f5fcbaa84ec6e6 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 17:43:16 -0800 Subject: [PATCH 043/119] modified daemonset reference to old image --- helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml | 2 +- helm-chart/splunk-kubernetes-logging/values.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 854e38a2..981ff894 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -31,7 +31,7 @@ spec: {{- end }} containers: - name: splunk-fluentd-k8s-logs - image: splunk/fluentd-hec:{{ .Values.image.tag }} + image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - -c diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 75ed7df0..9e3afa1b 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -221,8 +221,8 @@ logs: # Defines which version of image to use, and how it should be pulled. image: - tag: 1.0.1 - pullPolicy: IfNotPresent + name: splunk/fluentd-hec:1.1.0.Alpha + pullPolicy: Always # Controls the resources used by the fluentd daemonset From 02fcbd7f23ed4a1cdae3c442b0f52bc7d1dea608 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 19:19:48 -0800 Subject: [PATCH 044/119] sanity check --- .circleci/performance/run_perf_scenarios.py | 60 ++++++++++----------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index 0b56b525..a05532eb 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -121,40 +121,40 @@ # 'eps': 100, # 'sleep_duration': 3600 # + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 225000 , + # 'message_size': 1024, + # 'eps': 250, + # 'sleep_duration': 3600 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 225000 , + # 'message_size': 1024, + # 'eps': 250, + # 'sleep_duration': 3600 + # }, + # { + # 'deployment_name': 'perf-test-datagen', + # 'namespace': 'default', + # 'number_of_datagen': 100, + # 'message_count': 225000 , + # 'message_size': 1024, + # 'eps': 250, + # 'sleep_duration': 3600 + # }, { 'deployment_name': 'perf-test-datagen', 'namespace': 'default', - 'number_of_datagen': 100, - 'message_count': 225000 , - 'message_size': 1024, - 'eps': 250, - 'sleep_duration': 3600 - }, - { - 'deployment_name': 'perf-test-datagen', - 'namespace': 'default', - 'number_of_datagen': 100, - 'message_count': 225000 , - 'message_size': 1024, - 'eps': 250, - 'sleep_duration': 3600 - }, - { - 'deployment_name': 'perf-test-datagen', - 'namespace': 'default', - 'number_of_datagen': 100, - 'message_count': 225000 , - 'message_size': 1024, - 'eps': 250, - 'sleep_duration': 3600 - }, - { - 'deployment_name': 'perf-test-datagen', - 'namespace': 'default', - 'number_of_datagen': 100, - 'message_count': 225000 , + 'number_of_datagen': 10, + 'message_count': 100000 , 'message_size': 1024, - 'eps': 250, + 'eps': 100, 'sleep_duration': 3600 } ] From 05a216dda76868b00b8caf1a56b6f84f09c02832 Mon Sep 17 00:00:00 2001 From: dtregonning Date: Mon, 4 Mar 2019 19:20:37 -0800 Subject: [PATCH 045/119] modify --- .circleci/performance/run_perf_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/performance/run_perf_scenarios.py b/.circleci/performance/run_perf_scenarios.py index a05532eb..fc11ba1e 100644 --- a/.circleci/performance/run_perf_scenarios.py +++ b/.circleci/performance/run_perf_scenarios.py @@ -152,7 +152,7 @@ 'deployment_name': 'perf-test-datagen', 'namespace': 'default', 'number_of_datagen': 10, - 'message_count': 100000 , + 'message_count': 1000 , 'message_size': 1024, 'eps': 100, 'sleep_duration': 3600 From 53894feb24795b2d94410c059d2038ff198c1853 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 5 Mar 2019 14:08:21 -0800 Subject: [PATCH 046/119] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 712d851a..de1038c7 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ Splunk Connect for Kubernetes supports installation using Helm. Ensure that you Make sure you do the following before you install: 1. Create a minimum of two Splunk indexes: -* one events index, which will handle logs and objects (you may also create two separate indexes for logs and objects). -* one metrics index. +* One events index, which will handle logs and objects (you may also create two separate indexes for logs and objects). +* One metrics index. If you do not configure these indexes, Kubernetes Connect for Splunk uses the defaults created in your HEC token. 2. Create a HEC token if you do not already have one. If you are installing the connector on Splunk Cloud, file a ticket with Splunk Customer Service and they will deploy the indexes for your environment and generate your HEC token. From dcad36322e18d0ae828a7c65629440df71a28391 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 5 Mar 2019 14:14:47 -0800 Subject: [PATCH 047/119] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index de1038c7..b05037ea 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,11 @@ To configure the Splunk Connector for Kubernetes using YAML files: Splunk Connect for Kubernetes deploys a daemonset on each node. And in the daemonset, a Fluentd container runs and does the collecting job. Splunk Connector for Kubernetes collects three types of data: -* logs: Splunk Connectr for Kubernetes collects two types of logs: - * logs from Kubernetes system components (https://kubernetes.io/docs/concepts/overview/components/) - * applications (container) logs -* [objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) -* metrics +* Logs: Splunk Connect for Kubernetes collects two types of logs: + * Logs from Kubernetes system components (https://kubernetes.io/docs/concepts/overview/components/) + * Applications (container) logs +* [Objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) +* Metrics To collect the data, Splunk leverages: @@ -113,7 +113,7 @@ If you want to learn more about which metrics are collected and labels used with # Performance -Some parameters used with Splunk Connect for Kubernetes can have an impact on overall performance of log ingestion, objects, or metrics. In general, the more filters that are added to one of the streams, the greater the preformance impact. +Some parameters used with Splunk Connect for Kubernetes can have an impact on overall performance of log ingestion, objects, or metrics. In general, the more filters that are added to one of the streams, the greater the performance impact. Splunk Connect for Kubernetes can exceed the default throughput of HEC. To best address capacity needs, Splunk recommends that you monitor the HEC throughput and back pressure on Splunk Connect for Kubernetes deployments and be prepared to add additional nodes as needed. From a712c4ce0fde58d40e6d0d86040b034b4b2638f7 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 5 Mar 2019 14:48:29 -0800 Subject: [PATCH 048/119] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b05037ea..746b6147 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Make sure your Splunk configuration has a metrics index that is able to receive If you want to learn more about how metrics are monitored in a Kubernetes cluster, see Tools for [Monitoring Compute, Storage, and Network Resources](https://kubernetes.io/docs/tasks/debug-application-cluster/resource-usage-monitoring/). -If you want to learn more about which metrics are collected and labels used with Splunk Connect for Kubernetes, view the metrics [schema](https://github.com/kubernetes/heapster/blob/master/docs/storage-schema.md#metrics) within the Heapster docs. +If you want to learn more about which metrics are collected and labels used with Splunk Connect for Kubernetes, view the metrics [schema](https://github.com/splunk/fluent-plugin-kubernetes-metrics). # Performance From 684138e88dc17bbacb4ce6d85f68fbd205ddb6f4 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 6 Mar 2019 15:00:28 -0800 Subject: [PATCH 049/119] Added automated release for github (#100) --- .circleci/build_sck_artifacts_for_release.sh | 42 ++++++++++ .circleci/config.yml | 46 +++++++++- .circleci/deploy_connector_release.sh | 25 ++++++ .circleci/push_sck_release.sh | 10 +++ .circleci/sck_values.yml | 2 +- .circleci/update_sck_config.sh | 2 +- .circleci/update_sck_config_for_release.sh | 51 +++++++++++ README.md | 7 +- VERSION | 1 + .../splunk-connect-for-kubernetes/Chart.yaml | 8 +- .../requirements.yaml | 6 +- .../splunk-kubernetes-logging/Chart.yaml | 7 +- .../templates/daemonset.yaml | 2 +- .../splunk-kubernetes-logging/values.yaml | 4 +- .../splunk-kubernetes-metrics/Chart.yaml | 8 +- .../splunk-kubernetes-metrics/values.yaml | 6 +- .../splunk-kubernetes-objects/Chart.yaml | 7 +- .../splunk-kubernetes-objects/values.yaml | 2 +- .../splunk-kubernetes-logging/configMap.yaml | 7 +- .../splunk-kubernetes-logging/daemonset.yaml | 8 +- .../splunk-kubernetes-logging/secret.yaml | 2 +- .../clusterRole.yaml | 17 ++++ .../clusterRoleAggregator.yaml | 19 +++++ .../clusterRoleBinding.yaml | 4 +- .../clusterRoleBindingAggregator.yaml | 16 ++++ .../splunk-kubernetes-metrics/configMap.yaml | 63 +++++++++----- .../configMapMetricsAggregator.yaml | 84 +++++++++++++++++++ .../splunk-kubernetes-metrics/deployment.yaml | 65 ++++---------- .../deploymentMetricsAggregator.yaml | 56 +++++++++++++ .../splunk-kubernetes-metrics/secret.yaml | 2 +- .../serviceAccount.yaml | 2 +- .../clusterRole.yaml | 2 +- .../clusterRoleBinding.yaml | 2 +- .../splunk-kubernetes-objects/configMap.yaml | 15 +++- .../splunk-kubernetes-objects/deployment.yaml | 8 +- .../splunk-kubernetes-objects/secret.yaml | 2 +- .../serviceAccount.yaml | 2 +- tools/gen_manifest.rb | 2 +- 38 files changed, 492 insertions(+), 122 deletions(-) create mode 100755 .circleci/build_sck_artifacts_for_release.sh create mode 100755 .circleci/deploy_connector_release.sh create mode 100755 .circleci/push_sck_release.sh create mode 100755 .circleci/update_sck_config_for_release.sh create mode 100644 VERSION create mode 100644 manifests/splunk-kubernetes-metrics/clusterRole.yaml create mode 100644 manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml create mode 100644 manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml create mode 100644 manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml create mode 100644 manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml diff --git a/.circleci/build_sck_artifacts_for_release.sh b/.circleci/build_sck_artifacts_for_release.sh new file mode 100755 index 00000000..8e0dca12 --- /dev/null +++ b/.circleci/build_sck_artifacts_for_release.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -e +TAG=`cat VERSION` + +function replace_generic_version () +{ + file="$1" + _line=`awk '/version:/{print NR;exit}' $file` + replacement="version: $TAG" + # Escape backslash, forward slash and ampersand for use as a sed replacement. + replacement_escaped=$( echo "$replacement" | sed -e 's/[\/&]/\\&/g' ) + sed -i "${_line}s/.*/$replacement_escaped/" "$file" +} + +repos_array=( "helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) +sub_repos_array=( "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) + +for repo in "${repos_array[@]}" +do + filename="${repo}/Chart.yaml" + replace_generic_version $filename +done + +mkdir helm-artifacts-release +if [[ -d "helm-chart/splunk-connect-for-kubernetes/charts" ]]; then + rm -rf helm-chart/splunk-connect-for-kubernetes/charts +fi +mkdir helm-chart/splunk-connect-for-kubernetes/charts + +for sub_repo in "${sub_repos_array[@]}" +do + cp -rp $sub_repo helm-chart/splunk-connect-for-kubernetes/charts +done + +for repo in "${repos_array[@]}" +do + helm package -d helm-artifacts-release $repo +done + +rm -rf helm-chart/splunk-connect-for-kubernetes/charts diff --git a/.circleci/config.yml b/.circleci/config.yml index 39407615..0d524f7f 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,6 +76,44 @@ jobs: command: | python .circleci/performance/run_perf_scenarios.py + release: + docker: + - image: circleci/ruby:2.6.1-node-browsers + working_directory: ~/repo + steps: + - checkout + - setup_remote_docker: + reusable: true + - run: + name: Install dependencies + command: | + sudo apt-get update + sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1 + sudo apt-get install python3-pip python3-dev > /dev/null 2>&1 + sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash + helm init -c + - run: + name: Update configurables to build artifacts + command: | + .circleci/update_sck_config_for_release.sh + - run: + name: Build SCK artifacts + command: | + .circleci/build_sck_artifacts_for_release.sh + - run: + name: Push SCK artifacts to Github Release + command: | + .circleci/push_sck_release.sh + - run: + name: Deploy k8s connector + command: | + .circleci/deploy_connector_release.sh + - run: + name: Run functional tests + command: | + .circleci/run_functional_tests.sh + # Need to add teardown step workflows: @@ -89,4 +127,10 @@ workflows: - perf-test: filters: branches: - only: perf-testing \ No newline at end of file + only: perf-testing + - release: + requires: + - build + filters: + branches: + only: release-testing \ No newline at end of file diff --git a/.circleci/deploy_connector_release.sh b/.circleci/deploy_connector_release.sh new file mode 100755 index 00000000..1036c7ae --- /dev/null +++ b/.circleci/deploy_connector_release.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -e +echo "Setup kube client..." +curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +sudo apt-get -y install gnupg +sudo mkdir ~/.kube +echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg +sudo mv config ~/.kube/config +#Update helm server version +helm init --upgrade +#Make sure to check and clean previously failed deployment +echo "Checking if previous deployment exist..." +if [ "`helm ls`" == "" ]; then + echo "Nothing to clean, ready for deployment" +else + helm delete --purge $(helm ls --short) +fi +echo "Deploying k8s-connect with latest changes" +VERSION=`cat VERSION` +helm install --name=release -f .circleci/sck_values.yml helm-artifacts-release/splunk-connect-for-kubernetes*.tgz +#Todo-need to add test cases and run them before cleaning the deployment +#wait for deployment to finish, sleep longer for metrics-aggr to start sending metrics +sleep 90 \ No newline at end of file diff --git a/.circleci/push_sck_release.sh b/.circleci/push_sck_release.sh new file mode 100755 index 00000000..a698d20b --- /dev/null +++ b/.circleci/push_sck_release.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e +wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz +tar -xzvf ghr_v0.12.0_linux_amd64.tar.gz +sudo chmod +x ghr_v0.12.0_linux_amd64 +sudo mv ghr_v0.12.0_linux_amd64/ghr /usr/local/bin/ghr + +VERSION=`cat VERSION` +echo "Pushing SCK release to github releases...${VERSION}" +ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -n "${RELEASE_TITLE}" -b "${RELEASE_BODY}" -draft ${VERSION} ./helm-artifacts-release/ diff --git a/.circleci/sck_values.yml b/.circleci/sck_values.yml index 4f8eb41e..740d550d 100755 --- a/.circleci/sck_values.yml +++ b/.circleci/sck_values.yml @@ -1,5 +1,5 @@ global: - logLevel: info + logLevel: debug splunk: hec: protocol: https diff --git a/.circleci/update_sck_config.sh b/.circleci/update_sck_config.sh index 88fa730b..a7bb37c0 100755 --- a/.circleci/update_sck_config.sh +++ b/.circleci/update_sck_config.sh @@ -12,7 +12,7 @@ yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST_PERF yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN_PERF -#Pull docker images locally +# Pull docker images locally aws ecr get-login --region $AWS_REGION --no-include-email | bash docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-logging:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-metrics:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' diff --git a/.circleci/update_sck_config_for_release.sh b/.circleci/update_sck_config_for_release.sh new file mode 100755 index 00000000..1a2a778f --- /dev/null +++ b/.circleci/update_sck_config_for_release.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e +VERSION=`cat VERSION` + +# Install yq yaml parser +wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 +sudo chmod +x yq_linux_amd64 +sudo mv yq_linux_amd64 /usr/local/bin/yq + +## Modify docker images to be used +#image_name=$(echo "splunk/fluentd-hec:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name $image_name +# +#image_name=$(echo "splunk/k8s-metrics:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name $image_name +# +#image_name=$(echo "splunk/k8s-metrics-aggr:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name $image_name +# +#image_name=$(echo "splunk/kube-objects:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name $image_name +# +#yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION +#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION +#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml version $VERSION +#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml version $VERSION +# +#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml dependencies.splunk-kubernetes-logging.version $VERSION +#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml dependencies.splunk-kubernetes-metrics.version $VERSION +#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml dependencies.splunk-kubernetes-objects.version $VERSION + + +# Modify splunk environment values +yq w -i .circleci/sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST +yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN + +# Pull docker images locally +docker pull splunk/fluentd-hec:1.1.0 +docker pull splunk/k8s-metrics:1.1.0 +docker pull splunk/k8s-metrics-aggr:1.1.0 +docker pull splunk/kube-objects:1.1.0 + +# Modify docker images to be used +yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec:1.1.0 + +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics:1.1.0 + +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name splunk/k8s-metrics-aggr:1.1.0 + +yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name splunk/kube-objects:1.1.0 diff --git a/README.md b/README.md index 746b6147..9c71fa61 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,11 @@ Splunk Connect for Kubernetes can exceed the default throughput of HEC. To best One possible filter option is to enable the processing of multi-line events. This feature is currently experimental and considered to be community supported. +# Maintenance And Support + +The current maintainers of this project are the DataEdge team at Splunk. You can reach us at [DataEdge@splunk.com](mailto:DataEdge@splunk.com). +If you have any issues with the software, please file an issue at [Create a new issue in splunk connect for kubernetes project](https://github.com/splunk/splunk-connect-for-kubernetes/issues/new) + # License -See [LICENSE](LICENSE). +See [LICENSE](https://github.com/splunk/splunk-connect-for-kubernetes/blob/master/LICENSE). diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..1cc5f657 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.1.0 \ No newline at end of file diff --git a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml index 4903b1b8..fcb98667 100644 --- a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml @@ -1,15 +1,12 @@ apiVersion: v1 name: splunk-connect-for-kubernetes -version: 1.0.1 +version: 1.1.0 description: Collect logs, objects (metadata), and metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart - https://github.com/splunk/fluent-plugin-splunk-hec - https://github.com/splunk/fluent-plugin-kubernetes-objects - - https://github.com/splunk/docker-fluentd-hec - - https://github.com/splunk/docker-kube-objects - - https://github.com/kubernetes/heapster keywords: - splunk - monitoring @@ -18,4 +15,5 @@ keywords: - metadata - objects maintainers: - - name: Gimi Liang + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-connect-for-kubernetes/requirements.yaml b/helm-chart/splunk-connect-for-kubernetes/requirements.yaml index 8f597300..1d587f84 100644 --- a/helm-chart/splunk-connect-for-kubernetes/requirements.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/requirements.yaml @@ -1,13 +1,13 @@ dependencies: - name: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 repository: stable condition: logging.enabled - name: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 repository: stable condition: objects.enabled - name: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 repository: stable condition: metrics.enabled diff --git a/helm-chart/splunk-kubernetes-logging/Chart.yaml b/helm-chart/splunk-kubernetes-logging/Chart.yaml index 1c53be5b..8c3db932 100644 --- a/helm-chart/splunk-kubernetes-logging/Chart.yaml +++ b/helm-chart/splunk-kubernetes-logging/Chart.yaml @@ -1,16 +1,15 @@ apiVersion: v1 -version: 1.0.1 +version: 1.1.0 name: splunk-kubernetes-logging description: Collect logs with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-logging sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-logging - https://github.com/splunk/fluent-plugin-splunk-hec - - https://github.com/splunk/docker-fluentd-hec keywords: - splunk - monitoring - logging maintainers: - - name: Gimi Liang - email: zliang@splunk.com + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 854e38a2..981ff894 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -31,7 +31,7 @@ spec: {{- end }} containers: - name: splunk-fluentd-k8s-logs - image: splunk/fluentd-hec:{{ .Values.image.tag }} + image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - -c diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 75ed7df0..4e7a98bf 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -221,8 +221,8 @@ logs: # Defines which version of image to use, and how it should be pulled. image: - tag: 1.0.1 - pullPolicy: IfNotPresent + name: splunk/fluentd-hec:1.1.0 + pullPolicy: Always # Controls the resources used by the fluentd daemonset diff --git a/helm-chart/splunk-kubernetes-metrics/Chart.yaml b/helm-chart/splunk-kubernetes-metrics/Chart.yaml index 5b6f18f0..3fe17bf9 100644 --- a/helm-chart/splunk-kubernetes-metrics/Chart.yaml +++ b/helm-chart/splunk-kubernetes-metrics/Chart.yaml @@ -1,17 +1,15 @@ apiVersion: v1 -version: 1.0.1 +version: 1.1.0 name: splunk-kubernetes-metrics description: Collect metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-metrics sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-metrics - https://github.com/splunk/fluent-plugin-splunk-hec - - https://github.com/splunk/docker-fluentd-hec - - https://github.com/kubernetes/heapster keywords: - splunk - monitoring - metric maintainers: - - name: Gimi Liang - email: zliang@splunk.com + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index ede1ba63..1be791e0 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -8,7 +8,7 @@ # Global values have lower priority. # Values defined here are the default values. global: - logLevel: debug + logLevel: info splunk: hec: port: 8088 @@ -74,12 +74,12 @@ splunk: image: - name: splunk/k8s-metrics:1.1.0.Alpha + name: splunk/k8s-metrics:1.1.0 pullPolicy: Always imageAgg: - name: splunk/k8s-metrics-aggr:1.1.0.Alpha + name: splunk/k8s-metrics-aggr:1.1.0 pullPolicy: Always diff --git a/helm-chart/splunk-kubernetes-objects/Chart.yaml b/helm-chart/splunk-kubernetes-objects/Chart.yaml index b52868b2..b6b4bb11 100644 --- a/helm-chart/splunk-kubernetes-objects/Chart.yaml +++ b/helm-chart/splunk-kubernetes-objects/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -version: 1.0.1 +version: 1.1.0 name: splunk-kubernetes-objects description: Collect Kubernetes objects with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-objects @@ -7,12 +7,11 @@ sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-objects - https://github.com/splunk/fluent-plugin-splunk-hec - https://github.com/splunk/fluent-plugin-kubernetes-objects - - https://github.com/splunk/docker-kube-objects keywords: - splunk - monitoring - objects - metadata maintainers: - - name: Gimi Liang - email: zliang@splunk.com + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index e9aab0f6..a39d4b48 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -165,7 +165,7 @@ splunk: image: - name: splunk/kube-objects:1.0.1 + name: splunk/kube-objects:1.1.0 pullPolicy: IfNotPresent diff --git a/manifests/splunk-kubernetes-logging/configMap.yaml b/manifests/splunk-kubernetes-logging/configMap.yaml index 986cf7af..b86e1c01 100644 --- a/manifests/splunk-kubernetes-logging/configMap.yaml +++ b/manifests/splunk-kubernetes-logging/configMap.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-logging labels: app: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 data: fluent.conf: |- @include system.conf @@ -177,7 +177,7 @@ data: # extract index fields and sourcetype for container logs @type jq_transformer - jq "def find_sourcetype(pod; container_name): container_name + \"/\" + pod | if startswith(\"dns-controller/dns-controller\") then \"kube:dns-controller\" elif startswith(\"sidecar/kube-dns\") then \"kube:kubedns-sidecar\" elif startswith(\"dnsmasq/kube-dns\") then \"kube:dnsmasq\" elif startswith(\"etcd-container/etcd-server\") then \"kube:etcd\" elif startswith(\"etcd-container/etcd-server-events\") then \"kube:etcd-events\" elif startswith(\"kube-apiserver/kube-apiserver\") then \"kube:kube-apiserver\" elif startswith(\"kube-controller-manager/kube-controller-manager\") then \"kube:kube-controller-manager\" elif startswith(\"autoscaler/kube-dns-autoscaler\") then \"kube:kube-dns-autoscaler\" elif startswith(\"kube-proxy/kube-proxy\") then \"kube:kube-proxy\" elif startswith(\"kube-scheduler/kube-scheduler\") then \"kube:kube-scheduler\" elif startswith(\"kubedns/kube-dns\") then \"kube:kubedns\" else empty end; def extract_container_info: (.source | ltrimstr(\"/var/log/containers/\") | split(\"_\")) as $parts | ($parts[-1] | split(\"-\")) as $cparts | .pod = $parts[0] | .namespace = $parts[1] | .container_name = ($cparts[:-1] | join(\"-\")) | .container_id = ($cparts[-1] | rtrimstr(\".log\")) | .; .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // \"kube:container:\\(.container_name)\")" + jq "def find_sourcetype(pod; container_name): container_name + \"/\" + pod | if startswith(\"dns-controller/dns-controller\") then \"kube:dns-controller\" elif startswith(\"sidecar/kube-dns\") then \"kube:kubedns-sidecar\" elif startswith(\"dnsmasq/kube-dns\") then \"kube:dnsmasq\" elif startswith(\"etcd-container/etcd-server\") then \"kube:etcd\" elif startswith(\"etcd-container/etcd-server-events\") then \"kube:etcd-events\" elif startswith(\"kube-apiserver/kube-apiserver\") then \"kube:kube-apiserver\" elif startswith(\"kube-controller-manager/kube-controller-manager\") then \"kube:kube-controller-manager\" elif startswith(\"autoscaler/kube-dns-autoscaler\") then \"kube:kube-dns-autoscaler\" elif startswith(\"kube-proxy/kube-proxy\") then \"kube:kube-proxy\" elif startswith(\"kube-scheduler/kube-scheduler\") then \"kube:kube-scheduler\" elif startswith(\"kubedns/kube-dns\") then \"kube:kubedns\" else empty end; def set_namespace(value): if value == \"default\" then \"main\"else value end; def extract_container_info: (.source | ltrimstr(\"/var/log/containers/\") | split(\"_\")) as $parts | ($parts[-1] | split(\"-\")) as $cparts | .pod = $parts[0] | .namespace = set_namespace($parts[1]) | .container_name = ($cparts[:-1] | join(\"-\")) | .container_id = ($cparts[-1] | rtrimstr(\".log\")) | .cluster_name = \"cluster_name\" | .; .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // \"kube:container:\\(.container_name)\")" # = filters for journald logs = @@ -219,14 +219,17 @@ data: namespace container_name container_id + cluster_name @type memory chunk_limit_records 100000 + chunk_limit_size 200m flush_interval 5s flush_thread_count 1 overflow_action block retry_max_times 3 + total_limit_size 600m @type json diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index fab6d642..02c3a662 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -6,13 +6,13 @@ metadata: labels: app: splunk-kubernetes-logging engine: fluentd - version: 1.0.0 + version: 1.1.0 spec: template: metadata: labels: app: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 annotations: {} spec: tolerations: @@ -20,8 +20,8 @@ spec: key: node-role.kubernetes.io/master containers: - name: splunk-fluentd-k8s-logs - image: splunk/fluentd-hec:1.0.1 - imagePullPolicy: IfNotPresent + image: splunk/fluentd-hec:1.1.0 + imagePullPolicy: Always args: - "-c" - "/fluentd/etc/fluent.conf" diff --git a/manifests/splunk-kubernetes-logging/secret.yaml b/manifests/splunk-kubernetes-logging/secret.yaml index 46ad0ca8..0e9acaae 100644 --- a/manifests/splunk-kubernetes-logging/secret.yaml +++ b/manifests/splunk-kubernetes-logging/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-logging labels: app: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-metrics/clusterRole.yaml b/manifests/splunk-kubernetes-metrics/clusterRole.yaml new file mode 100644 index 00000000..22fe784d --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/clusterRole.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubelet-summary-api-read + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +rules: +- apiGroups: + - '' + resources: + - nodes + - nodes/stats + - nodes/metrics + verbs: + - get diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml new file mode 100644 index 00000000..d2c7b64f --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-api-aggregator + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +rules: +- apiGroups: + - '' + resources: + - nodes + - nodes/stats + - nodes/proxy + - pods + verbs: + - get + - list diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml index 287a90dc..4bf9780c 100644 --- a/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml +++ b/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml @@ -5,11 +5,11 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:heapster + name: kubelet-summary-api-read subjects: - kind: ServiceAccount name: splunk-kubernetes-metrics diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml new file mode 100644 index 00000000..42f39156 --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: splunk-kubernetes-metrics-aggregator + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-api-aggregator +subjects: +- kind: ServiceAccount + name: splunk-kubernetes-metrics + namespace: default diff --git a/manifests/splunk-kubernetes-metrics/configMap.yaml b/manifests/splunk-kubernetes-metrics/configMap.yaml index 9d878718..c190c455 100644 --- a/manifests/splunk-kubernetes-metrics/configMap.yaml +++ b/manifests/splunk-kubernetes-metrics/configMap.yaml @@ -5,35 +5,55 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 data: fluent.conf: | # system wide configurations log_level info - - @type udp - tag raw.metrics.udp - port 9001 - message_length_limit 100m - - @type jq - jq 'split("\n") | reduce .[] as $item ({}; ($item | rindex(":")) as $i | .[$item[:$i]] = ($item[$i+1:-2] | tonumber))' - + @type kubernetes_metrics + tag kube.* + node_name "#{ENV['SPLUNK_HEC_HOST']}" + use_rest_client_ssl true + cluster_name cluster_name - - - @type jq - jq "def extract_labels:\n . as $labels | reduce range(length) as $n ({}; if $n % 2 == 0 then .[\"label.\" + $labels[$n]] = $labels[$n + 1] else . end);\n\ndef extract_metric:\n if length % 2 == 0\n then (.[:-2] | extract_labels) + {metric: (.[-2] | gsub(\"/\"; \".\")), resource_id: .[-1]}\n else (.[:-1] | extract_labels) + {metric: (.[-1] | gsub(\"/\"; \".\"))}\n end;\n \ndef extract_container:\n split(\".\") | {container_type: \"pod\", node: .[1], namespace: .[3], pod: .[5], container: .[7]} + (.[8:] | extract_metric) | .metric = \"kube.container.\" + .metric | . ;\n \ndef extract_syscontainer:\n split(\".\") | {container_type: \"sys\", node: .[1], container: .[3]} + (.[4:] | extract_metric) | .metric = \"kube.container.\" + .metric | . ;\n \ndef extract_pod:\n split(\".\") | {node: .[1], namespace: .[3], pod: .[5]} + (.[6:] | extract_metric) | .metric = \"kube.pod.\" + .metric | . ;\n \ndef extract_namespace:\n split(\".\") | {namespace: .[1]} + (.[2:] | extract_metric) | .metric = \"kube.namespace.\" + .metric | . ;\n \ndef extract_node:\n split(\".\") | {node: .[1]} + (.[2:] | extract_metric) | .metric = \"kube.node.\" + .metric | . ;\n \ndef extract_cluster:\n split(\".\") | .[1:] | extract_metric | .metric = \"kube.cluster.\" + .metric | . ;\n\ndef extract:\n if contains(\".container.\")\n then extract_container\n elif contains(\".sys-container.\")\n then extract_syscontainer\n elif contains(\".pod.\")\n then extract_pod\n elif startswith(\"namespace.\")\n then extract_namespace\n elif startswith(\"node.\")\n then extract_node\n elif startswith(\"cluster.\")\n then extract_cluster\n else {}\n end;\n\n \"heapster/namespace:\\(env.MY_NAMESPACE)/pod:\\(env.MY_POD_NAME)\" as $source | .record | to_entries | map({value, source: $source} + (.key | extract)) | ." - remove_tag_prefix raw - - - + + @type record_modifier + + metric_name ${tag} + cluster_name cluster_name + + + + @type record_modifier + + source ${record['node']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['container-name']} + + + @type splunk_hec data_type metric - metric_name_key metric + metric_name_key metric_name metric_value_key value protocol https hec_host MY-SPLUNK-HOST @@ -44,11 +64,12 @@ data: insecure_ssl true @type memory - chunk_limit_size 500m - chunk_limit_records 100000 + chunk_limit_records 1e+06 + chunk_limit_size 100m flush_interval 5s flush_thread_count 1 overflow_action block retry_max_times 3 + total_limit_size 400m diff --git a/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml new file mode 100644 index 00000000..df2d00d9 --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-kubernetes-metrics-aggregator + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +data: + fluent.conf: | + # system wide configurations + + log_level info + + + @type kubernetes_metrics_aggregator + tag kube.* + + + @type record_modifier + + metric_name ${tag} + cluster_name cluster_name + + + + @type record_modifier + + source ${record['name']} + + + + @type record_modifier + + source ${record['name']} + + + + @type record_modifier + + source ${record['node']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['container-name']} + + + + @type splunk_hec + data_type metric + metric_name_key metric_name + metric_value_key value + protocol https + hec_host MY-SPLUNK-HOST + hec_port 8088 + hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" + host "#{ENV['MY_NODE_NAME']}" + source source + insecure_ssl true + + @type memory + chunk_limit_records 1e+06 + chunk_limit_size 100m + flush_interval 5s + flush_thread_count 1 + overflow_action block + retry_max_times 3 + total_limit_size 400m + + diff --git a/manifests/splunk-kubernetes-metrics/deployment.yaml b/manifests/splunk-kubernetes-metrics/deployment.yaml index f3f05d2c..6e00c180 100644 --- a/manifests/splunk-kubernetes-metrics/deployment.yaml +++ b/manifests/splunk-kubernetes-metrics/deployment.yaml @@ -1,66 +1,37 @@ --- -apiVersion: apps/v1beta1 -kind: Deployment +apiVersion: apps/v1 +kind: DaemonSet metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - k8s-app: heapster-fluentd - version: 1.0.0 + engine: fluentd + version: 1.1.0 spec: - replicas: 1 selector: matchLabels: - app: splunk-kubernetes-metrics - k8s-app: heapster - version: 1.0.0 + name: splunk-kubernetes-metrics + version: 1.1.0 template: metadata: + name: splunk-kubernetes-metrics labels: + name: splunk-kubernetes-metrics app: splunk-kubernetes-metrics - k8s-app: heapster - version: 1.0.0 + engine: fluentd + version: 1.1.0 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: splunk-kubernetes-metrics + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master containers: - - image: k8s.gcr.io/heapster-amd64:v1.5.1 - imagePullPolicy: IfNotPresent - name: splunk-heapster - command: - - "/heapster" - - "--source=kubernetes" - - "--sink=statsd:udp://127.0.0.1:9001" - livenessProbe: - httpGet: - path: "/healthz" - port: 8082 - scheme: HTTP - initialDelaySeconds: 180 - timeoutSeconds: 5 - resources: - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - - image: splunk/fluentd-hec:1.0.1 - name: splunk-fluentd-heapster - imagePullPolicy: IfNotPresent - args: - - "-c" - - "/fluentd/etc/fluent.conf" + - name: splunk-fluentd-k8s-metrics + image: splunk/k8s-metrics:1.1.0 + imagePullPolicy: Always env: - - name: MY_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - name: SPLUNK_HEC_HOST valueFrom: fieldRef: @@ -72,10 +43,10 @@ spec: key: splunk_hec_token resources: limits: - cpu: 100m + cpu: 200m memory: 300Mi requests: - cpu: 100m + cpu: 200m memory: 300Mi volumeMounts: - name: conf-configmap diff --git a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml new file mode 100644 index 00000000..efd98b9d --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: splunk-kubernetes-metrics-agg + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +spec: + replicas: 1 + selector: + matchLabels: + app: splunk-kubernetes-metrics + version: 1.1.0 + template: + metadata: + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: splunk-kubernetes-metrics + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + containers: + - name: splunk-fluentd-k8s-metrics-agg + image: splunk/k8s-metrics-aggr:1.1.0 + imagePullPolicy: Always + env: + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: splunk-kubernetes-metrics + key: splunk_hec_token + resources: + limits: + cpu: 200m + memory: 300Mi + requests: + cpu: 200m + memory: 300Mi + volumeMounts: + - name: conf-configmap + mountPath: "/fluentd/etc" + - name: secrets + mountPath: "/fluentd/etc/splunk" + readOnly: true + volumes: + - name: conf-configmap + configMap: + name: splunk-kubernetes-metrics-aggregator + - name: secrets + secret: + secretName: splunk-kubernetes-metrics diff --git a/manifests/splunk-kubernetes-metrics/secret.yaml b/manifests/splunk-kubernetes-metrics/secret.yaml index 210caa46..f7e1c77d 100644 --- a/manifests/splunk-kubernetes-metrics/secret.yaml +++ b/manifests/splunk-kubernetes-metrics/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-metrics/serviceAccount.yaml b/manifests/splunk-kubernetes-metrics/serviceAccount.yaml index f40ad992..5c1f46fb 100644 --- a/manifests/splunk-kubernetes-metrics/serviceAccount.yaml +++ b/manifests/splunk-kubernetes-metrics/serviceAccount.yaml @@ -5,4 +5,4 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 diff --git a/manifests/splunk-kubernetes-objects/clusterRole.yaml b/manifests/splunk-kubernetes-objects/clusterRole.yaml index f53154a2..55928486 100644 --- a/manifests/splunk-kubernetes-objects/clusterRole.yaml +++ b/manifests/splunk-kubernetes-objects/clusterRole.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 rules: - apiGroups: - '' diff --git a/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml b/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml index 7724a472..398dbcc5 100644 --- a/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml +++ b/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 roleRef: kind: ClusterRole name: splunk-kubernetes-objects diff --git a/manifests/splunk-kubernetes-objects/configMap.yaml b/manifests/splunk-kubernetes-objects/configMap.yaml index adecfe9c..f4de7f31 100644 --- a/manifests/splunk-kubernetes-objects/configMap.yaml +++ b/manifests/splunk-kubernetes-objects/configMap.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 data: fluent.conf: | @@ -36,6 +36,11 @@ data: jq '.record.source = "namespace:\(env.MY_NAMESPACE)/pod:\(env.MY_POD_NAME)" | .record.sourcetype = (.tag | gsub("\\\\."; ":")) | .record' + + @type jq_transformer + jq '.record.cluster_name = "cluster_name" | .record' + + @type splunk_hec protocol https @@ -46,11 +51,17 @@ data: source_key source sourcetype_key sourcetype insecure_ssl true + + cluster_name + @type memory - chunk_limit_size 200m chunk_limit_records 10000 + chunk_limit_size 200m flush_interval 3s + flush_thread_count 1 + overflow_action block retry_max_times 3 + total_limit_size 600m diff --git a/manifests/splunk-kubernetes-objects/deployment.yaml b/manifests/splunk-kubernetes-objects/deployment.yaml index 6faee87a..11b47c57 100644 --- a/manifests/splunk-kubernetes-objects/deployment.yaml +++ b/manifests/splunk-kubernetes-objects/deployment.yaml @@ -5,27 +5,27 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 spec: selector: matchLabels: app: splunk-kubernetes-objects engine: fluentd - version: 1.0.0 + version: 1.1.0 replicas: 1 template: metadata: labels: app: splunk-kubernetes-objects engine: fluentd - version: 1.0.0 + version: 1.1.0 annotations: {} spec: serviceAccountName: splunk-kubernetes-objects terminationGracePeriodSeconds: 30 containers: - name: splunk-fluentd-k8s-objects - image: splunk/kube-objects:1.0.1 + image: splunk/kube-objects:1.1.0 imagePullPolicy: IfNotPresent args: - "-c" diff --git a/manifests/splunk-kubernetes-objects/secret.yaml b/manifests/splunk-kubernetes-objects/secret.yaml index d4369009..ce1e022b 100644 --- a/manifests/splunk-kubernetes-objects/secret.yaml +++ b/manifests/splunk-kubernetes-objects/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-objects/serviceAccount.yaml b/manifests/splunk-kubernetes-objects/serviceAccount.yaml index 102fa513..f1a75f67 100644 --- a/manifests/splunk-kubernetes-objects/serviceAccount.yaml +++ b/manifests/splunk-kubernetes-objects/serviceAccount.yaml @@ -5,4 +5,4 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 diff --git a/tools/gen_manifest.rb b/tools/gen_manifest.rb index d6e2aa7a..b575b1d7 100644 --- a/tools/gen_manifest.rb +++ b/tools/gen_manifest.rb @@ -2,7 +2,7 @@ require 'yaml' release_name = 'RELEASE-NAME' -@version = '1.0.0' +@version = File.read('VERSION').strip name_prefix = "#{release_name}-splunk-kubernetes-" def sanitize_yaml(yaml) From 39aa066bd44c04ba6f400d2a41e448324df3d1d0 Mon Sep 17 00:00:00 2001 From: gp510 Date: Thu, 28 Feb 2019 15:16:21 -0800 Subject: [PATCH 050/119] Updated index search for logging func tests (#102) --- test/k8s_logging_tests/test_config_logging.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/k8s_logging_tests/test_config_logging.py b/test/k8s_logging_tests/test_config_logging.py index 06d8595d..d403e1b5 100644 --- a/test/k8s_logging_tests/test_config_logging.py +++ b/test/k8s_logging_tests/test_config_logging.py @@ -19,7 +19,7 @@ def test_splunk_index(setup, test_input, expected): ''' logging.getLogger().info("testing test_splunk_index input={0} \ expected={1} event(s)".format(test_input, expected)) - search_query = "index=circleci_events" + search_query = "index=circleci_events OR index=kube-system" events = check_events_from_splunk(start_time="-1h@h", url=setup["splunkd_url"], user=setup["splunk_user"], @@ -39,7 +39,7 @@ def test_cluster_name(setup, test_input, expected): ''' logging.getLogger().info("testing test_clusterName input={0} \ expected={1} event(s)".format(test_input, expected)) - search_query = "index=circleci_events cluster_name=circleci-k8s-cluster | head 1" + search_query = "index=circleci_events OR index=kube-system cluster_name=circleci-k8s-cluster | head 1" events = check_events_from_splunk(start_time="-1h@h", url=setup["splunkd_url"], user=setup["splunk_user"], @@ -160,7 +160,7 @@ def test_sourcetype(setup, test_input, expected): events = check_events_from_splunk(start_time="-24h@h", url=setup["splunkd_url"], user=setup["splunk_user"], - query=["search index=circleci_events sourcetype={0}".format(test_input)], + query=["search index=circleci_events OR index=kube-system sourcetype={0}".format(test_input)], password=setup["splunk_password"]) logging.getLogger().info("Splunk received %s events in the last minute", len(events)) @@ -182,7 +182,7 @@ def test_source(setup, test_input, expected): events = check_events_from_splunk(start_time="-24h@h", url=setup["splunkd_url"], user=setup["splunk_user"], - query=["search index=circleci_events source={0}".format(test_input)], + query=["search index=circleci_events OR index=kube-system source={0}".format(test_input)], password=setup["splunk_password"]) logging.getLogger().info("Splunk received %s events in the last minute", len(events)) From a3859d2f5abc7c1e3abe049efcf0dd290e608d4d Mon Sep 17 00:00:00 2001 From: Shubham Jain Date: Thu, 28 Feb 2019 15:49:49 -0800 Subject: [PATCH 051/119] Update helm version on the fly (#101) * update helm version on the fly --- .circleci/deploy_connector.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh index 6649b2b7..951d8103 100755 --- a/.circleci/deploy_connector.sh +++ b/.circleci/deploy_connector.sh @@ -8,6 +8,8 @@ sudo apt-get -y install gnupg sudo mkdir ~/.kube echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg sudo mv config ~/.kube/config +#Update helm server version +helm init --upgrade #Make sure to check and clean previously failed deployment echo "Checking if previous deployment exist..." if [ "`helm ls`" == "" ]; then From 7b81c2b463f7a28fb7695c8f34978bdf9b342ec4 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 5 Mar 2019 14:08:21 -0800 Subject: [PATCH 052/119] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 712d851a..de1038c7 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ Splunk Connect for Kubernetes supports installation using Helm. Ensure that you Make sure you do the following before you install: 1. Create a minimum of two Splunk indexes: -* one events index, which will handle logs and objects (you may also create two separate indexes for logs and objects). -* one metrics index. +* One events index, which will handle logs and objects (you may also create two separate indexes for logs and objects). +* One metrics index. If you do not configure these indexes, Kubernetes Connect for Splunk uses the defaults created in your HEC token. 2. Create a HEC token if you do not already have one. If you are installing the connector on Splunk Cloud, file a ticket with Splunk Customer Service and they will deploy the indexes for your environment and generate your HEC token. From f967abc34cea5d084217b7dfe036b190cdc300d1 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 5 Mar 2019 14:14:47 -0800 Subject: [PATCH 053/119] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index de1038c7..b05037ea 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,11 @@ To configure the Splunk Connector for Kubernetes using YAML files: Splunk Connect for Kubernetes deploys a daemonset on each node. And in the daemonset, a Fluentd container runs and does the collecting job. Splunk Connector for Kubernetes collects three types of data: -* logs: Splunk Connectr for Kubernetes collects two types of logs: - * logs from Kubernetes system components (https://kubernetes.io/docs/concepts/overview/components/) - * applications (container) logs -* [objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) -* metrics +* Logs: Splunk Connect for Kubernetes collects two types of logs: + * Logs from Kubernetes system components (https://kubernetes.io/docs/concepts/overview/components/) + * Applications (container) logs +* [Objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) +* Metrics To collect the data, Splunk leverages: @@ -113,7 +113,7 @@ If you want to learn more about which metrics are collected and labels used with # Performance -Some parameters used with Splunk Connect for Kubernetes can have an impact on overall performance of log ingestion, objects, or metrics. In general, the more filters that are added to one of the streams, the greater the preformance impact. +Some parameters used with Splunk Connect for Kubernetes can have an impact on overall performance of log ingestion, objects, or metrics. In general, the more filters that are added to one of the streams, the greater the performance impact. Splunk Connect for Kubernetes can exceed the default throughput of HEC. To best address capacity needs, Splunk recommends that you monitor the HEC throughput and back pressure on Splunk Connect for Kubernetes deployments and be prepared to add additional nodes as needed. From 43437cea8ee6a535d6cf7e16c11720bcc43b77e6 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 5 Mar 2019 14:48:29 -0800 Subject: [PATCH 054/119] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b05037ea..746b6147 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Make sure your Splunk configuration has a metrics index that is able to receive If you want to learn more about how metrics are monitored in a Kubernetes cluster, see Tools for [Monitoring Compute, Storage, and Network Resources](https://kubernetes.io/docs/tasks/debug-application-cluster/resource-usage-monitoring/). -If you want to learn more about which metrics are collected and labels used with Splunk Connect for Kubernetes, view the metrics [schema](https://github.com/kubernetes/heapster/blob/master/docs/storage-schema.md#metrics) within the Heapster docs. +If you want to learn more about which metrics are collected and labels used with Splunk Connect for Kubernetes, view the metrics [schema](https://github.com/splunk/fluent-plugin-kubernetes-metrics). # Performance From faad8d5b090b71c85fb96d2b19a5b2798a387343 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 6 Mar 2019 15:00:28 -0800 Subject: [PATCH 055/119] Added automated release for github (#100) --- .circleci/build_sck_artifacts_for_release.sh | 42 ++++++++++ .circleci/config.yml | 46 +++++++++- .circleci/deploy_connector_release.sh | 25 ++++++ .circleci/push_sck_release.sh | 10 +++ .circleci/sck_values.yml | 2 +- .circleci/update_sck_config.sh | 2 +- .circleci/update_sck_config_for_release.sh | 51 +++++++++++ README.md | 7 +- VERSION | 1 + .../splunk-connect-for-kubernetes/Chart.yaml | 8 +- .../requirements.yaml | 6 +- .../splunk-kubernetes-logging/Chart.yaml | 7 +- .../splunk-kubernetes-logging/values.yaml | 2 +- .../splunk-kubernetes-metrics/Chart.yaml | 8 +- .../splunk-kubernetes-metrics/values.yaml | 6 +- .../splunk-kubernetes-objects/Chart.yaml | 7 +- .../splunk-kubernetes-objects/values.yaml | 2 +- .../splunk-kubernetes-logging/configMap.yaml | 7 +- .../splunk-kubernetes-logging/daemonset.yaml | 8 +- .../splunk-kubernetes-logging/secret.yaml | 2 +- .../clusterRole.yaml | 17 ++++ .../clusterRoleAggregator.yaml | 19 +++++ .../clusterRoleBinding.yaml | 4 +- .../clusterRoleBindingAggregator.yaml | 16 ++++ .../splunk-kubernetes-metrics/configMap.yaml | 63 +++++++++----- .../configMapMetricsAggregator.yaml | 84 +++++++++++++++++++ .../splunk-kubernetes-metrics/deployment.yaml | 65 ++++---------- .../deploymentMetricsAggregator.yaml | 56 +++++++++++++ .../splunk-kubernetes-metrics/secret.yaml | 2 +- .../serviceAccount.yaml | 2 +- .../clusterRole.yaml | 2 +- .../clusterRoleBinding.yaml | 2 +- .../splunk-kubernetes-objects/configMap.yaml | 15 +++- .../splunk-kubernetes-objects/deployment.yaml | 8 +- .../splunk-kubernetes-objects/secret.yaml | 2 +- .../serviceAccount.yaml | 2 +- tools/gen_manifest.rb | 2 +- 37 files changed, 490 insertions(+), 120 deletions(-) create mode 100755 .circleci/build_sck_artifacts_for_release.sh create mode 100755 .circleci/deploy_connector_release.sh create mode 100755 .circleci/push_sck_release.sh create mode 100755 .circleci/update_sck_config_for_release.sh create mode 100644 VERSION create mode 100644 manifests/splunk-kubernetes-metrics/clusterRole.yaml create mode 100644 manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml create mode 100644 manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml create mode 100644 manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml create mode 100644 manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml diff --git a/.circleci/build_sck_artifacts_for_release.sh b/.circleci/build_sck_artifacts_for_release.sh new file mode 100755 index 00000000..8e0dca12 --- /dev/null +++ b/.circleci/build_sck_artifacts_for_release.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -e +TAG=`cat VERSION` + +function replace_generic_version () +{ + file="$1" + _line=`awk '/version:/{print NR;exit}' $file` + replacement="version: $TAG" + # Escape backslash, forward slash and ampersand for use as a sed replacement. + replacement_escaped=$( echo "$replacement" | sed -e 's/[\/&]/\\&/g' ) + sed -i "${_line}s/.*/$replacement_escaped/" "$file" +} + +repos_array=( "helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) +sub_repos_array=( "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) + +for repo in "${repos_array[@]}" +do + filename="${repo}/Chart.yaml" + replace_generic_version $filename +done + +mkdir helm-artifacts-release +if [[ -d "helm-chart/splunk-connect-for-kubernetes/charts" ]]; then + rm -rf helm-chart/splunk-connect-for-kubernetes/charts +fi +mkdir helm-chart/splunk-connect-for-kubernetes/charts + +for sub_repo in "${sub_repos_array[@]}" +do + cp -rp $sub_repo helm-chart/splunk-connect-for-kubernetes/charts +done + +for repo in "${repos_array[@]}" +do + helm package -d helm-artifacts-release $repo +done + +rm -rf helm-chart/splunk-connect-for-kubernetes/charts diff --git a/.circleci/config.yml b/.circleci/config.yml index 9fd9e533..c8a862a2 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,6 +76,44 @@ jobs: command: | python .circleci/performance/run_perf_scenarios.py + release: + docker: + - image: circleci/ruby:2.6.1-node-browsers + working_directory: ~/repo + steps: + - checkout + - setup_remote_docker: + reusable: true + - run: + name: Install dependencies + command: | + sudo apt-get update + sudo apt-get install build-essential libpq-dev libssl-dev openssl libffi-dev zlib1g-dev > /dev/null 2>&1 + sudo apt-get install python3-pip python3-dev > /dev/null 2>&1 + sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash + helm init -c + - run: + name: Update configurables to build artifacts + command: | + .circleci/update_sck_config_for_release.sh + - run: + name: Build SCK artifacts + command: | + .circleci/build_sck_artifacts_for_release.sh + - run: + name: Push SCK artifacts to Github Release + command: | + .circleci/push_sck_release.sh + - run: + name: Deploy k8s connector + command: | + .circleci/deploy_connector_release.sh + - run: + name: Run functional tests + command: | + .circleci/run_functional_tests.sh + # Need to add teardown step workflows: @@ -89,4 +127,10 @@ workflows: - perf-test: filters: branches: - only: perf-testing \ No newline at end of file + only: perf-testing + - release: + requires: + - build + filters: + branches: + only: release-testing \ No newline at end of file diff --git a/.circleci/deploy_connector_release.sh b/.circleci/deploy_connector_release.sh new file mode 100755 index 00000000..1036c7ae --- /dev/null +++ b/.circleci/deploy_connector_release.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -e +echo "Setup kube client..." +curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +sudo apt-get -y install gnupg +sudo mkdir ~/.kube +echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg +sudo mv config ~/.kube/config +#Update helm server version +helm init --upgrade +#Make sure to check and clean previously failed deployment +echo "Checking if previous deployment exist..." +if [ "`helm ls`" == "" ]; then + echo "Nothing to clean, ready for deployment" +else + helm delete --purge $(helm ls --short) +fi +echo "Deploying k8s-connect with latest changes" +VERSION=`cat VERSION` +helm install --name=release -f .circleci/sck_values.yml helm-artifacts-release/splunk-connect-for-kubernetes*.tgz +#Todo-need to add test cases and run them before cleaning the deployment +#wait for deployment to finish, sleep longer for metrics-aggr to start sending metrics +sleep 90 \ No newline at end of file diff --git a/.circleci/push_sck_release.sh b/.circleci/push_sck_release.sh new file mode 100755 index 00000000..a698d20b --- /dev/null +++ b/.circleci/push_sck_release.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e +wget https://github.com/tcnksm/ghr/releases/download/v0.12.0/ghr_v0.12.0_linux_amd64.tar.gz +tar -xzvf ghr_v0.12.0_linux_amd64.tar.gz +sudo chmod +x ghr_v0.12.0_linux_amd64 +sudo mv ghr_v0.12.0_linux_amd64/ghr /usr/local/bin/ghr + +VERSION=`cat VERSION` +echo "Pushing SCK release to github releases...${VERSION}" +ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -n "${RELEASE_TITLE}" -b "${RELEASE_BODY}" -draft ${VERSION} ./helm-artifacts-release/ diff --git a/.circleci/sck_values.yml b/.circleci/sck_values.yml index 4f8eb41e..740d550d 100755 --- a/.circleci/sck_values.yml +++ b/.circleci/sck_values.yml @@ -1,5 +1,5 @@ global: - logLevel: info + logLevel: debug splunk: hec: protocol: https diff --git a/.circleci/update_sck_config.sh b/.circleci/update_sck_config.sh index 88fa730b..a7bb37c0 100755 --- a/.circleci/update_sck_config.sh +++ b/.circleci/update_sck_config.sh @@ -12,7 +12,7 @@ yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST_PERF yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN_PERF -#Pull docker images locally +# Pull docker images locally aws ecr get-login --region $AWS_REGION --no-include-email | bash docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-logging:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-metrics:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' diff --git a/.circleci/update_sck_config_for_release.sh b/.circleci/update_sck_config_for_release.sh new file mode 100755 index 00000000..1a2a778f --- /dev/null +++ b/.circleci/update_sck_config_for_release.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e +VERSION=`cat VERSION` + +# Install yq yaml parser +wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 +sudo chmod +x yq_linux_amd64 +sudo mv yq_linux_amd64 /usr/local/bin/yq + +## Modify docker images to be used +#image_name=$(echo "splunk/fluentd-hec:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name $image_name +# +#image_name=$(echo "splunk/k8s-metrics:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name $image_name +# +#image_name=$(echo "splunk/k8s-metrics-aggr:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name $image_name +# +#image_name=$(echo "splunk/kube-objects:$VERSION") +#yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name $image_name +# +#yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION +#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION +#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml version $VERSION +#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml version $VERSION +# +#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml dependencies.splunk-kubernetes-logging.version $VERSION +#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml dependencies.splunk-kubernetes-metrics.version $VERSION +#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml dependencies.splunk-kubernetes-objects.version $VERSION + + +# Modify splunk environment values +yq w -i .circleci/sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST +yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN + +# Pull docker images locally +docker pull splunk/fluentd-hec:1.1.0 +docker pull splunk/k8s-metrics:1.1.0 +docker pull splunk/k8s-metrics-aggr:1.1.0 +docker pull splunk/kube-objects:1.1.0 + +# Modify docker images to be used +yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec:1.1.0 + +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics:1.1.0 + +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name splunk/k8s-metrics-aggr:1.1.0 + +yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name splunk/kube-objects:1.1.0 diff --git a/README.md b/README.md index 746b6147..9c71fa61 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,11 @@ Splunk Connect for Kubernetes can exceed the default throughput of HEC. To best One possible filter option is to enable the processing of multi-line events. This feature is currently experimental and considered to be community supported. +# Maintenance And Support + +The current maintainers of this project are the DataEdge team at Splunk. You can reach us at [DataEdge@splunk.com](mailto:DataEdge@splunk.com). +If you have any issues with the software, please file an issue at [Create a new issue in splunk connect for kubernetes project](https://github.com/splunk/splunk-connect-for-kubernetes/issues/new) + # License -See [LICENSE](LICENSE). +See [LICENSE](https://github.com/splunk/splunk-connect-for-kubernetes/blob/master/LICENSE). diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..1cc5f657 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.1.0 \ No newline at end of file diff --git a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml index 4903b1b8..fcb98667 100644 --- a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml @@ -1,15 +1,12 @@ apiVersion: v1 name: splunk-connect-for-kubernetes -version: 1.0.1 +version: 1.1.0 description: Collect logs, objects (metadata), and metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart - https://github.com/splunk/fluent-plugin-splunk-hec - https://github.com/splunk/fluent-plugin-kubernetes-objects - - https://github.com/splunk/docker-fluentd-hec - - https://github.com/splunk/docker-kube-objects - - https://github.com/kubernetes/heapster keywords: - splunk - monitoring @@ -18,4 +15,5 @@ keywords: - metadata - objects maintainers: - - name: Gimi Liang + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-connect-for-kubernetes/requirements.yaml b/helm-chart/splunk-connect-for-kubernetes/requirements.yaml index 8f597300..1d587f84 100644 --- a/helm-chart/splunk-connect-for-kubernetes/requirements.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/requirements.yaml @@ -1,13 +1,13 @@ dependencies: - name: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 repository: stable condition: logging.enabled - name: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 repository: stable condition: objects.enabled - name: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 repository: stable condition: metrics.enabled diff --git a/helm-chart/splunk-kubernetes-logging/Chart.yaml b/helm-chart/splunk-kubernetes-logging/Chart.yaml index 1c53be5b..8c3db932 100644 --- a/helm-chart/splunk-kubernetes-logging/Chart.yaml +++ b/helm-chart/splunk-kubernetes-logging/Chart.yaml @@ -1,16 +1,15 @@ apiVersion: v1 -version: 1.0.1 +version: 1.1.0 name: splunk-kubernetes-logging description: Collect logs with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-logging sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-logging - https://github.com/splunk/fluent-plugin-splunk-hec - - https://github.com/splunk/docker-fluentd-hec keywords: - splunk - monitoring - logging maintainers: - - name: Gimi Liang - email: zliang@splunk.com + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 9e3afa1b..4e7a98bf 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -221,7 +221,7 @@ logs: # Defines which version of image to use, and how it should be pulled. image: - name: splunk/fluentd-hec:1.1.0.Alpha + name: splunk/fluentd-hec:1.1.0 pullPolicy: Always diff --git a/helm-chart/splunk-kubernetes-metrics/Chart.yaml b/helm-chart/splunk-kubernetes-metrics/Chart.yaml index 5b6f18f0..3fe17bf9 100644 --- a/helm-chart/splunk-kubernetes-metrics/Chart.yaml +++ b/helm-chart/splunk-kubernetes-metrics/Chart.yaml @@ -1,17 +1,15 @@ apiVersion: v1 -version: 1.0.1 +version: 1.1.0 name: splunk-kubernetes-metrics description: Collect metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-metrics sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-metrics - https://github.com/splunk/fluent-plugin-splunk-hec - - https://github.com/splunk/docker-fluentd-hec - - https://github.com/kubernetes/heapster keywords: - splunk - monitoring - metric maintainers: - - name: Gimi Liang - email: zliang@splunk.com + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index ede1ba63..1be791e0 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -8,7 +8,7 @@ # Global values have lower priority. # Values defined here are the default values. global: - logLevel: debug + logLevel: info splunk: hec: port: 8088 @@ -74,12 +74,12 @@ splunk: image: - name: splunk/k8s-metrics:1.1.0.Alpha + name: splunk/k8s-metrics:1.1.0 pullPolicy: Always imageAgg: - name: splunk/k8s-metrics-aggr:1.1.0.Alpha + name: splunk/k8s-metrics-aggr:1.1.0 pullPolicy: Always diff --git a/helm-chart/splunk-kubernetes-objects/Chart.yaml b/helm-chart/splunk-kubernetes-objects/Chart.yaml index b52868b2..b6b4bb11 100644 --- a/helm-chart/splunk-kubernetes-objects/Chart.yaml +++ b/helm-chart/splunk-kubernetes-objects/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -version: 1.0.1 +version: 1.1.0 name: splunk-kubernetes-objects description: Collect Kubernetes objects with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-objects @@ -7,12 +7,11 @@ sources: - https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-objects - https://github.com/splunk/fluent-plugin-splunk-hec - https://github.com/splunk/fluent-plugin-kubernetes-objects - - https://github.com/splunk/docker-kube-objects keywords: - splunk - monitoring - objects - metadata maintainers: - - name: Gimi Liang - email: zliang@splunk.com + - name: Splunk Inc. + email: DataEdge@splunk.com diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index e9aab0f6..a39d4b48 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -165,7 +165,7 @@ splunk: image: - name: splunk/kube-objects:1.0.1 + name: splunk/kube-objects:1.1.0 pullPolicy: IfNotPresent diff --git a/manifests/splunk-kubernetes-logging/configMap.yaml b/manifests/splunk-kubernetes-logging/configMap.yaml index 986cf7af..b86e1c01 100644 --- a/manifests/splunk-kubernetes-logging/configMap.yaml +++ b/manifests/splunk-kubernetes-logging/configMap.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-logging labels: app: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 data: fluent.conf: |- @include system.conf @@ -177,7 +177,7 @@ data: # extract index fields and sourcetype for container logs @type jq_transformer - jq "def find_sourcetype(pod; container_name): container_name + \"/\" + pod | if startswith(\"dns-controller/dns-controller\") then \"kube:dns-controller\" elif startswith(\"sidecar/kube-dns\") then \"kube:kubedns-sidecar\" elif startswith(\"dnsmasq/kube-dns\") then \"kube:dnsmasq\" elif startswith(\"etcd-container/etcd-server\") then \"kube:etcd\" elif startswith(\"etcd-container/etcd-server-events\") then \"kube:etcd-events\" elif startswith(\"kube-apiserver/kube-apiserver\") then \"kube:kube-apiserver\" elif startswith(\"kube-controller-manager/kube-controller-manager\") then \"kube:kube-controller-manager\" elif startswith(\"autoscaler/kube-dns-autoscaler\") then \"kube:kube-dns-autoscaler\" elif startswith(\"kube-proxy/kube-proxy\") then \"kube:kube-proxy\" elif startswith(\"kube-scheduler/kube-scheduler\") then \"kube:kube-scheduler\" elif startswith(\"kubedns/kube-dns\") then \"kube:kubedns\" else empty end; def extract_container_info: (.source | ltrimstr(\"/var/log/containers/\") | split(\"_\")) as $parts | ($parts[-1] | split(\"-\")) as $cparts | .pod = $parts[0] | .namespace = $parts[1] | .container_name = ($cparts[:-1] | join(\"-\")) | .container_id = ($cparts[-1] | rtrimstr(\".log\")) | .; .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // \"kube:container:\\(.container_name)\")" + jq "def find_sourcetype(pod; container_name): container_name + \"/\" + pod | if startswith(\"dns-controller/dns-controller\") then \"kube:dns-controller\" elif startswith(\"sidecar/kube-dns\") then \"kube:kubedns-sidecar\" elif startswith(\"dnsmasq/kube-dns\") then \"kube:dnsmasq\" elif startswith(\"etcd-container/etcd-server\") then \"kube:etcd\" elif startswith(\"etcd-container/etcd-server-events\") then \"kube:etcd-events\" elif startswith(\"kube-apiserver/kube-apiserver\") then \"kube:kube-apiserver\" elif startswith(\"kube-controller-manager/kube-controller-manager\") then \"kube:kube-controller-manager\" elif startswith(\"autoscaler/kube-dns-autoscaler\") then \"kube:kube-dns-autoscaler\" elif startswith(\"kube-proxy/kube-proxy\") then \"kube:kube-proxy\" elif startswith(\"kube-scheduler/kube-scheduler\") then \"kube:kube-scheduler\" elif startswith(\"kubedns/kube-dns\") then \"kube:kubedns\" else empty end; def set_namespace(value): if value == \"default\" then \"main\"else value end; def extract_container_info: (.source | ltrimstr(\"/var/log/containers/\") | split(\"_\")) as $parts | ($parts[-1] | split(\"-\")) as $cparts | .pod = $parts[0] | .namespace = set_namespace($parts[1]) | .container_name = ($cparts[:-1] | join(\"-\")) | .container_id = ($cparts[-1] | rtrimstr(\".log\")) | .cluster_name = \"cluster_name\" | .; .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // \"kube:container:\\(.container_name)\")" # = filters for journald logs = @@ -219,14 +219,17 @@ data: namespace container_name container_id + cluster_name @type memory chunk_limit_records 100000 + chunk_limit_size 200m flush_interval 5s flush_thread_count 1 overflow_action block retry_max_times 3 + total_limit_size 600m @type json diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index fab6d642..02c3a662 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -6,13 +6,13 @@ metadata: labels: app: splunk-kubernetes-logging engine: fluentd - version: 1.0.0 + version: 1.1.0 spec: template: metadata: labels: app: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 annotations: {} spec: tolerations: @@ -20,8 +20,8 @@ spec: key: node-role.kubernetes.io/master containers: - name: splunk-fluentd-k8s-logs - image: splunk/fluentd-hec:1.0.1 - imagePullPolicy: IfNotPresent + image: splunk/fluentd-hec:1.1.0 + imagePullPolicy: Always args: - "-c" - "/fluentd/etc/fluent.conf" diff --git a/manifests/splunk-kubernetes-logging/secret.yaml b/manifests/splunk-kubernetes-logging/secret.yaml index 46ad0ca8..0e9acaae 100644 --- a/manifests/splunk-kubernetes-logging/secret.yaml +++ b/manifests/splunk-kubernetes-logging/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-logging labels: app: splunk-kubernetes-logging - version: 1.0.0 + version: 1.1.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-metrics/clusterRole.yaml b/manifests/splunk-kubernetes-metrics/clusterRole.yaml new file mode 100644 index 00000000..22fe784d --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/clusterRole.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubelet-summary-api-read + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +rules: +- apiGroups: + - '' + resources: + - nodes + - nodes/stats + - nodes/metrics + verbs: + - get diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml new file mode 100644 index 00000000..d2c7b64f --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-api-aggregator + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +rules: +- apiGroups: + - '' + resources: + - nodes + - nodes/stats + - nodes/proxy + - pods + verbs: + - get + - list diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml index 287a90dc..4bf9780c 100644 --- a/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml +++ b/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml @@ -5,11 +5,11 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:heapster + name: kubelet-summary-api-read subjects: - kind: ServiceAccount name: splunk-kubernetes-metrics diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml new file mode 100644 index 00000000..42f39156 --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: splunk-kubernetes-metrics-aggregator + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-api-aggregator +subjects: +- kind: ServiceAccount + name: splunk-kubernetes-metrics + namespace: default diff --git a/manifests/splunk-kubernetes-metrics/configMap.yaml b/manifests/splunk-kubernetes-metrics/configMap.yaml index 9d878718..c190c455 100644 --- a/manifests/splunk-kubernetes-metrics/configMap.yaml +++ b/manifests/splunk-kubernetes-metrics/configMap.yaml @@ -5,35 +5,55 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 data: fluent.conf: | # system wide configurations log_level info - - @type udp - tag raw.metrics.udp - port 9001 - message_length_limit 100m - - @type jq - jq 'split("\n") | reduce .[] as $item ({}; ($item | rindex(":")) as $i | .[$item[:$i]] = ($item[$i+1:-2] | tonumber))' - + @type kubernetes_metrics + tag kube.* + node_name "#{ENV['SPLUNK_HEC_HOST']}" + use_rest_client_ssl true + cluster_name cluster_name - - - @type jq - jq "def extract_labels:\n . as $labels | reduce range(length) as $n ({}; if $n % 2 == 0 then .[\"label.\" + $labels[$n]] = $labels[$n + 1] else . end);\n\ndef extract_metric:\n if length % 2 == 0\n then (.[:-2] | extract_labels) + {metric: (.[-2] | gsub(\"/\"; \".\")), resource_id: .[-1]}\n else (.[:-1] | extract_labels) + {metric: (.[-1] | gsub(\"/\"; \".\"))}\n end;\n \ndef extract_container:\n split(\".\") | {container_type: \"pod\", node: .[1], namespace: .[3], pod: .[5], container: .[7]} + (.[8:] | extract_metric) | .metric = \"kube.container.\" + .metric | . ;\n \ndef extract_syscontainer:\n split(\".\") | {container_type: \"sys\", node: .[1], container: .[3]} + (.[4:] | extract_metric) | .metric = \"kube.container.\" + .metric | . ;\n \ndef extract_pod:\n split(\".\") | {node: .[1], namespace: .[3], pod: .[5]} + (.[6:] | extract_metric) | .metric = \"kube.pod.\" + .metric | . ;\n \ndef extract_namespace:\n split(\".\") | {namespace: .[1]} + (.[2:] | extract_metric) | .metric = \"kube.namespace.\" + .metric | . ;\n \ndef extract_node:\n split(\".\") | {node: .[1]} + (.[2:] | extract_metric) | .metric = \"kube.node.\" + .metric | . ;\n \ndef extract_cluster:\n split(\".\") | .[1:] | extract_metric | .metric = \"kube.cluster.\" + .metric | . ;\n\ndef extract:\n if contains(\".container.\")\n then extract_container\n elif contains(\".sys-container.\")\n then extract_syscontainer\n elif contains(\".pod.\")\n then extract_pod\n elif startswith(\"namespace.\")\n then extract_namespace\n elif startswith(\"node.\")\n then extract_node\n elif startswith(\"cluster.\")\n then extract_cluster\n else {}\n end;\n\n \"heapster/namespace:\\(env.MY_NAMESPACE)/pod:\\(env.MY_POD_NAME)\" as $source | .record | to_entries | map({value, source: $source} + (.key | extract)) | ." - remove_tag_prefix raw - - - + + @type record_modifier + + metric_name ${tag} + cluster_name cluster_name + + + + @type record_modifier + + source ${record['node']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['container-name']} + + + @type splunk_hec data_type metric - metric_name_key metric + metric_name_key metric_name metric_value_key value protocol https hec_host MY-SPLUNK-HOST @@ -44,11 +64,12 @@ data: insecure_ssl true @type memory - chunk_limit_size 500m - chunk_limit_records 100000 + chunk_limit_records 1e+06 + chunk_limit_size 100m flush_interval 5s flush_thread_count 1 overflow_action block retry_max_times 3 + total_limit_size 400m diff --git a/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml new file mode 100644 index 00000000..df2d00d9 --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-kubernetes-metrics-aggregator + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +data: + fluent.conf: | + # system wide configurations + + log_level info + + + @type kubernetes_metrics_aggregator + tag kube.* + + + @type record_modifier + + metric_name ${tag} + cluster_name cluster_name + + + + @type record_modifier + + source ${record['name']} + + + + @type record_modifier + + source ${record['name']} + + + + @type record_modifier + + source ${record['node']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['name']} + + + + @type record_modifier + + source ${record['node']}/${record['pod-name']}/${record['container-name']} + + + + @type splunk_hec + data_type metric + metric_name_key metric_name + metric_value_key value + protocol https + hec_host MY-SPLUNK-HOST + hec_port 8088 + hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" + host "#{ENV['MY_NODE_NAME']}" + source source + insecure_ssl true + + @type memory + chunk_limit_records 1e+06 + chunk_limit_size 100m + flush_interval 5s + flush_thread_count 1 + overflow_action block + retry_max_times 3 + total_limit_size 400m + + diff --git a/manifests/splunk-kubernetes-metrics/deployment.yaml b/manifests/splunk-kubernetes-metrics/deployment.yaml index f3f05d2c..6e00c180 100644 --- a/manifests/splunk-kubernetes-metrics/deployment.yaml +++ b/manifests/splunk-kubernetes-metrics/deployment.yaml @@ -1,66 +1,37 @@ --- -apiVersion: apps/v1beta1 -kind: Deployment +apiVersion: apps/v1 +kind: DaemonSet metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - k8s-app: heapster-fluentd - version: 1.0.0 + engine: fluentd + version: 1.1.0 spec: - replicas: 1 selector: matchLabels: - app: splunk-kubernetes-metrics - k8s-app: heapster - version: 1.0.0 + name: splunk-kubernetes-metrics + version: 1.1.0 template: metadata: + name: splunk-kubernetes-metrics labels: + name: splunk-kubernetes-metrics app: splunk-kubernetes-metrics - k8s-app: heapster - version: 1.0.0 + engine: fluentd + version: 1.1.0 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: splunk-kubernetes-metrics + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master containers: - - image: k8s.gcr.io/heapster-amd64:v1.5.1 - imagePullPolicy: IfNotPresent - name: splunk-heapster - command: - - "/heapster" - - "--source=kubernetes" - - "--sink=statsd:udp://127.0.0.1:9001" - livenessProbe: - httpGet: - path: "/healthz" - port: 8082 - scheme: HTTP - initialDelaySeconds: 180 - timeoutSeconds: 5 - resources: - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 100m - memory: 300Mi - - image: splunk/fluentd-hec:1.0.1 - name: splunk-fluentd-heapster - imagePullPolicy: IfNotPresent - args: - - "-c" - - "/fluentd/etc/fluent.conf" + - name: splunk-fluentd-k8s-metrics + image: splunk/k8s-metrics:1.1.0 + imagePullPolicy: Always env: - - name: MY_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - name: SPLUNK_HEC_HOST valueFrom: fieldRef: @@ -72,10 +43,10 @@ spec: key: splunk_hec_token resources: limits: - cpu: 100m + cpu: 200m memory: 300Mi requests: - cpu: 100m + cpu: 200m memory: 300Mi volumeMounts: - name: conf-configmap diff --git a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml new file mode 100644 index 00000000..efd98b9d --- /dev/null +++ b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: splunk-kubernetes-metrics-agg + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 +spec: + replicas: 1 + selector: + matchLabels: + app: splunk-kubernetes-metrics + version: 1.1.0 + template: + metadata: + labels: + app: splunk-kubernetes-metrics + version: 1.1.0 + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: splunk-kubernetes-metrics + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + containers: + - name: splunk-fluentd-k8s-metrics-agg + image: splunk/k8s-metrics-aggr:1.1.0 + imagePullPolicy: Always + env: + - name: SPLUNK_HEC_TOKEN + valueFrom: + secretKeyRef: + name: splunk-kubernetes-metrics + key: splunk_hec_token + resources: + limits: + cpu: 200m + memory: 300Mi + requests: + cpu: 200m + memory: 300Mi + volumeMounts: + - name: conf-configmap + mountPath: "/fluentd/etc" + - name: secrets + mountPath: "/fluentd/etc/splunk" + readOnly: true + volumes: + - name: conf-configmap + configMap: + name: splunk-kubernetes-metrics-aggregator + - name: secrets + secret: + secretName: splunk-kubernetes-metrics diff --git a/manifests/splunk-kubernetes-metrics/secret.yaml b/manifests/splunk-kubernetes-metrics/secret.yaml index 210caa46..f7e1c77d 100644 --- a/manifests/splunk-kubernetes-metrics/secret.yaml +++ b/manifests/splunk-kubernetes-metrics/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-metrics/serviceAccount.yaml b/manifests/splunk-kubernetes-metrics/serviceAccount.yaml index f40ad992..5c1f46fb 100644 --- a/manifests/splunk-kubernetes-metrics/serviceAccount.yaml +++ b/manifests/splunk-kubernetes-metrics/serviceAccount.yaml @@ -5,4 +5,4 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.0.0 + version: 1.1.0 diff --git a/manifests/splunk-kubernetes-objects/clusterRole.yaml b/manifests/splunk-kubernetes-objects/clusterRole.yaml index f53154a2..55928486 100644 --- a/manifests/splunk-kubernetes-objects/clusterRole.yaml +++ b/manifests/splunk-kubernetes-objects/clusterRole.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 rules: - apiGroups: - '' diff --git a/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml b/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml index 7724a472..398dbcc5 100644 --- a/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml +++ b/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 roleRef: kind: ClusterRole name: splunk-kubernetes-objects diff --git a/manifests/splunk-kubernetes-objects/configMap.yaml b/manifests/splunk-kubernetes-objects/configMap.yaml index adecfe9c..f4de7f31 100644 --- a/manifests/splunk-kubernetes-objects/configMap.yaml +++ b/manifests/splunk-kubernetes-objects/configMap.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 data: fluent.conf: | @@ -36,6 +36,11 @@ data: jq '.record.source = "namespace:\(env.MY_NAMESPACE)/pod:\(env.MY_POD_NAME)" | .record.sourcetype = (.tag | gsub("\\\\."; ":")) | .record' + + @type jq_transformer + jq '.record.cluster_name = "cluster_name" | .record' + + @type splunk_hec protocol https @@ -46,11 +51,17 @@ data: source_key source sourcetype_key sourcetype insecure_ssl true + + cluster_name + @type memory - chunk_limit_size 200m chunk_limit_records 10000 + chunk_limit_size 200m flush_interval 3s + flush_thread_count 1 + overflow_action block retry_max_times 3 + total_limit_size 600m diff --git a/manifests/splunk-kubernetes-objects/deployment.yaml b/manifests/splunk-kubernetes-objects/deployment.yaml index 6faee87a..11b47c57 100644 --- a/manifests/splunk-kubernetes-objects/deployment.yaml +++ b/manifests/splunk-kubernetes-objects/deployment.yaml @@ -5,27 +5,27 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 spec: selector: matchLabels: app: splunk-kubernetes-objects engine: fluentd - version: 1.0.0 + version: 1.1.0 replicas: 1 template: metadata: labels: app: splunk-kubernetes-objects engine: fluentd - version: 1.0.0 + version: 1.1.0 annotations: {} spec: serviceAccountName: splunk-kubernetes-objects terminationGracePeriodSeconds: 30 containers: - name: splunk-fluentd-k8s-objects - image: splunk/kube-objects:1.0.1 + image: splunk/kube-objects:1.1.0 imagePullPolicy: IfNotPresent args: - "-c" diff --git a/manifests/splunk-kubernetes-objects/secret.yaml b/manifests/splunk-kubernetes-objects/secret.yaml index d4369009..ce1e022b 100644 --- a/manifests/splunk-kubernetes-objects/secret.yaml +++ b/manifests/splunk-kubernetes-objects/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-objects/serviceAccount.yaml b/manifests/splunk-kubernetes-objects/serviceAccount.yaml index 102fa513..f1a75f67 100644 --- a/manifests/splunk-kubernetes-objects/serviceAccount.yaml +++ b/manifests/splunk-kubernetes-objects/serviceAccount.yaml @@ -5,4 +5,4 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.0.0 + version: 1.1.0 diff --git a/tools/gen_manifest.rb b/tools/gen_manifest.rb index d6e2aa7a..b575b1d7 100644 --- a/tools/gen_manifest.rb +++ b/tools/gen_manifest.rb @@ -2,7 +2,7 @@ require 'yaml' release_name = 'RELEASE-NAME' -@version = '1.0.0' +@version = File.read('VERSION').strip name_prefix = "#{release_name}-splunk-kubernetes-" def sanitize_yaml(yaml) From 11a70dadbcc76ab14799f7be3779330a72e5d3c1 Mon Sep 17 00:00:00 2001 From: Donald Tregonning Date: Wed, 6 Mar 2019 17:47:20 -0800 Subject: [PATCH 056/119] Update all references from v1.0.1 to v1.1.0, doc updates, chart (#104) --- LICENSE | 1 - README.md | 27 ++++++++++++++----- .../splunk-connect-for-kubernetes/OWNERS | 4 +++ .../splunk-connect-for-kubernetes/README.md | 4 +-- .../templates/NOTES.txt | 4 +-- helm-chart/splunk-kubernetes-logging/OWNERS | 4 +++ .../splunk-kubernetes-logging/README.md | 2 +- helm-chart/splunk-kubernetes-metrics/OWNERS | 4 +++ .../splunk-kubernetes-metrics/README.md | 16 +++++++---- helm-chart/splunk-kubernetes-objects/OWNERS | 4 +++ .../splunk-kubernetes-objects/README.md | 2 +- 11 files changed, 54 insertions(+), 18 deletions(-) diff --git a/LICENSE b/LICENSE index 520439cd..bd425384 100644 --- a/LICENSE +++ b/LICENSE @@ -218,7 +218,6 @@ The following components are provided under the Apache License 2.0. See project (Apache License 2.0) fluentd-plugin-splunk-hec (https://github.com/splunk/fluent-plugin-splunk-hec/blob/master/LICENSE) (Apache License 2.0) fluentd-plugin-kubernetes-objects (https://github.com/splunk/fluent-plugin-kubernetes-objects/blob/master/LICENSE) (Apache License 2.0) fluent-plugin-prometheus (https://github.com/fluent/fluent-plugin-prometheus/blob/master/LICENSE) - (Apache License 2.0) heapster (https://github.com/kubernetes/heapster/blob/master/LICENSE) ======================================================================== MIT licenses diff --git a/README.md b/README.md index 9c71fa61..1f1edafa 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ Splunk Connect for Kubernetes provides a way to import and search your Kubernete * Splunk Enterprise 7.0 or later * An HEC token. See the following topics for more information: - * http://docs.splunk.com/Documentation/Splunk/7.0.3/Data/UsetheHTTPEventCollector - * http://docs.splunk.com/Documentation/Splunk/7.0.3/Data/ScaleHTTPEventCollector + * http://docs.splunk.com/Documentation/Splunk/7.2.4/Data/UsetheHTTPEventCollector + * http://docs.splunk.com/Documentation/Splunk/7.2.4/Data/ScaleHTTPEventCollector * You should be familiar with your Kubernetes configuration and know where your log info is collected in Kubernetes. * You must have administrator access to your Kubernetes cluster. * To install using Helm (recommended), make sure you are running Helm in your Kubernetes configuration. See https://github.com/kubernetes/helm @@ -35,7 +35,7 @@ Helm, maintained by the CNCF, allows the Kubernetes administrator to install, up To install and configure defaults with Helm: ``` -$ helm install --name my-release -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-connect-for-kubernetes-1.0.1.tgz +$ helm install --name my-release -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz ``` To learn more about using and modifying charts, see: @@ -105,11 +105,11 @@ Splunk Connect for Kubernetes deploys daemonsets on the Kubernetes cluster. Thes * [Fluentd metrics plugin](https://github.com/splunk/fluent-plugin-kubernetes-metrics) collects the metrics, formats the metrics for Splunk ingestion by assuring the metrics have proper metric_name, dimensions, etc., and then sends the metrics to Splunk using out_splunk_hec using Fluentd engine. -Make sure your Splunk configuration has a metrics index that is able to receive the data. See [Get started with metrics](http://docs.splunk.com/Documentation/Splunk/7.1.0/Metrics/GetStarted) in the Splunk Enterprise documentaiton. +Make sure your Splunk configuration has a metrics index that is able to receive the data. See [Get started with metrics](http://docs.splunk.com/Documentation/Splunk/7.2.4/Metrics/GetStarted) in the Splunk Enterprise documentaiton. If you want to learn more about how metrics are monitored in a Kubernetes cluster, see Tools for [Monitoring Compute, Storage, and Network Resources](https://kubernetes.io/docs/tasks/debug-application-cluster/resource-usage-monitoring/). -If you want to learn more about which metrics are collected and labels used with Splunk Connect for Kubernetes, view the metrics [schema](https://github.com/splunk/fluent-plugin-kubernetes-metrics). +If you want to learn more about which metrics tha are collected and metric names used with Splunk Connect for Kubernetes, view the metrics [schema](https://github.com/splunk/fluent-plugin-kubernetes-metrics). # Performance @@ -117,11 +117,26 @@ Some parameters used with Splunk Connect for Kubernetes can have an impact on ov Splunk Connect for Kubernetes can exceed the default throughput of HEC. To best address capacity needs, Splunk recommends that you monitor the HEC throughput and back pressure on Splunk Connect for Kubernetes deployments and be prepared to add additional nodes as needed. - # Processing Multi-Line Logs One possible filter option is to enable the processing of multi-line events. This feature is currently experimental and considered to be community supported. +# Namespace to Index Routing + +Splunk Connect for Kubernetes has the functionality to route logs and metrics from different namespaces to Splunk indexers of the same name. This can be configured by +using the two configurable parameters `indexRouting` and `indexRoutingDefaultIndex` + +`indexRouting` is a boolean configurable that enables the feature +`indexRoutingDefaultIndex` is the Splunk index used for the events from the default Kubernetes namespace + +Warning: Before enabling this feature it is essential to have Splunk indexes created which map to your Kubernetes namespaces. +Example: + +* (Namespace) -> (Splunk Index) +* kube-system -> kube-system +* kube-public -> kube-public +* default -> indexRoutingDefaultIndex + # Maintenance And Support The current maintainers of this project are the DataEdge team at Splunk. You can reach us at [DataEdge@splunk.com](mailto:DataEdge@splunk.com). diff --git a/helm-chart/splunk-connect-for-kubernetes/OWNERS b/helm-chart/splunk-connect-for-kubernetes/OWNERS index 59302fe4..16d32853 100644 --- a/helm-chart/splunk-connect-for-kubernetes/OWNERS +++ b/helm-chart/splunk-connect-for-kubernetes/OWNERS @@ -1,7 +1,11 @@ approvers: - Gimi - dbaldwin-splunk +- dtregonning +- chaitanyaphalak reviewers: - Gimi - abisnar - dbaldwin-splunk +- dtregonning +- chaitanyaphalak diff --git a/helm-chart/splunk-connect-for-kubernetes/README.md b/helm-chart/splunk-connect-for-kubernetes/README.md index 6b411a56..0a66e10a 100644 --- a/helm-chart/splunk-connect-for-kubernetes/README.md +++ b/helm-chart/splunk-connect-for-kubernetes/README.md @@ -6,7 +6,7 @@ The connector contains three Helm Charts to manage Splunk Connect for Kubernetes * splunk-kubernetes-logging: Daemonset that runs [fluentd](https://www.fluentd.org/) to collect logs for both Kubernetes system components (e.g. kubelet, apiserver, etc.), and applications which are running in the cluster. See [README](charts/splunk-kubernetes-logging/README.md) for details. * splunk-kubernetes-objects: Deployment that runs Fluentd to collect data of Kubernetes objects, e.g. namespaces, nodes, pods, etc. See [README](charts/splunk-kubernetes-objects/README.md) for details. -* splunk-kubernetes-metrics: Deployment that runs [heapster](https://github.com/kubernetes/heapster) and Fluentd together to collect metrics. See [README](charts/splunk-kubernetes-metrics/README.md) for details. +* splunk-kubernetes-metrics: Deployment and Daemonset that runs Fluentd to collect metrics from Kubernetes. [README](/helm-chart/splunk-kubernetes-metrics/README.md) ## Install ## @@ -17,7 +17,7 @@ First, prepare a Values file. Check the Values files in each sub-chart for detai Once you have a Values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-connect -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-connect-for-kubernetes-1.0.1.tgz +$ helm install --name my-splunk-connect -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz ``` ## Uninstall ## diff --git a/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt b/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt index ddcfab6f..141587d5 100644 --- a/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt +++ b/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt @@ -70,7 +70,7 @@ Once set up run: helm upgrade {{ .Release.Name }} \ --set global.splunk.hec.host=YOUR-HEC-HOST \ --set global.splunk.hec.token=YOUR-HEC-TOKEN \ - https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-connect-for-kubernetes-1.0.1.tgz + https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz {{- end }} {{- $lInd := or $lHec.indexName $gHec.indexName | default "" }} {{- $oInd := or $oHec.indexName $gHec.indexName | default "" }} @@ -94,5 +94,5 @@ Then run: helm upgrade {{ .Release.Name }} \ --set splunk-kubernetes-metrics.splunk.hec.indexName=YOUR-METRICS-INDEX \ - https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-connect-for-kubernetes-1.0.1.tgz + https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz {{- end -}} diff --git a/helm-chart/splunk-kubernetes-logging/OWNERS b/helm-chart/splunk-kubernetes-logging/OWNERS index 59302fe4..4dbe7d9d 100644 --- a/helm-chart/splunk-kubernetes-logging/OWNERS +++ b/helm-chart/splunk-kubernetes-logging/OWNERS @@ -1,7 +1,11 @@ approvers: - Gimi - dbaldwin-splunk +- dtregonning +- chaitanyaphalak reviewers: - Gimi - abisnar - dbaldwin-splunk +- dtregonning +- chaitanyaphalak \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-logging/README.md b/helm-chart/splunk-kubernetes-logging/README.md index a8fd1129..53481439 100644 --- a/helm-chart/splunk-kubernetes-logging/README.md +++ b/helm-chart/splunk-kubernetes-logging/README.md @@ -17,7 +17,7 @@ First, prepare a values file. You can also check the [examples](examples) for qu Once you have a values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-logging -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-kubernetes-logging-1.0.1.tgz +$ helm install --name my-splunk-logging -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-kubernetes-logging-1.1.0.tgz ``` ## Uninstall ## diff --git a/helm-chart/splunk-kubernetes-metrics/OWNERS b/helm-chart/splunk-kubernetes-metrics/OWNERS index 59302fe4..4dbe7d9d 100644 --- a/helm-chart/splunk-kubernetes-metrics/OWNERS +++ b/helm-chart/splunk-kubernetes-metrics/OWNERS @@ -1,7 +1,11 @@ approvers: - Gimi - dbaldwin-splunk +- dtregonning +- chaitanyaphalak reviewers: - Gimi - abisnar - dbaldwin-splunk +- dtregonning +- chaitanyaphalak \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/README.md b/helm-chart/splunk-kubernetes-metrics/README.md index b23fa7d9..a18e3873 100644 --- a/helm-chart/splunk-kubernetes-metrics/README.md +++ b/helm-chart/splunk-kubernetes-metrics/README.md @@ -1,8 +1,8 @@ # Splunk Collect for Kubernetes Metrics # -`splunk-kubernetes-metrics` is a [Helm](https://github.com/kubernetes/helm) chart that creates a kubernetes deployment along with other kubernetes objects in a kubernetes cluster to collect the cluster's metrics and send them to [splunk](https://www.splunk.com/). +`splunk-kubernetes-metrics` is a [Helm](https://github.com/kubernetes/helm) chart that creates a kubernetes deployment along with other kubernetes objects in a kubernetes cluster to collect the metrics and metric aggregations and send them to [splunk](https://www.splunk.com/). -The deployment runs a daemonset which runs a pod that has one container on each node. [Fluentd metrics plugin](https://github.com/splunk/fluent-plugin-kubernetes-metrics) collects the metrics, formats the metrics for Splunk ingestion by assuring the metrics have proper metric_name, dimensions, etc., and then sends the metrics to a splunk instance with the [Splunk HEC output plugin](https://github.com/splunk/fluent-plugin-splunk-hec) over [Splunk HEC](http://docs.splunk.com/Documentation/Splunk/7.1.0/Data/AboutHEC). +The deployment runs a daemonset and a deployment which runs a pod that has one container on each node. [Fluentd metrics plugin](https://github.com/splunk/fluent-plugin-kubernetes-metrics) collects the metrics, formats the metrics for Splunk ingestion by assuring the metrics have proper metric_name, dimensions, etc., and then sends the metrics to a splunk instance with the [Splunk HEC output plugin](https://github.com/splunk/fluent-plugin-splunk-hec) over [Splunk HEC](http://docs.splunk.com/Documentation/Splunk/7.2.4/Data/AboutHEC). Although it works well by itself, this chart is a part of [`splunk-connect-for-kubernetes`](https://github.com/splunk/splunk-connect-for-kubernetes). If you want a full Splunk monitoring solution for kubernetes, check it out. @@ -17,7 +17,8 @@ Then, prepare a values file. You can also check the [examples](examples) for qui Once you have a values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-metrics -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-kubernetes-metrics-1.0.1.tgz +$ helm install --name my-splunk-metrics -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-kubernetes-metrics-1.1.0.tgz + ``` ## Uninstall ## @@ -40,11 +41,16 @@ The following table lists all components (i.e. kubernetes objects) of this chart Component | Description | Template --- | --- | --- -`Deployment` | deploys one pod that runs fluentd plugins and fluentd to collect metrics. | [deployment.yaml](templates/deployment.yaml) +`Deployment` | deploys daemonset that runs fluentd plugins and fluentd to collect metrics. | [deployment.yaml](templates/deployment.yaml) +`DeploymentMetricsAggregator` | deploys deployment that runs fluentd plugins and fluentd to collect metrics and calculates aggregations. | [deploymentMetricsAggregator.yaml](templates/deploymentMetricsAggregator.yaml) `ConfigMap` | contains configuration files for fluentd. | [configmap.yaml](templates/configmap.yaml) +`ConfigMapMetricsAggregator` | contains configuration files for fluentd for Metrics Aggregator. | [configmap.yaml](templates/configmap.yaml) `Secret` | stores credentials like the Splunk HEC token, and SSL certs and keys for HTTPS connection, etc. | [secret.yaml](templates/secret.yaml) `ServiceAccount` | a service account to run the daemonset. | [serviceaccount.yaml](templates/serviceaccount.yaml) -`ClusterRoleBinding` | binds the system:heapster cluster role to the service account. | [clusterrolebinding.yaml](templates/clusterrolebinding.yaml) +`ClusterRoleBinding` | binds the kubelet-summary-api-read cluster role to the service account. | [clusterRoleBinding.yaml](templates/clusterrolebinding.yaml) +`ClusterRoleBindingAggregator` | binds the kube-api-aggregator cluster role to the service account. | [ClusterRoleBindingAggregator.yaml](templates/ClusterRoleBindingAggregator.yaml) +`ClusterRole` | cluster role which gives access to "nodes", "nodes/stats", "nodes/metrics". | [clusterRole.yaml](templates/clusterrole.yaml) +`ClusterRoleAggregator` | cluster role which gives access to "nodes", "nodes/stats", "nodes/proxy" and "pods". | [ClusterRoleAggregator.yaml](templates/clusterroleaggregator.yaml) Note: when `rbac.create` is set to `false` (it should be when RBAC is not enabled in the kubernetes cluster), the `ClusterRoleBinding` won't be created. diff --git a/helm-chart/splunk-kubernetes-objects/OWNERS b/helm-chart/splunk-kubernetes-objects/OWNERS index 59302fe4..4dbe7d9d 100644 --- a/helm-chart/splunk-kubernetes-objects/OWNERS +++ b/helm-chart/splunk-kubernetes-objects/OWNERS @@ -1,7 +1,11 @@ approvers: - Gimi - dbaldwin-splunk +- dtregonning +- chaitanyaphalak reviewers: - Gimi - abisnar - dbaldwin-splunk +- dtregonning +- chaitanyaphalak \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-objects/README.md b/helm-chart/splunk-kubernetes-objects/README.md index 45b3ab82..9dbb21b7 100644 --- a/helm-chart/splunk-kubernetes-objects/README.md +++ b/helm-chart/splunk-kubernetes-objects/README.md @@ -17,7 +17,7 @@ First, prepare a values file. You can also check the [examples](examples) for qu Once you have a values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-objects -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/v1.0.1/splunk-kubernetes-objects-1.0.1.tgz +$ helm install --name my-splunk-objects -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-kubernetes-objects-1.1.0.tgz ``` ## Uninstall ## From d8e97242878cc88e8aaa3559a4b2489b1102de3b Mon Sep 17 00:00:00 2001 From: Donald Tregonning Date: Thu, 21 Mar 2019 11:55:25 -0700 Subject: [PATCH 057/119] Backfill develop with release of 1.1.0 (#112) * Adding an optional clusterName value in order to have the ability to filter by a cluster_name field in splunk. (#42) * making a test release with a filter that adds the cluster name to every record * Adding an optional clusterName value in order to have the ability to filter by a cluster_name field in splunk. * Fixing typo * Fixing typo with cluster where it was cluser. * Revert "Adding an optional clusterName value in order to have the ability to filter by a cluster_name field in splunk. (#42)" This reverts commit 42fbbed7f44cf6c6ac8dddf6c6f6ecb66cd583cd. * Release of Splunk Connect for Kubernetes (#105) Splunk Connect for Kubernetes 1.1.0 Splunk Connect for Kubernetes 1.1.0 replaces Heapster as the method for collecting metrics from Kubernetes. Splunk built Fluentd plugins will now query, aggregate and send Kubernetes metrics to Splunk. [Feature] Kubernetes namespace to Splunk Index routing. Enable routing of events and metrics from multiple namespaces to multiple indexes. [Feature] Ability to tag up events and metrics with custom cluster_name metadata Improvement to the projects CI and testing framework to enable quicker project development and ability to test and accept community contributions. The update to 1.1.0 also includes a refresh of the sub-components that make up SCK. From d1e282da7e6198ed2b7c8b2f9f5c8eef5afa88a7 Mon Sep 17 00:00:00 2001 From: Ivan Etchart Date: Wed, 17 Apr 2019 13:17:04 -0300 Subject: [PATCH 058/119] Add missing configurations to metrics & metrics aggregator (#125) * Add cluster_name and interval to metrics aggregator * Add interval to metrics --- .../splunk-kubernetes-metrics/templates/configMap.yaml | 7 +++++-- .../templates/configMapMetricsAggregator.yaml | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index fe9285ef..d6bf4dde 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -36,6 +36,9 @@ data: {{- with .Values.kubernetes.clusterName }} cluster_name {{ . }} {{- end }} + {{- with .Values.metricsInterval }} + interval {{ . }} + {{- end }} @type record_modifier @@ -97,7 +100,7 @@ data: {{- end }} {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file - {{- end }} + {{- end }} {{- with .Values.buffer }} {{- range $parameter, $value := . }} @@ -105,4 +108,4 @@ data: {{- end }} {{- end }} - \ No newline at end of file + diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index 8af9fab8..6e706678 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -19,6 +19,12 @@ data: {{- with .Values.kubernetes.kubeletPort }} kubelet_port {{ . }} {{- end }} + {{- with .Values.kubernetes.clusterName }} + cluster_name {{ . }} + {{- end }} + {{- with .Values.metricsInterval }} + interval {{ . }} + {{- end }} @type record_modifier @@ -100,4 +106,4 @@ data: {{- end }} {{- end }} - \ No newline at end of file + From 7946da3b6a78afd5251e2d33bed3ded1d092b337 Mon Sep 17 00:00:00 2001 From: Mattia Date: Fri, 26 Apr 2019 16:45:54 +0200 Subject: [PATCH 059/119] Missing rbac create condition (#141) * Missing rbac condition --- helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml | 2 ++ .../templates/clusterRoleAggregator.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml index 32ed5d0b..2f5eef07 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRole.yaml @@ -1,4 +1,5 @@ # This role allows read access to the kubelet summary API +{{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -17,3 +18,4 @@ rules: - "nodes/metrics" verbs: - "get" +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml index 916dbec1..29bbf384 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml @@ -1,4 +1,5 @@ # This role allows read access to the kubelet summary API +{{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -19,3 +20,4 @@ rules: verbs: - get - list +{{- end -}} \ No newline at end of file From bf90c1c20fc7a0e61cb7bc09d8e1568e6c466c99 Mon Sep 17 00:00:00 2001 From: gp510 Date: Fri, 26 Apr 2019 16:08:24 -0700 Subject: [PATCH 060/119] Adding issue templates (#139) --- .github/ISSUE_TEMPLATE/bug_report.md | 29 +++++++++++++++++++ .github/ISSUE_TEMPLATE/enhancement_request.md | 14 +++++++++ 2 files changed, 43 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/enhancement_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..2b8461e6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,29 @@ +--- +name: Bug report +about: Report a bug encountered while operating Splunk Connect for Kubernetes +title: '' +labels: '' +assignees: '' + +--- + + + + +**What happened**: + +**What you expected to happen**: + +**How to reproduce it (as minimally and precisely as possible)**: + +**Anything else we need to know?**: + +**Environment**: +- Kubernetes version (use `kubectl version`): +- Ruby version (use `ruby --version`): +- OS (e.g: `cat /etc/os-release`): +- Splunk version: +- Others: diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md new file mode 100644 index 00000000..b28164dd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement_request.md @@ -0,0 +1,14 @@ +--- +name: Enhancement Request +about: Suggest an enhancement to the Splunk Connect for Kubernetes project +title: '' +labels: '' +assignees: '' + +--- + +< + +**What would you like to be added**: + +**Why is this needed**: From ebe2fec9605aa6d64e36c4d4a1482b83a2e211f8 Mon Sep 17 00:00:00 2001 From: Paul Braham Date: Wed, 1 May 2019 16:38:03 +0100 Subject: [PATCH 061/119] added aggregatorTolerations variable to allow metrics aggregation tolerations to be set independently to the metrics pods (#135) --- .../templates/deploymentMetricsAggregator.yaml | 2 +- helm-chart/splunk-kubernetes-metrics/values.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index 005227c1..641e0951 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -31,7 +31,7 @@ spec: affinity: {{ toYaml . | indent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with .Values.aggregatorTolerations }} tolerations: {{ toYaml . | indent 8 }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 1be791e0..4103a0fb 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -136,6 +136,9 @@ tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule +# Tolerations for the aggregator pod. We do not really want this running on the master nodes, so we leave this +# blank by default. +aggregatorTolerations: {} affinity: {} From 02c0361e6af9b6a4c0355ee8bdad88af1d3e772a Mon Sep 17 00:00:00 2001 From: gp510 Date: Fri, 3 May 2019 09:38:58 -0700 Subject: [PATCH 062/119] Removing invalid logging tests (#142) --- test/k8s_logging_tests/test_config_logging.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/k8s_logging_tests/test_config_logging.py b/test/k8s_logging_tests/test_config_logging.py index d403e1b5..2fe8862d 100644 --- a/test/k8s_logging_tests/test_config_logging.py +++ b/test/k8s_logging_tests/test_config_logging.py @@ -141,7 +141,6 @@ def test_namespace_routing(setup, test_input, expected): @pytest.mark.parametrize("test_input,expected", [ ("kube:kube-apiserver", 1), - ("fluentd:monitor-agent", 1), ("kube:container:splunk-fluentd-k8s-metrics-agg", 1), ("kube:container:splunk-fluentd-k8s-metrics", 1), ("kube:container:splunk-fluentd-k8s-logs", 1), @@ -169,7 +168,6 @@ def test_sourcetype(setup, test_input, expected): @pytest.mark.parametrize("test_input,expected", [ ("/var/log/containers/kube-apiserver-ip*", 1), ("/var/log/containers/ci*", 1), - ("namespace:default/pod:ci*", 1), ("/var/log/containers/dns-controller*", 1), ("/var/log/containers/etcd-server-ip*", 1) ]) From 91c2bd510281b41e4a91fa3d619f4ec1689866bf Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 8 May 2019 16:02:01 -0700 Subject: [PATCH 063/119] Add appVersion to Charts (#145) Pull request from community contributor - #123 --- helm-chart/splunk-connect-for-kubernetes/Chart.yaml | 1 + helm-chart/splunk-kubernetes-logging/Chart.yaml | 1 + helm-chart/splunk-kubernetes-metrics/Chart.yaml | 1 + helm-chart/splunk-kubernetes-objects/Chart.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml index fcb98667..4b2d8e53 100644 --- a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml @@ -1,6 +1,7 @@ apiVersion: v1 name: splunk-connect-for-kubernetes version: 1.1.0 +appVersion: 1.1.0 description: Collect logs, objects (metadata), and metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart sources: diff --git a/helm-chart/splunk-kubernetes-logging/Chart.yaml b/helm-chart/splunk-kubernetes-logging/Chart.yaml index 8c3db932..98b81447 100644 --- a/helm-chart/splunk-kubernetes-logging/Chart.yaml +++ b/helm-chart/splunk-kubernetes-logging/Chart.yaml @@ -1,5 +1,6 @@ apiVersion: v1 version: 1.1.0 +appVersion: 1.1.0 name: splunk-kubernetes-logging description: Collect logs with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-logging diff --git a/helm-chart/splunk-kubernetes-metrics/Chart.yaml b/helm-chart/splunk-kubernetes-metrics/Chart.yaml index 3fe17bf9..33e4c86b 100644 --- a/helm-chart/splunk-kubernetes-metrics/Chart.yaml +++ b/helm-chart/splunk-kubernetes-metrics/Chart.yaml @@ -1,5 +1,6 @@ apiVersion: v1 version: 1.1.0 +appVersion: 1.1.0 name: splunk-kubernetes-metrics description: Collect metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-metrics diff --git a/helm-chart/splunk-kubernetes-objects/Chart.yaml b/helm-chart/splunk-kubernetes-objects/Chart.yaml index b6b4bb11..dcb5bb8e 100644 --- a/helm-chart/splunk-kubernetes-objects/Chart.yaml +++ b/helm-chart/splunk-kubernetes-objects/Chart.yaml @@ -1,5 +1,6 @@ apiVersion: v1 version: 1.1.0 +appVersion: 1.1.0 name: splunk-kubernetes-objects description: Collect Kubernetes objects with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-objects From 77dd8967a19162b96c63267666116cb25f2249f5 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 8 May 2019 16:11:15 -0700 Subject: [PATCH 064/119] Create CODE_OF_CONDUCT.md (#146) merging #103 --- CODE_OF_CONDUCT.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..9417009a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at splunk-oss-admin@splunk.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From 3d6a0105ce2a9c8b925218d8e5087b4e2e7df796 Mon Sep 17 00:00:00 2001 From: Dipesh Mitthalal Date: Tue, 4 Jun 2019 19:14:52 +0200 Subject: [PATCH 065/119] Allow setting cluster name gloablly (#118) --- .../splunk-connect-for-kubernetes/examples/insecure.yaml | 3 ++- helm-chart/splunk-connect-for-kubernetes/values.yaml | 3 ++- helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl | 4 ++-- .../splunk-kubernetes-logging/templates/configMap.yaml | 2 +- helm-chart/splunk-kubernetes-logging/values.yaml | 5 +++-- .../splunk-kubernetes-metrics/templates/configMap.yaml | 6 +++--- .../templates/configMapMetricsAggregator.yaml | 2 +- .../splunk-kubernetes-metrics/templates/deployment.yaml | 4 ++++ helm-chart/splunk-kubernetes-metrics/values.yaml | 3 ++- .../splunk-kubernetes-objects/templates/configMap.yaml | 4 ++-- helm-chart/splunk-kubernetes-objects/values.yaml | 3 ++- 11 files changed, 24 insertions(+), 15 deletions(-) diff --git a/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml b/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml index 066aaa83..54b6a789 100644 --- a/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml @@ -8,7 +8,8 @@ global: indexName: main # connection to splunk is insecure insecureSSL: true - + kubernetes: + clusterName: "cluster_name" splunk-kubernetes-objects: # RBAC is disabled diff --git a/helm-chart/splunk-connect-for-kubernetes/values.yaml b/helm-chart/splunk-connect-for-kubernetes/values.yaml index c2aeb238..cfec826a 100644 --- a/helm-chart/splunk-connect-for-kubernetes/values.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/values.yaml @@ -9,7 +9,8 @@ # splunk: # # HEC related configurations # hec: -# +# kubernetes: +# clusterName: "cluster_name" # For other configurations for sub-charts, please check their values.yaml for details. ## Enabling logging will install the `splunk-kubernetes-logging` chart to a kubernetes diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index cd8d8fc4..e82a532a 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -100,8 +100,8 @@ def extract_container_info: | .namespace = set_namespace($parts[1]) | .container_name = ($cparts[:-1] | join("-")) | .container_id = ($cparts[-1] | rtrimstr(".log")) - {{- if .Values.kubernetes.clusterName }} - | .cluster_name = "{{ .Values.kubernetes.clusterName }}" + {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} + | .cluster_name = "{{ . }}" {{- end }} | .; diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index c32f6a93..b07a55ba 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -232,7 +232,7 @@ data: namespace container_name container_id - {{- if .Values.kubernetes.clusterName }} + {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{- end }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 4e7a98bf..58e2094d 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -18,7 +18,8 @@ global: indexRouting: # indexRoutingDefaultIndex tells which index to use for the default kubenetes namespace. Used with indexRouting. Default is main. indexRoutingDefaultIndex: - + kubernetes: + clusterName: "cluster_name" # logLevel is to set log level of the Splunk log collector. Avaiable values are: # * trace @@ -271,4 +272,4 @@ affinity: {} # = Kubernetes Connection Configs = kubernetes: # The cluster name used to tag logs. Default is cluster_name - clusterName: "cluster_name" + clusterName: "cluster_name" \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index d6bf4dde..f718a059 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -16,7 +16,7 @@ data: @type kubernetes_metrics tag kube.* - node_name "#{ENV['SPLUNK_HEC_HOST']}" + node_name "#{ENV['MY_NODE_IP']}" {{- with .Values.kubernetes.kubeletPort }} kubelet_port {{ . }} {{- end }} @@ -33,7 +33,7 @@ data: {{- with .Values.kubernetes.secretDir }} secret_dir {{ . }} {{- end }} - {{- with .Values.kubernetes.clusterName }} + {{- with or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{ . }} {{- end }} {{- with .Values.metricsInterval }} @@ -44,7 +44,7 @@ data: @type record_modifier metric_name ${tag} - {{- with .Values.kubernetes.clusterName }} + {{- with or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{ . }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index 6e706678..675d06dd 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -30,7 +30,7 @@ data: @type record_modifier metric_name ${tag} - {{- with .Values.kubernetes.clusterName }} + {{- with or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{ . }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index 104ac1d1..47f652e7 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -48,6 +48,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: MY_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: SPLUNK_HEC_TOKEN valueFrom: secretKeyRef: diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 4103a0fb..b971f06d 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -14,7 +14,8 @@ global: port: 8088 protocol: https insecureSSL: false - + kubernetes: + clusterName: "cluster_name" # = Log Level = # logLevel is to set log level of the Splunk kubernetes metrics collector. Avaiable values are: diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index bd036f5e..60b5c7ca 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -84,7 +84,7 @@ data: @type jq_transformer - jq '.record.cluster_name = "{{ .Values.kubernetes.clusterName }}" | .record' + jq '.record.cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }}" | .record' @@ -114,7 +114,7 @@ data: ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} - {{- if .Values.kubernetes.clusterName }} + {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{- end }} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index a39d4b48..30eb12ab 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -15,7 +15,8 @@ global: protocol: https port: 8088 insecureSSL: false - + kubernetes: + clusterName: "cluster_name" # = Log Level = # logLevel is to set log level of the object collector. Avaiable values are: From 600c31428cc2e30deb8ad50bb41c28713a9f4156 Mon Sep 17 00:00:00 2001 From: Peter Moffatt <44172972+peter-moffatt@users.noreply.github.com> Date: Tue, 4 Jun 2019 19:14:00 +0100 Subject: [PATCH 066/119] Fix for checkpoint issues in the splunk-kubernetes-objects Helm chart (#99) --- helm-chart/splunk-kubernetes-objects/templates/configMap.yaml | 2 +- helm-chart/splunk-kubernetes-objects/templates/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index 60b5c7ca..0c6ab2e0 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -44,7 +44,7 @@ data: {{- end }} {{- if $.Values.checkpointFile.volume }} - path /fluentd/var/checkpoints/{{ .Values.checkpointFile.name | default "kubernetes-objects" }} + path /fluentd/var/checkpoints/{{ $.Values.checkpointFile.name | default "kubernetes-objects" }} {{- end }} {{- range $objects }} diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index 95a5a89e..e95b7a77 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -68,7 +68,7 @@ spec: - name: conf-configmap mountPath: /fluentd/etc {{- if .Values.checkpointFile.volume }} - - name: conf-configmap + - name: checkpoints mountPath: /fluentd/var/checkpoints {{- end }} - name: secrets From 60d05ab6bb483a8328f044d1cad7692cbeddcd4d Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 5 Jun 2019 08:11:23 -0700 Subject: [PATCH 067/119] Merge community PR for variable docker logs path (#161) --- helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl | 4 +--- helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml | 4 ++-- helm-chart/splunk-kubernetes-logging/values.yaml | 4 ++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index e82a532a..bafe1eef 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -100,9 +100,7 @@ def extract_container_info: | .namespace = set_namespace($parts[1]) | .container_name = ($cparts[:-1] | join("-")) | .container_id = ($cparts[-1] | rtrimstr(".log")) - {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} - | .cluster_name = "{{ . }}" - {{- end }} + | .cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" | .; .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // "kube:container:\(.container_name)") diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 981ff894..1c99bf25 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -60,7 +60,7 @@ spec: - name: varlog mountPath: /var/log - name: varlibdockercontainers - mountPath: /var/lib/docker/containers + mountPath: {{ with .Values.dockerPath }}{{ printf "%s/containers" . | quote }}{{ end }} readOnly: true - name: journallogpath mountPath: {{ .Values.journalLogPath | quote }} @@ -77,7 +77,7 @@ spec: path: /var/log - name: varlibdockercontainers hostPath: - path: /var/lib/docker/containers + path: {{ with .Values.dockerPath }}{{ printf "%s/containers" . | quote }}{{ end }} - name: journallogpath hostPath: path: {{ .Values.journalLogPath | quote }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 58e2094d..64e3f28e 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -68,6 +68,10 @@ splunk: journalLogPath: /run/log/journal +# Directory where the docker daemon's data is stored +dockerPath: /var/lib/docker + + # `logs` defines the source of logs, multiline support, and their sourcetypes. # # The scheme to define a log is: From 7292e173e4fdc496e92ba4ebc3a187abb62e0a37 Mon Sep 17 00:00:00 2001 From: Peter Wu <4483014+hexecute@users.noreply.github.com> Date: Wed, 5 Jun 2019 08:16:55 -0700 Subject: [PATCH 068/119] ADDON-21681: Splunk Connect for Kubernetes tries to connect to the Kubernetes node name (#137) --- .../splunk-kubernetes-logging/templates/daemonset.yaml | 2 +- .../splunk-kubernetes-metrics/templates/configMap.yaml | 5 ++++- .../splunk-kubernetes-metrics/templates/deployment.yaml | 6 +++++- helm-chart/splunk-kubernetes-metrics/values.yaml | 5 ++++- .../splunk-kubernetes-objects/templates/deployment.yaml | 2 +- manifests/splunk-kubernetes-logging/daemonset.yaml | 2 +- manifests/splunk-kubernetes-metrics/configMap.yaml | 2 +- manifests/splunk-kubernetes-metrics/deployment.yaml | 6 +++++- manifests/splunk-kubernetes-objects/deployment.yaml | 2 +- 9 files changed, 23 insertions(+), 9 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 1c99bf25..91cf716f 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -37,7 +37,7 @@ spec: - -c - /fluentd/etc/fluent.conf env: - - name: SPLUNK_HEC_HOST + - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index f718a059..cfcae981 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -16,7 +16,10 @@ data: @type kubernetes_metrics tag kube.* - node_name "#{ENV['MY_NODE_IP']}" + node_name "#{ENV['NODE_NAME']}" + {{- with .Values.kubernetes.kubeletAddress }} + kubelet_address {{ . }} + {{- end }} {{- with .Values.kubernetes.kubeletPort }} kubelet_port {{ . }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index 47f652e7..d2b0cf67 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -44,7 +44,11 @@ spec: image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: SPLUNK_HEC_HOST + - name: KUBERNETES_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index b971f06d..73f428af 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -145,7 +145,10 @@ affinity: {} # = Kubernetes Connection Configs = kubernetes: - # The port that kubelet is listening to. Default is 10250 + # The hostname or IP address that kubelet will use to connect to. If not supplied, status.hostIP of the node is used to fetch metrics from the Kubelet API (via the $KUBERNETES_NODE_IP environment variable). + # Default is "#{ENV['KUBERNETES_NODE_IP']}" + kubeletAddress: + # The port that kubelet is listening on. Default is 10250 kubeletPort: # This option is used to get the metrics from summary api on each kubelet using ssl useRestClientSSL: true diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index e95b7a77..5c75bed7 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -53,7 +53,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - - name: SPLUNK_HEC_HOST + - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index 02c3a662..8373465f 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -26,7 +26,7 @@ spec: - "-c" - "/fluentd/etc/fluent.conf" env: - - name: SPLUNK_HEC_HOST + - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName diff --git a/manifests/splunk-kubernetes-metrics/configMap.yaml b/manifests/splunk-kubernetes-metrics/configMap.yaml index c190c455..09847f3a 100644 --- a/manifests/splunk-kubernetes-metrics/configMap.yaml +++ b/manifests/splunk-kubernetes-metrics/configMap.yaml @@ -15,7 +15,7 @@ data: @type kubernetes_metrics tag kube.* - node_name "#{ENV['SPLUNK_HEC_HOST']}" + node_name "#{ENV['NODE_NAME']}" use_rest_client_ssl true cluster_name cluster_name diff --git a/manifests/splunk-kubernetes-metrics/deployment.yaml b/manifests/splunk-kubernetes-metrics/deployment.yaml index 6e00c180..6259c99f 100644 --- a/manifests/splunk-kubernetes-metrics/deployment.yaml +++ b/manifests/splunk-kubernetes-metrics/deployment.yaml @@ -32,7 +32,11 @@ spec: image: splunk/k8s-metrics:1.1.0 imagePullPolicy: Always env: - - name: SPLUNK_HEC_HOST + - name: KUBERNETES_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName diff --git a/manifests/splunk-kubernetes-objects/deployment.yaml b/manifests/splunk-kubernetes-objects/deployment.yaml index 11b47c57..4cc9a4d9 100644 --- a/manifests/splunk-kubernetes-objects/deployment.yaml +++ b/manifests/splunk-kubernetes-objects/deployment.yaml @@ -39,7 +39,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - - name: SPLUNK_HEC_HOST + - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName From 72a2d8dd943b6ed796fa873432f9ec4fa527bf5f Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Wed, 5 Jun 2019 13:59:44 -0600 Subject: [PATCH 069/119] Make systemd components persist a position file (#96) --- Makefile | 3 +++ helm-chart/splunk-kubernetes-logging/templates/configMap.yaml | 1 + manifests/splunk-kubernetes-logging/configMap.yaml | 2 ++ manifests/splunk-kubernetes-metrics/configMap.yaml | 1 + .../splunk-kubernetes-metrics/configMapMetricsAggregator.yaml | 2 ++ .../splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml | 3 --- 6 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 445aaec2..15d99995 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,9 @@ manifests: connect-chart --set global.splunk.hec.host=MY-SPLUNK-HOST \ --set global.splunk.hec.token=MY-SPLUNK-TOKEN \ --set global.splunk.hec.insecureSSL=true \ + --set splunk-kubernetes-logging.fullnameOverride="splunk-kubernetes-logging" \ + --set splunk-kubernetes-metrics.fullnameOverride="splunk-kubernetes-metrics" \ + --set splunk-kubernetes-objects.fullnameOverride="splunk-kubernetes-objects" \ --set splunk-kubernetes-objects.kubernetes.insecureSSL=true \ $$(ls build/splunk-connect-for-kubernetes-*.tgz) \ | ruby tools/gen_manifest.rb manifests diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index b07a55ba..33e1bb41 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -117,6 +117,7 @@ data: @type local persistent true + path /var/log/splunkd-fluentd-journald-{{ $name }}.pos.json field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} diff --git a/manifests/splunk-kubernetes-logging/configMap.yaml b/manifests/splunk-kubernetes-logging/configMap.yaml index b86e1c01..99624905 100644 --- a/manifests/splunk-kubernetes-logging/configMap.yaml +++ b/manifests/splunk-kubernetes-logging/configMap.yaml @@ -68,6 +68,7 @@ data: @type local persistent true + path /var/log/splunkd-fluentd-journald-docker.pos.json field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} @@ -85,6 +86,7 @@ data: @type local persistent true + path /var/log/splunkd-fluentd-journald-kubelet.pos.json field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"} diff --git a/manifests/splunk-kubernetes-metrics/configMap.yaml b/manifests/splunk-kubernetes-metrics/configMap.yaml index 09847f3a..6c80e9ae 100644 --- a/manifests/splunk-kubernetes-metrics/configMap.yaml +++ b/manifests/splunk-kubernetes-metrics/configMap.yaml @@ -18,6 +18,7 @@ data: node_name "#{ENV['NODE_NAME']}" use_rest_client_ssl true cluster_name cluster_name + interval 15s @type record_modifier diff --git a/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml index df2d00d9..9b685ef0 100644 --- a/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml @@ -15,6 +15,8 @@ data: @type kubernetes_metrics_aggregator tag kube.* + cluster_name cluster_name + interval 15s @type record_modifier diff --git a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml index efd98b9d..2d2e034f 100644 --- a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml @@ -21,9 +21,6 @@ spec: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: splunk-kubernetes-metrics - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master containers: - name: splunk-fluentd-k8s-metrics-agg image: splunk/k8s-metrics-aggr:1.1.0 From c4508477f8bb4821a5c7883cb32974e30d2d001a Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 5 Jun 2019 13:26:26 -0700 Subject: [PATCH 070/119] Patch to support SII (#162) --- helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml | 2 +- .../splunk-kubernetes-metrics/templates/deployment.yaml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index cfcae981..8e61bbb2 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -89,7 +89,7 @@ data: hec_port {{ . }} {{- end }} hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" - host "#{ENV['SPLUNK_HEC_HOST']}" + host "#{ENV['NODE_NAME']}" {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} index {{ . }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index d2b0cf67..3ac2fd40 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -52,10 +52,6 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - - name: MY_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - name: SPLUNK_HEC_TOKEN valueFrom: secretKeyRef: From b1e5256baf68a678abbca90f7332ef7f4c99c5f8 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 6 Jun 2019 09:55:47 -0700 Subject: [PATCH 071/119] Add an option to use existing secret for HEC token (#164) --- .../splunk-kubernetes-logging/templates/_helpers.tpl | 11 +++++++++++ .../templates/daemonset.yaml | 4 ++-- .../splunk-kubernetes-logging/templates/secret.yaml | 4 +++- helm-chart/splunk-kubernetes-logging/values.yaml | 6 ++++++ .../splunk-kubernetes-metrics/templates/_helpers.tpl | 11 +++++++++++ .../templates/deployment.yaml | 4 ++-- .../templates/deploymentMetricsAggregator.yaml | 4 ++-- .../splunk-kubernetes-metrics/templates/secret.yaml | 4 +++- helm-chart/splunk-kubernetes-metrics/values.yaml | 6 ++++++ .../splunk-kubernetes-objects/templates/_helpers.tpl | 11 +++++++++++ .../templates/deployment.yaml | 4 ++-- .../splunk-kubernetes-objects/templates/secret.yaml | 4 +++- helm-chart/splunk-kubernetes-objects/values.yaml | 6 ++++++ 13 files changed, 68 insertions(+), 11 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index bafe1eef..f093c5a3 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -31,6 +31,17 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "splunk-kubernetes-logging.secret" -}} +{{- if .Values.secret.name -}} +{{- printf "%s" .Values.secret.name -}} +{{- else -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + {{/* Convert memory value from resources.limit to fluentd buffer. Rules: diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 91cf716f..50e47c36 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -52,7 +52,7 @@ spec: - name: SPLUNK_HEC_TOKEN valueFrom: secretKeyRef: - name: {{ template "splunk-kubernetes-logging.fullname" . }} + name: {{ template "splunk-kubernetes-logging.secret" . }} key: splunk_hec_token resources: {{ toYaml .Values.resources | indent 10 }} @@ -86,4 +86,4 @@ spec: name: {{ template "splunk-kubernetes-logging.fullname" . }} - name: secrets secret: - secretName: {{ template "splunk-kubernetes-logging.fullname" . }} + secretName: {{ template "splunk-kubernetes-logging.secret" . }} diff --git a/helm-chart/splunk-kubernetes-logging/templates/secret.yaml b/helm-chart/splunk-kubernetes-logging/templates/secret.yaml index b4a482cf..a45ee919 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/secret.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/secret.yaml @@ -1,7 +1,8 @@ +{{- if .Values.secret.create -}} apiVersion: v1 kind: Secret metadata: - name: {{ template "splunk-kubernetes-logging.fullname" . }} + name: {{ template "splunk-kubernetes-logging.secret" . }} labels: app: {{ template "splunk-kubernetes-logging.name" . }} chart: {{ template "splunk-kubernetes-logging.chart" . }} @@ -21,3 +22,4 @@ data: {{- with or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} hec_ca_file: {{ . | b64enc }} {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 64e3f28e..085d2521 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -64,6 +64,12 @@ splunk: indexRoutingDefaultIndex: +# Create or use existing secret if name is empty default name is used +secret: + create: true + name: + + # Directory where to read journald logs. journalLogPath: /run/log/journal diff --git a/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl index 461fe48e..fe7c5a6c 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl @@ -24,6 +24,17 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{- end -}} +{{/* +Create secret to be used. +*/}} +{{- define "splunk-kubernetes-metrics.secret" -}} +{{- if .Values.secret.name -}} +{{- printf "%s" .Values.secret.name -}} +{{- else -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + {{/* Create chart name and version as used by the chart label. */}} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index 3ac2fd40..c8dfba17 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -55,7 +55,7 @@ spec: - name: SPLUNK_HEC_TOKEN valueFrom: secretKeyRef: - name: {{ template "splunk-kubernetes-metrics.fullname" . }} + name: {{ template "splunk-kubernetes-metrics.secret" . }} key: splunk_hec_token resources: {{ toYaml .Values.resources.fluent | indent 12 }} @@ -71,4 +71,4 @@ spec: name: {{ template "splunk-kubernetes-metrics.fullname" . }} - name: secrets secret: - secretName: {{ template "splunk-kubernetes-metrics.fullname" . }} + secretName: {{ template "splunk-kubernetes-metrics.secret" . }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index 641e0951..508bf9df 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -43,7 +43,7 @@ spec: - name: SPLUNK_HEC_TOKEN valueFrom: secretKeyRef: - name: {{ template "splunk-kubernetes-metrics.fullname" . }} + name: {{ template "splunk-kubernetes-metrics.secret" . }} key: splunk_hec_token resources: {{ toYaml .Values.resources.fluent | indent 12 }} @@ -59,4 +59,4 @@ spec: name: {{ template "splunk-kubernetes-metrics.fullname" . }}-aggregator - name: secrets secret: - secretName: {{ template "splunk-kubernetes-metrics.fullname" . }} + secretName: {{ template "splunk-kubernetes-metrics.secret" . }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/secret.yaml b/helm-chart/splunk-kubernetes-metrics/templates/secret.yaml index bb9c0357..c2112e85 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/secret.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/secret.yaml @@ -1,7 +1,8 @@ +{{- if .Values.secret.create -}} apiVersion: v1 kind: Secret metadata: - name: {{ template "splunk-kubernetes-metrics.fullname" . }} + name: {{ template "splunk-kubernetes-metrics.secret" . }} labels: app: {{ template "splunk-kubernetes-metrics.name" . }} chart: {{ template "splunk-kubernetes-metrics.chart" . }} @@ -21,3 +22,4 @@ data: {{- with or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} hec_ca_file: {{ . | b64enc }} {{- end }} +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 73f428af..ad810b76 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -74,6 +74,12 @@ splunk: caFile: +# Create or use existing secret if name is empty default name is used +secret: + create: true + name: + + image: name: splunk/k8s-metrics:1.1.0 pullPolicy: Always diff --git a/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl index a81bfe4c..63b98795 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl @@ -42,6 +42,17 @@ Create the name of the service account to use {{- end -}} {{- end -}} +{{/* +Create secret to be used. +*/}} +{{- define "splunk-kubernetes-objects.secret" -}} +{{- if .Values.secret.name -}} +{{- printf "%s" .Values.secret.name -}} +{{- else -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + {{/* Convert memory value from resources.limit to fluentd buffer. Rules: diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index 5c75bed7..b1c7c7f7 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -60,7 +60,7 @@ spec: - name: SPLUNK_HEC_TOKEN valueFrom: secretKeyRef: - name: {{ template "splunk-kubernetes-objects.fullname" . }} + name: {{ template "splunk-kubernetes-objects.secret" . }} key: splunk_hec_token resources: {{ toYaml .Values.resources | indent 10 }} @@ -80,7 +80,7 @@ spec: name: {{ template "splunk-kubernetes-objects.fullname" . }} - name: secrets secret: - secretName: {{ template "splunk-kubernetes-objects.fullname" . }} + secretName: {{ template "splunk-kubernetes-objects.secret" . }} {{- if .Values.checkpointFile.volume }} - name: checkpoints {{ toYaml .Values.checkpointFile.volume | indent 8 }} diff --git a/helm-chart/splunk-kubernetes-objects/templates/secret.yaml b/helm-chart/splunk-kubernetes-objects/templates/secret.yaml index 2e43e186..1573c6b0 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/secret.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/secret.yaml @@ -1,7 +1,8 @@ +{{- if .Values.secret.create -}} apiVersion: v1 kind: Secret metadata: - name: {{ template "splunk-kubernetes-objects.fullname" . }} + name: {{ template "splunk-kubernetes-objects.secret" . }} labels: app: {{ template "splunk-kubernetes-objects.name" . }} chart: {{ template "splunk-kubernetes-objects.chart" . }} @@ -30,3 +31,4 @@ data: {{- with .Values.kubernetes.caFile }} k8s_ca_file: {{ . | b64enc }} {{- end }} +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 30eb12ab..fd1c4e83 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -165,6 +165,12 @@ splunk: caPath: +# Create or use existing secret if name is empty default name is used +secret: + create: true + name: + + image: name: splunk/kube-objects:1.1.0 pullPolicy: IfNotPresent From 06c7b286dad2343a7d8773a7f7deb5300b48fc2f Mon Sep 17 00:00:00 2001 From: gp510 Date: Fri, 7 Jun 2019 08:12:16 -0700 Subject: [PATCH 072/119] Updating contrubution guidelines (#165) --- CONTRIBUTING.md | 67 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46e6e4e9..c10a1c7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,20 +1,59 @@ -CONTRIBUTING +## How to contribute to Splunk Connect for Kubernetes -By submitting a Contribution to this Work, You agree that Your Contribution is made subject to the primary LICENSE -file applicable to this Work. In addition, You represent that: (i) You are the copyright owner of the Contribution -or (ii) You have the requisite rights to make the Contribution. +### **Did you find a bug?** + +* **Do not open up a GitHub issue if the bug is a security vulnerability. Please [report security issues here]( https://www.splunk.com/goto/report_vulnerabilities_prodsec)**. +* Ensure the bug was not already reported by searching on GitHub under Issues. +* If you're unable to find an open issue addressing the problem, open a new one. Be sure to include a title and clear description, as much relevant information as possible, and a code sample or an executable test case demonstrating the expected behavior that is not occurring. + +### **Did you write a patch that fixes a bug?** +* Open a new GitHub pull request against the develop branch with the patch. +* Ensure the pull request description clearly describes the problem and solution. Please create an issue and reference it or the relevant existing issue number as part of the title for the PR. + +### **Do you have an idea for a new feature or change an existing one?** +* If you developed a new feature, open a new GitHub pull request against the develop branch with the new feature. +* Ensure the PR description clearly describes the new feature and the benefits. Include the relevant issue number if applicable. +* If you have an idea for a new feature, open a GitHub issue that clearly explains the suggested feature and it’s benefits. + +### **Reporting an issue** +We use GitHub Issue Tracking to track issues. If you've found a problem which is not a security risk, check to see if it has already been reported. If not, open a new issue. (See the next section for reporting security issues.) + +Your issue report should contain a title and a clear description of the issue at the bare minimum. You should include as much relevant information as possible and should at least post a code sample that demonstrates the issue. Please include relevant unit test cases that show how the expected behavior is not occurring. Your goal should be to make it easy for yourself - and others - to reproduce the bug and figure out a fix. + +If possible, create an executable test case. This can be very helpful for others to help confirm, investigate, and ultimately fix your issue. +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +### **When filing an issue, please include answers for questions below:** +* What version of Ruby and Kubernetes are you using? +* What operating system and processor architecture are you using? +* What did you do? +* What did you expect to see? +* What did you see instead? + +#### **When filing an issue, please do NOT include:** +* Internal identifiers such as JIRA tickets +* Any sensitive information related to your environment, users, etc. + +### **Reporting a security issue** +**Please do not report security vulnerabilities with public GitHub issue reports. Please [report security issues here]( https://www.splunk.com/goto/report_vulnerabilities_prodsec)**. + +### **Verifying existing issues** +We appreciate any help you can provide us with existing issues. Are you able to reproduce a current issue? If so please provide any additional information that might help us to reproduce or fix the issue. If you can contribute a failing test that’s even better. + +### **Contributing code** +If you're planning to submit your change back for inclusion in the main repo, keep a few things in mind: +* Please review our [Code of Conduct](https://github.com/splunk/splunk-connect-for-kubernetes/blob/develop/CODE_OF_CONDUCT.md). +* Fork the repo and create your working branch from the develop branch. +* Include unit tests that fail without your code, and pass with it. +* Update the (surrounding) documentation, examples, and whatever is affected by your contribution. + +By submitting a Contribution to this Work, You agree that Your Contribution is made subject to the primary license in the LICENSE file applicable to this Work. In addition, You represent that: (i) You are the copyright owner of the Contribution or (ii) You have the requisite rights to make the Contribution. Definitions: -“You” shall mean: (i) yourself if you are making a Contribution on your own behalf; or (ii) your company, -if you are making a Contribution on behalf of your company. If you are making a Contribution on behalf of your -company, you represent that you have the requisite authority to do so. - -"Contribution" shall mean any original work of authorship, including any modifications or additions to an existing -work, that is intentionally submitted by You for inclusion in, or documentation of, this project/repository. For the -purposes of this definition, "submitted" means any form of electronic, verbal, or written communication submitted for -inclusion in this project/repository, including but not limited to communication on electronic mailing lists, source -code control systems, and issue tracking systems that are managed by, or on behalf of, the maintainers of -the project/repository. +“You” shall mean: (i) yourself if you are making a Contribution on your own behalf; or (ii) your company, if you are making a Contribution on behalf of your company. If you are making a Contribution on behalf of your company, you represent that you have the requisite authority to do so. + +"Contribution" shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You for inclusion in, or documentation of, this project/repository. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication submitted for inclusion in this project/repository, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the maintainers of the project/repository. “Work” shall mean the collective software, content, and documentation in this project/repository. + From 06e877daa21cff4420ededd069e20774e5b584d8 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 11 Jun 2019 15:53:04 -0700 Subject: [PATCH 073/119] Dev patch 1.2.0 (#167) --- .circleci/build_sck_artifacts_for_release.sh | 43 +++++++++++-------- .circleci/update_sck_config_for_release.sh | 35 +++------------ VERSION | 2 +- .../splunk-kubernetes-logging/configMap.yaml | 2 +- .../splunk-kubernetes-logging/daemonset.yaml | 4 +- .../splunk-kubernetes-logging/secret.yaml | 2 +- .../clusterRole.yaml | 2 +- .../clusterRoleAggregator.yaml | 2 +- .../clusterRoleBinding.yaml | 2 +- .../clusterRoleBindingAggregator.yaml | 2 +- .../splunk-kubernetes-metrics/configMap.yaml | 4 +- .../configMapMetricsAggregator.yaml | 2 +- .../splunk-kubernetes-metrics/deployment.yaml | 6 +-- .../deploymentMetricsAggregator.yaml | 6 +-- .../splunk-kubernetes-metrics/secret.yaml | 2 +- .../serviceAccount.yaml | 2 +- .../clusterRole.yaml | 2 +- .../clusterRoleBinding.yaml | 2 +- .../splunk-kubernetes-objects/configMap.yaml | 2 +- .../splunk-kubernetes-objects/deployment.yaml | 6 +-- .../splunk-kubernetes-objects/secret.yaml | 2 +- .../serviceAccount.yaml | 2 +- 22 files changed, 58 insertions(+), 76 deletions(-) diff --git a/.circleci/build_sck_artifacts_for_release.sh b/.circleci/build_sck_artifacts_for_release.sh index 8e0dca12..d852e754 100755 --- a/.circleci/build_sck_artifacts_for_release.sh +++ b/.circleci/build_sck_artifacts_for_release.sh @@ -2,26 +2,29 @@ set -e TAG=`cat VERSION` -function replace_generic_version () -{ - file="$1" - _line=`awk '/version:/{print NR;exit}' $file` - replacement="version: $TAG" - # Escape backslash, forward slash and ampersand for use as a sed replacement. - replacement_escaped=$( echo "$replacement" | sed -e 's/[\/&]/\\&/g' ) - sed -i "${_line}s/.*/$replacement_escaped/" "$file" -} +# Install yq yaml parser +wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 +sudo chmod +x yq_linux_amd64 +sudo mv yq_linux_amd64 /usr/local/bin/yq -repos_array=( "helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-kubernetes-logging" - "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) -sub_repos_array=( "helm-chart/splunk-kubernetes-logging" - "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) +yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name splunk/fluentd-hec:1.1.1 +yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name splunk/k8s-metrics:1.1.1 +yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name splunk/k8s-metrics-aggr:1.1.0 +yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name splunk/kube-objects:1.1.0 -for repo in "${repos_array[@]}" -do - filename="${repo}/Chart.yaml" - replace_generic_version $filename -done +yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION +yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION +yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml version $VERSION +yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml version $VERSION + +yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml appVersion $VERSION +yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml appVersion $VERSION +yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml appVersion $VERSION +yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml appVersion $VERSION + +yq w -i helm-chart/splunk-kubernetes-logging/requirements.yaml dependencies.splunk-kubernetes-logging.version $VERSION +yq w -i helm-chart/splunk-kubernetes-metrics/requirements.yaml dependencies.splunk-kubernetes-metrics.version $VERSION +yq w -i helm-chart/splunk-kubernetes-objects/requirements.yaml dependencies.splunk-kubernetes-objects.version $VERSION mkdir helm-artifacts-release if [[ -d "helm-chart/splunk-connect-for-kubernetes/charts" ]]; then @@ -29,11 +32,15 @@ if [[ -d "helm-chart/splunk-connect-for-kubernetes/charts" ]]; then fi mkdir helm-chart/splunk-connect-for-kubernetes/charts +sub_repos_array=( "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) for sub_repo in "${sub_repos_array[@]}" do cp -rp $sub_repo helm-chart/splunk-connect-for-kubernetes/charts done +repos_array=( "helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-kubernetes-logging" + "helm-chart/splunk-kubernetes-metrics" "helm-chart/splunk-kubernetes-objects" ) for repo in "${repos_array[@]}" do helm package -d helm-artifacts-release $repo diff --git a/.circleci/update_sck_config_for_release.sh b/.circleci/update_sck_config_for_release.sh index 1a2a778f..e802c7e1 100755 --- a/.circleci/update_sck_config_for_release.sh +++ b/.circleci/update_sck_config_for_release.sh @@ -8,44 +8,19 @@ wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 sudo chmod +x yq_linux_amd64 sudo mv yq_linux_amd64 /usr/local/bin/yq -## Modify docker images to be used -#image_name=$(echo "splunk/fluentd-hec:$VERSION") -#yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name $image_name -# -#image_name=$(echo "splunk/k8s-metrics:$VERSION") -#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name $image_name -# -#image_name=$(echo "splunk/k8s-metrics-aggr:$VERSION") -#yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name $image_name -# -#image_name=$(echo "splunk/kube-objects:$VERSION") -#yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name $image_name -# -#yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION -#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION -#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml version $VERSION -#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml version $VERSION -# -#yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml dependencies.splunk-kubernetes-logging.version $VERSION -#yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml dependencies.splunk-kubernetes-metrics.version $VERSION -#yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml dependencies.splunk-kubernetes-objects.version $VERSION - - +# setup necessary for functional tests # Modify splunk environment values yq w -i .circleci/sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN # Pull docker images locally -docker pull splunk/fluentd-hec:1.1.0 -docker pull splunk/k8s-metrics:1.1.0 +docker pull splunk/fluentd-hec:1.1.1 +docker pull splunk/k8s-metrics:1.1.1 docker pull splunk/k8s-metrics-aggr:1.1.0 docker pull splunk/kube-objects:1.1.0 # Modify docker images to be used -yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec:1.1.0 - -yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics:1.1.0 - +yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec:1.1.1 +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics:1.1.1 yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name splunk/k8s-metrics-aggr:1.1.0 - yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name splunk/kube-objects:1.1.0 diff --git a/VERSION b/VERSION index 1cc5f657..867e5243 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 \ No newline at end of file +1.2.0 \ No newline at end of file diff --git a/manifests/splunk-kubernetes-logging/configMap.yaml b/manifests/splunk-kubernetes-logging/configMap.yaml index 99624905..60d011d5 100644 --- a/manifests/splunk-kubernetes-logging/configMap.yaml +++ b/manifests/splunk-kubernetes-logging/configMap.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-logging labels: app: splunk-kubernetes-logging - version: 1.1.0 + version: 1.2.0 data: fluent.conf: |- @include system.conf diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index 8373465f..68ea2086 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -6,13 +6,13 @@ metadata: labels: app: splunk-kubernetes-logging engine: fluentd - version: 1.1.0 + version: 1.2.0 spec: template: metadata: labels: app: splunk-kubernetes-logging - version: 1.1.0 + version: 1.2.0 annotations: {} spec: tolerations: diff --git a/manifests/splunk-kubernetes-logging/secret.yaml b/manifests/splunk-kubernetes-logging/secret.yaml index 0e9acaae..6a2446c0 100644 --- a/manifests/splunk-kubernetes-logging/secret.yaml +++ b/manifests/splunk-kubernetes-logging/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-logging labels: app: splunk-kubernetes-logging - version: 1.1.0 + version: 1.2.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-metrics/clusterRole.yaml b/manifests/splunk-kubernetes-metrics/clusterRole.yaml index 22fe784d..3ea08e41 100644 --- a/manifests/splunk-kubernetes-metrics/clusterRole.yaml +++ b/manifests/splunk-kubernetes-metrics/clusterRole.yaml @@ -5,7 +5,7 @@ metadata: name: kubelet-summary-api-read labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 rules: - apiGroups: - '' diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml index d2c7b64f..2d0e04d6 100644 --- a/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/clusterRoleAggregator.yaml @@ -5,7 +5,7 @@ metadata: name: kube-api-aggregator labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 rules: - apiGroups: - '' diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml index 4bf9780c..33e49ce5 100644 --- a/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml +++ b/manifests/splunk-kubernetes-metrics/clusterRoleBinding.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml b/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml index 42f39156..e39213b6 100644 --- a/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/clusterRoleBindingAggregator.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-metrics-aggregator labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/manifests/splunk-kubernetes-metrics/configMap.yaml b/manifests/splunk-kubernetes-metrics/configMap.yaml index 6c80e9ae..30ae3e1a 100644 --- a/manifests/splunk-kubernetes-metrics/configMap.yaml +++ b/manifests/splunk-kubernetes-metrics/configMap.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 data: fluent.conf: | # system wide configurations @@ -60,7 +60,7 @@ data: hec_host MY-SPLUNK-HOST hec_port 8088 hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" - host "#{ENV['SPLUNK_HEC_HOST']}" + host "#{ENV['NODE_NAME']}" source ${tag} insecure_ssl true diff --git a/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml index 9b685ef0..d8a3aa67 100644 --- a/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/configMapMetricsAggregator.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-metrics-aggregator labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 data: fluent.conf: | # system wide configurations diff --git a/manifests/splunk-kubernetes-metrics/deployment.yaml b/manifests/splunk-kubernetes-metrics/deployment.yaml index 6259c99f..a91be2cd 100644 --- a/manifests/splunk-kubernetes-metrics/deployment.yaml +++ b/manifests/splunk-kubernetes-metrics/deployment.yaml @@ -6,12 +6,12 @@ metadata: labels: app: splunk-kubernetes-metrics engine: fluentd - version: 1.1.0 + version: 1.2.0 spec: selector: matchLabels: name: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 template: metadata: name: splunk-kubernetes-metrics @@ -19,7 +19,7 @@ spec: name: splunk-kubernetes-metrics app: splunk-kubernetes-metrics engine: fluentd - version: 1.1.0 + version: 1.2.0 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: diff --git a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml index 2d2e034f..991f3974 100644 --- a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml @@ -5,18 +5,18 @@ metadata: name: splunk-kubernetes-metrics-agg labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 spec: replicas: 1 selector: matchLabels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 template: metadata: labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: diff --git a/manifests/splunk-kubernetes-metrics/secret.yaml b/manifests/splunk-kubernetes-metrics/secret.yaml index f7e1c77d..4e7e71e8 100644 --- a/manifests/splunk-kubernetes-metrics/secret.yaml +++ b/manifests/splunk-kubernetes-metrics/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-metrics/serviceAccount.yaml b/manifests/splunk-kubernetes-metrics/serviceAccount.yaml index 5c1f46fb..c5d4439c 100644 --- a/manifests/splunk-kubernetes-metrics/serviceAccount.yaml +++ b/manifests/splunk-kubernetes-metrics/serviceAccount.yaml @@ -5,4 +5,4 @@ metadata: name: splunk-kubernetes-metrics labels: app: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 diff --git a/manifests/splunk-kubernetes-objects/clusterRole.yaml b/manifests/splunk-kubernetes-objects/clusterRole.yaml index 55928486..3aeb6d21 100644 --- a/manifests/splunk-kubernetes-objects/clusterRole.yaml +++ b/manifests/splunk-kubernetes-objects/clusterRole.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.1.0 + version: 1.2.0 rules: - apiGroups: - '' diff --git a/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml b/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml index 398dbcc5..8a72345e 100644 --- a/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml +++ b/manifests/splunk-kubernetes-objects/clusterRoleBinding.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.1.0 + version: 1.2.0 roleRef: kind: ClusterRole name: splunk-kubernetes-objects diff --git a/manifests/splunk-kubernetes-objects/configMap.yaml b/manifests/splunk-kubernetes-objects/configMap.yaml index f4de7f31..81c1a901 100644 --- a/manifests/splunk-kubernetes-objects/configMap.yaml +++ b/manifests/splunk-kubernetes-objects/configMap.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.1.0 + version: 1.2.0 data: fluent.conf: | diff --git a/manifests/splunk-kubernetes-objects/deployment.yaml b/manifests/splunk-kubernetes-objects/deployment.yaml index 4cc9a4d9..a58dfb1b 100644 --- a/manifests/splunk-kubernetes-objects/deployment.yaml +++ b/manifests/splunk-kubernetes-objects/deployment.yaml @@ -5,20 +5,20 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.1.0 + version: 1.2.0 spec: selector: matchLabels: app: splunk-kubernetes-objects engine: fluentd - version: 1.1.0 + version: 1.2.0 replicas: 1 template: metadata: labels: app: splunk-kubernetes-objects engine: fluentd - version: 1.1.0 + version: 1.2.0 annotations: {} spec: serviceAccountName: splunk-kubernetes-objects diff --git a/manifests/splunk-kubernetes-objects/secret.yaml b/manifests/splunk-kubernetes-objects/secret.yaml index ce1e022b..f5f429f6 100644 --- a/manifests/splunk-kubernetes-objects/secret.yaml +++ b/manifests/splunk-kubernetes-objects/secret.yaml @@ -5,7 +5,7 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.1.0 + version: 1.2.0 type: Opaque data: splunk_hec_token: TVktU1BMVU5LLVRPS0VO diff --git a/manifests/splunk-kubernetes-objects/serviceAccount.yaml b/manifests/splunk-kubernetes-objects/serviceAccount.yaml index f1a75f67..396c3d51 100644 --- a/manifests/splunk-kubernetes-objects/serviceAccount.yaml +++ b/manifests/splunk-kubernetes-objects/serviceAccount.yaml @@ -5,4 +5,4 @@ metadata: name: splunk-kubernetes-objects labels: app: splunk-kubernetes-objects - version: 1.1.0 + version: 1.2.0 From 9699cab34d7cd37b0b77e0a7c6417a625b553fea Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 13 Jun 2019 07:58:22 -0700 Subject: [PATCH 074/119] merge Release/1.2.0 back to develop (#170) --- README.md | 4 ++-- helm-chart/splunk-connect-for-kubernetes/Chart.yaml | 4 ++-- helm-chart/splunk-connect-for-kubernetes/README.md | 2 +- helm-chart/splunk-connect-for-kubernetes/requirements.yaml | 6 +++--- .../splunk-connect-for-kubernetes/templates/NOTES.txt | 4 ++-- helm-chart/splunk-kubernetes-logging/Chart.yaml | 4 ++-- helm-chart/splunk-kubernetes-logging/README.md | 2 +- helm-chart/splunk-kubernetes-logging/values.yaml | 5 +++-- helm-chart/splunk-kubernetes-metrics/Chart.yaml | 4 ++-- helm-chart/splunk-kubernetes-metrics/README.md | 2 +- .../templates/clusterRoleAggregator.yaml | 2 +- helm-chart/splunk-kubernetes-metrics/values.yaml | 6 +++--- helm-chart/splunk-kubernetes-objects/Chart.yaml | 4 ++-- helm-chart/splunk-kubernetes-objects/README.md | 2 +- manifests/splunk-kubernetes-logging/daemonset.yaml | 2 +- manifests/splunk-kubernetes-metrics/deployment.yaml | 2 +- 16 files changed, 28 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index c8514806..3c70256c 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Helm, maintained by the CNCF, allows the Kubernetes administrator to install, up To install and configure defaults with Helm: ``` -$ helm install --name my-release -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz +$ helm install --name my-release -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.2.0/splunk-connect-for-kubernetes-1.2.0.tgz ``` To learn more about using and modifying charts, see: @@ -144,4 +144,4 @@ If you have any issues with the software, please file an issue at [Create a new # License -See [LICENSE](https://github.com/splunk/splunk-connect-for-kubernetes/blob/master/LICENSE). \ No newline at end of file +See [LICENSE](https://github.com/splunk/splunk-connect-for-kubernetes/blob/master/LICENSE). diff --git a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml index 4b2d8e53..19db9364 100644 --- a/helm-chart/splunk-connect-for-kubernetes/Chart.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: splunk-connect-for-kubernetes -version: 1.1.0 -appVersion: 1.1.0 +version: 1.2.0 +appVersion: 1.2.0 description: Collect logs, objects (metadata), and metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart sources: diff --git a/helm-chart/splunk-connect-for-kubernetes/README.md b/helm-chart/splunk-connect-for-kubernetes/README.md index 0a66e10a..cb880cd2 100644 --- a/helm-chart/splunk-connect-for-kubernetes/README.md +++ b/helm-chart/splunk-connect-for-kubernetes/README.md @@ -17,7 +17,7 @@ First, prepare a Values file. Check the Values files in each sub-chart for detai Once you have a Values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-connect -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz +$ helm install --name my-splunk-connect -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.2.0/splunk-connect-for-kubernetes-1.2.0.tgz ``` ## Uninstall ## diff --git a/helm-chart/splunk-connect-for-kubernetes/requirements.yaml b/helm-chart/splunk-connect-for-kubernetes/requirements.yaml index 1d587f84..0f2db9a2 100644 --- a/helm-chart/splunk-connect-for-kubernetes/requirements.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/requirements.yaml @@ -1,13 +1,13 @@ dependencies: - name: splunk-kubernetes-logging - version: 1.1.0 + version: 1.2.0 repository: stable condition: logging.enabled - name: splunk-kubernetes-objects - version: 1.1.0 + version: 1.2.0 repository: stable condition: objects.enabled - name: splunk-kubernetes-metrics - version: 1.1.0 + version: 1.2.0 repository: stable condition: metrics.enabled diff --git a/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt b/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt index 141587d5..91ca3194 100644 --- a/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt +++ b/helm-chart/splunk-connect-for-kubernetes/templates/NOTES.txt @@ -70,7 +70,7 @@ Once set up run: helm upgrade {{ .Release.Name }} \ --set global.splunk.hec.host=YOUR-HEC-HOST \ --set global.splunk.hec.token=YOUR-HEC-TOKEN \ - https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz + https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.2.0/splunk-connect-for-kubernetes-1.2.0.tgz {{- end }} {{- $lInd := or $lHec.indexName $gHec.indexName | default "" }} {{- $oInd := or $oHec.indexName $gHec.indexName | default "" }} @@ -94,5 +94,5 @@ Then run: helm upgrade {{ .Release.Name }} \ --set splunk-kubernetes-metrics.splunk.hec.indexName=YOUR-METRICS-INDEX \ - https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-connect-for-kubernetes-1.1.0.tgz + https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.2.0/splunk-connect-for-kubernetes-1.2.0.tgz {{- end -}} diff --git a/helm-chart/splunk-kubernetes-logging/Chart.yaml b/helm-chart/splunk-kubernetes-logging/Chart.yaml index 98b81447..5f310f50 100644 --- a/helm-chart/splunk-kubernetes-logging/Chart.yaml +++ b/helm-chart/splunk-kubernetes-logging/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -version: 1.1.0 -appVersion: 1.1.0 +version: 1.2.0 +appVersion: 1.2.0 name: splunk-kubernetes-logging description: Collect logs with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-logging diff --git a/helm-chart/splunk-kubernetes-logging/README.md b/helm-chart/splunk-kubernetes-logging/README.md index 53481439..9370bb74 100644 --- a/helm-chart/splunk-kubernetes-logging/README.md +++ b/helm-chart/splunk-kubernetes-logging/README.md @@ -17,7 +17,7 @@ First, prepare a values file. You can also check the [examples](examples) for qu Once you have a values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-logging -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-kubernetes-logging-1.1.0.tgz +$ helm install --name my-splunk-logging -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.2.0/splunk-kubernetes-logging-1.2.0.tgz ``` ## Uninstall ## diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 085d2521..a8b813b4 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -21,6 +21,7 @@ global: kubernetes: clusterName: "cluster_name" + # logLevel is to set log level of the Splunk log collector. Avaiable values are: # * trace # * debug @@ -232,7 +233,7 @@ logs: # Defines which version of image to use, and how it should be pulled. image: - name: splunk/fluentd-hec:1.1.0 + name: splunk/fluentd-hec:1.1.1 pullPolicy: Always @@ -282,4 +283,4 @@ affinity: {} # = Kubernetes Connection Configs = kubernetes: # The cluster name used to tag logs. Default is cluster_name - clusterName: "cluster_name" \ No newline at end of file + clusterName: "cluster_name" diff --git a/helm-chart/splunk-kubernetes-metrics/Chart.yaml b/helm-chart/splunk-kubernetes-metrics/Chart.yaml index 33e4c86b..63a8ab5b 100644 --- a/helm-chart/splunk-kubernetes-metrics/Chart.yaml +++ b/helm-chart/splunk-kubernetes-metrics/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -version: 1.1.0 -appVersion: 1.1.0 +version: 1.2.0 +appVersion: 1.2.0 name: splunk-kubernetes-metrics description: Collect metrics with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-metrics diff --git a/helm-chart/splunk-kubernetes-metrics/README.md b/helm-chart/splunk-kubernetes-metrics/README.md index a18e3873..c1afcfd6 100644 --- a/helm-chart/splunk-kubernetes-metrics/README.md +++ b/helm-chart/splunk-kubernetes-metrics/README.md @@ -17,7 +17,7 @@ Then, prepare a values file. You can also check the [examples](examples) for qui Once you have a values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-metrics -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-kubernetes-metrics-1.1.0.tgz +$ helm install --name my-splunk-metrics -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.2.0/splunk-kubernetes-metrics-1.2.0.tgz ``` diff --git a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml index 29bbf384..281d8ba4 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/clusterRoleAggregator.yaml @@ -20,4 +20,4 @@ rules: verbs: - get - list -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index ad810b76..96bec932 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -81,7 +81,7 @@ secret: image: - name: splunk/k8s-metrics:1.1.0 + name: splunk/k8s-metrics:1.1.1 pullPolicy: Always @@ -111,7 +111,7 @@ buffer: "@type": memory total_limit_size: 400m chunk_limit_size: 100m - chunk_limit_records: 1000000 + chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block @@ -122,7 +122,7 @@ aggregatorBuffer: "@type": memory total_limit_size: 400m chunk_limit_size: 100m - chunk_limit_records: 1000000 + chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block diff --git a/helm-chart/splunk-kubernetes-objects/Chart.yaml b/helm-chart/splunk-kubernetes-objects/Chart.yaml index dcb5bb8e..7d84d3e8 100644 --- a/helm-chart/splunk-kubernetes-objects/Chart.yaml +++ b/helm-chart/splunk-kubernetes-objects/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 -version: 1.1.0 -appVersion: 1.1.0 +version: 1.2.0 +appVersion: 1.2.0 name: splunk-kubernetes-objects description: Collect Kubernetes objects with Splunk. home: https://github.com/splunk/splunk-connect-for-kubernetes/tree/master/helm-chart/charts/splunk-kubernetes-objects diff --git a/helm-chart/splunk-kubernetes-objects/README.md b/helm-chart/splunk-kubernetes-objects/README.md index 9dbb21b7..3ec2a746 100644 --- a/helm-chart/splunk-kubernetes-objects/README.md +++ b/helm-chart/splunk-kubernetes-objects/README.md @@ -17,7 +17,7 @@ First, prepare a values file. You can also check the [examples](examples) for qu Once you have a values file, you can simply install the chart with a release name (optional) by running ```bash -$ helm install --name my-splunk-objects -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.1.0/splunk-kubernetes-objects-1.1.0.tgz +$ helm install --name my-splunk-objects -f my_values.yaml https://github.com/splunk/splunk-connect-for-kubernetes/releases/download/1.2.0/splunk-kubernetes-objects-1.2.0.tgz ``` ## Uninstall ## diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index 68ea2086..9ae20901 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -20,7 +20,7 @@ spec: key: node-role.kubernetes.io/master containers: - name: splunk-fluentd-k8s-logs - image: splunk/fluentd-hec:1.1.0 + image: splunk/fluentd-hec:1.1.1 imagePullPolicy: Always args: - "-c" diff --git a/manifests/splunk-kubernetes-metrics/deployment.yaml b/manifests/splunk-kubernetes-metrics/deployment.yaml index a91be2cd..117804b4 100644 --- a/manifests/splunk-kubernetes-metrics/deployment.yaml +++ b/manifests/splunk-kubernetes-metrics/deployment.yaml @@ -29,7 +29,7 @@ spec: key: node-role.kubernetes.io/master containers: - name: splunk-fluentd-k8s-metrics - image: splunk/k8s-metrics:1.1.0 + image: splunk/k8s-metrics:1.1.1 imagePullPolicy: Always env: - name: KUBERNETES_NODE_IP From 06af4dbe6978a6fd208d458de71d04c52fa4969f Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 13 Jun 2019 10:54:05 -0700 Subject: [PATCH 075/119] Fix for CI - force upgrade tiller to current helm version (#173) --- .circleci/deploy_connector.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh index 951d8103..96d29cf2 100755 --- a/.circleci/deploy_connector.sh +++ b/.circleci/deploy_connector.sh @@ -9,7 +9,7 @@ sudo mkdir ~/.kube echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg sudo mv config ~/.kube/config #Update helm server version -helm init --upgrade +helm init --force-upgrade #Make sure to check and clean previously failed deployment echo "Checking if previous deployment exist..." if [ "`helm ls`" == "" ]; then From 20ad19026f7fcf195e5b332bd4b76d5527e0dbcc Mon Sep 17 00:00:00 2001 From: gp510 Date: Thu, 13 Jun 2019 11:14:02 -0700 Subject: [PATCH 076/119] Separating out CLA and adding PR template (#158) --- .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++++++++++++++++ CLA.md | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CLA.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..2be28346 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ +## Proposed changes + +Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request. If it fixes a bug or resolves a feature request, be sure to link to that issue. + +## Types of changes + +What types of changes does your code introduce? +_Put an `x` in the boxes that apply_ + +- [ ] Bugfix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) + +## Checklist + +_Put an `x` in the boxes that apply._ + +- [ ] I have read the [CONTRIBUTING](https://github.com/splunk/splunk-connect-for-kubernetes/blob/develop/CONTRIBUTING.md) doc +- [ ] I have read the [CLA](https://github.com/splunk/splunk-connect-for-kubernetes/blob/develop/CLA.md) +- [ ] I have added necessary documentation (if appropriate) +- [ ] Any dependent changes have been merged and published in downstream modules + diff --git a/CLA.md b/CLA.md new file mode 100644 index 00000000..8964061a --- /dev/null +++ b/CLA.md @@ -0,0 +1,18 @@ +By submitting a Contribution to this Work, You agree that Your Contribution is made subject to the primary LICENSE +file applicable to this Work. In addition, You represent that: (i) You are the copyright owner of the Contribution +or (ii) You have the requisite rights to make the Contribution. + +Definitions: + +“You” shall mean: (i) yourself if you are making a Contribution on your own behalf; or (ii) your company, +if you are making a Contribution on behalf of your company. If you are making a Contribution on behalf of your +company, you represent that you have the requisite authority to do so. + +"Contribution" shall mean any original work of authorship, including any modifications or additions to an existing +work, that is intentionally submitted by You for inclusion in, or documentation of, this project/repository. For the +purposes of this definition, "submitted" means any form of electronic, verbal, or written communication submitted for +inclusion in this project/repository, including but not limited to communication on electronic mailing lists, source +code control systems, and issue tracking systems that are managed by, or on behalf of, the maintainers of +the project/repository. + +“Work” shall mean the collective software, content, and documentation in this project/repository. From 5af424b686e0ca400f721d0b9a3b350bd88da85b Mon Sep 17 00:00:00 2001 From: Paul Braham Date: Fri, 14 Jun 2019 16:06:55 +0100 Subject: [PATCH 077/119] =?UTF-8?q?add=20priorityClassName=20to=20all=20he?= =?UTF-8?q?lm=20charts=20to=20allow=20definition=20of=20pod=20p=E2=80=A6?= =?UTF-8?q?=20(#172)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add priorityClassName to all helm charts to allow definition of pod priority class for logging --- helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml | 2 ++ helm-chart/splunk-kubernetes-logging/values.yaml | 2 ++ helm-chart/splunk-kubernetes-metrics/values.yaml | 1 + 3 files changed, 5 insertions(+) diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 50e47c36..0d3e2422 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -87,3 +87,5 @@ spec: - name: secrets secret: secretName: {{ template "splunk-kubernetes-logging.secret" . }} + priorityClassName: {{ .Values.priorityClassName }} + \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index a8b813b4..31b7289c 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -279,6 +279,8 @@ nodeSelector: {} # Defines node affinity to restrict pod deployment. affinity: {} +# Defines priorityClassName to assign a priority class to pods. +priorityClassName: # = Kubernetes Connection Configs = kubernetes: diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 96bec932..bd0dc84e 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -147,6 +147,7 @@ tolerations: # blank by default. aggregatorTolerations: {} + affinity: {} # = Kubernetes Connection Configs = From 12695d0184853e68411b86fcd60625b0917c080b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vidar=20Waagb=C3=B8?= Date: Fri, 14 Jun 2019 17:12:04 +0200 Subject: [PATCH 078/119] Added support for path and exclude_path in fluentd (#174) --- .../templates/configMap.yaml | 5 ++++- helm-chart/splunk-kubernetes-logging/values.yaml | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 33e1bb41..2be1d9a2 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -44,7 +44,10 @@ data: @type tail @label @SPLUNK tag tail.containers.* - path /var/log/containers/*.log + path {{ .Values.global.fluentd.path | default "/var/log/containers/*.log" }} + {{- if .Values.global.fluentd.exclude_path }} + exclude_path {{ .Values.global.fluentd.exclude_path | toJson }} + {{- end }} pos_file /var/log/splunk-fluentd-containers.log.pos path_key source read_from_head true diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 31b7289c..fa20139f 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -20,6 +20,17 @@ global: indexRoutingDefaultIndex: kubernetes: clusterName: "cluster_name" + + # Override global Fluentd config path and exclude path. + # This is can be used to exclude verbose logs including various system and Helm/Tiller related logs. + fluentd: + # path of logfiles, default /var/log/containers/*.log + path: /var/log/containers/*.log + # paths of logfiles to exclude. object type is array as per fluentd specification: + # https://docs.fluentd.org/input/tail#exclude_path + exclude_path: + # - /var/log/containers/kube-svc-redirect*.log + # - /var/log/containers/tiller*.log # logLevel is to set log level of the Splunk log collector. Avaiable values are: From 2180e598877610c666a378a923c91df15017c635 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 20 Jun 2019 10:53:41 -0700 Subject: [PATCH 079/119] Patch for priority class name (#175) --- .../splunk-kubernetes-logging/templates/daemonset.yaml | 5 +++-- helm-chart/splunk-kubernetes-logging/values.yaml | 5 ++++- .../splunk-kubernetes-metrics/templates/deployment.yaml | 3 +++ helm-chart/splunk-kubernetes-metrics/values.yaml | 5 +++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 0d3e2422..49c4abdc 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -87,5 +87,6 @@ spec: - name: secrets secret: secretName: {{ template "splunk-kubernetes-logging.secret" . }} - priorityClassName: {{ .Values.priorityClassName }} - \ No newline at end of file + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} {{ toYaml . | indent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index fa20139f..8711405c 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -20,7 +20,8 @@ global: indexRoutingDefaultIndex: kubernetes: clusterName: "cluster_name" - + + # Override global Fluentd config path and exclude path. # This is can be used to exclude verbose logs including various system and Helm/Tiller related logs. fluentd: @@ -290,9 +291,11 @@ nodeSelector: {} # Defines node affinity to restrict pod deployment. affinity: {} + # Defines priorityClassName to assign a priority class to pods. priorityClassName: + # = Kubernetes Connection Configs = kubernetes: # The cluster name used to tag logs. Default is cluster_name diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index c8dfba17..2ccd30f7 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -72,3 +72,6 @@ spec: - name: secrets secret: secretName: {{ template "splunk-kubernetes-metrics.secret" . }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} {{ toYaml . | indent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index bd0dc84e..c7c9099e 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -143,11 +143,16 @@ tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule + # Tolerations for the aggregator pod. We do not really want this running on the master nodes, so we leave this # blank by default. aggregatorTolerations: {} +# Defines priorityClassName to assign a priority class to pods. +priorityClassName: + + affinity: {} # = Kubernetes Connection Configs = From a3d73eaa678e9954a6f054fa0917060c5eb162ea Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 20 Jun 2019 10:53:53 -0700 Subject: [PATCH 080/119] Add helm linting to SCK (#176) --- .circleci/chart_schema.yaml | 20 ++++++++++++++++++ .circleci/config.yml | 6 +++++- .circleci/lint.sh | 17 +++++++++++++++ .circleci/lintconf.yaml | 42 +++++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 .circleci/chart_schema.yaml create mode 100755 .circleci/lint.sh create mode 100644 .circleci/lintconf.yaml diff --git a/.circleci/chart_schema.yaml b/.circleci/chart_schema.yaml new file mode 100644 index 00000000..e7502236 --- /dev/null +++ b/.circleci/chart_schema.yaml @@ -0,0 +1,20 @@ +name: str() +home: str() +version: str() +appVersion: any(str(), num()) +description: str() +keywords: list(str(), required=False) +sources: list(str(), required=False) +maintainers: list(include('maintainer'), required=False) +icon: str(required=False) +engine: str(required=False) +condition: str(required=False) +tags: str(required=False) +deprecated: bool(required=False) +kubeVersion: str(required=False) +annotations: map(str(), str(), required=False) +--- +maintainer: + name: str() + email: str(required=False) + url: str(required=False) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8a862a2..0117824b 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,6 +19,10 @@ jobs: helm init -c echo "Installing aws cli..." sudo pip install awscli > /dev/null 2>&1 + - run: + name: Lint helm charts + command: | + .circleci/lint.sh - run: name: Update configurables to build artifacts command: | @@ -133,4 +137,4 @@ workflows: - build filters: branches: - only: release-testing \ No newline at end of file + only: release-testing diff --git a/.circleci/lint.sh b/.circleci/lint.sh new file mode 100755 index 00000000..86ecddbb --- /dev/null +++ b/.circleci/lint.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e + +# chart template linting +wget https://github.com/helm/chart-testing/releases/download/v2.3.3/chart-testing_2.3.3_linux_amd64.tar.gz +tar -xvf chart-testing_2.3.3_linux_amd64.tar.gz +sudo chmod +x ct +sudo mv ct /usr/local/bin/ct + +# lint helm charts +echo "Linting charts" +helm lint helm-chart/* +echo "Linting charts with CI values" +helm lint --values .circleci/sck_values.yml helm-chart/* +cd .circleci +echo "Linting charts ct lint tool" +ct lint helm-chart/* diff --git a/.circleci/lintconf.yaml b/.circleci/lintconf.yaml new file mode 100644 index 00000000..90f48c88 --- /dev/null +++ b/.circleci/lintconf.yaml @@ -0,0 +1,42 @@ +--- +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + colons: + max-spaces-before: 0 + max-spaces-after: 1 + commas: + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + require-starting-space: true + min-spaces-from-content: 2 + document-end: disable + document-start: disable # No --- to start a file + empty-lines: + max: 2 + max-start: 0 + max-end: 0 + hyphens: + max-spaces-after: 1 + indentation: + spaces: consistent + indent-sequences: whatever # - list indentation will handle both indentation and without + check-multi-line-strings: false + key-duplicates: enable + line-length: disable # Lines can be any length + new-line-at-end-of-file: enable + new-lines: + type: unix + trailing-spaces: enable + truthy: + level: warning From 0567cd56ecc2a86b5fc6b8982eb3a921880c799e Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 20 Jun 2019 14:37:47 -0700 Subject: [PATCH 081/119] Suppress stdout and remove sensitive logs (#179) --- .circleci/run_functional_tests.sh | 2 +- test/common.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/run_functional_tests.sh b/.circleci/run_functional_tests.sh index 0b87ceb4..e1c31318 100755 --- a/.circleci/run_functional_tests.sh +++ b/.circleci/run_functional_tests.sh @@ -9,4 +9,4 @@ python3 -m pytest \ --splunkd-url https://$SPLUNK_HEC_HOST:8089 \ --splunk-user admin \ --splunk-password $SPLUNK_PASSWORD \ - -p no:warnings \ No newline at end of file + -p no:warnings -s diff --git a/test/common.py b/test/common.py index e58074fa..40cc5668 100644 --- a/test/common.py +++ b/test/common.py @@ -45,7 +45,7 @@ def check_events_from_splunk(index="circleci_events", ''' send a search request to splunk and return the events from the result ''' - logger.info("search query = " + str(query)) + #logger.info("search query = " + str(query)) events = _collect_events(query, start_time, end_time, url, user, password) return events @@ -178,19 +178,19 @@ def _collect_metrics(start_time, end_time, url="", user="", password="", index=" url) logger.debug('requesting: %s', api_url) - + create_job = _requests_retry_session().get( api_url, auth=(user, password), verify=False ) - + _check_request_status(create_job) json_res = create_job.json() events = json_res['entry'] - logger.info('events: %s', events) + #logger.info('events: %s', events) return events From 771bdd60ff41fa34fdb40d6a12268d08b2eefd16 Mon Sep 17 00:00:00 2001 From: Kousik Sundar Date: Thu, 18 Jul 2019 08:43:50 -0700 Subject: [PATCH 082/119] Fix host field with kubernetes node name (#193) --- helm-chart/splunk-kubernetes-logging/templates/configMap.yaml | 2 +- helm-chart/splunk-kubernetes-objects/templates/configMap.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 2be1d9a2..13ba6eb6 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -211,7 +211,7 @@ data: hec_port {{ . }} {{- end }} hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" - host "#{ENV['SPLUNK_HEC_HOST']}" + host "#{ENV['NODE_NAME']}" source_key source sourcetype_key sourcetype {{- if or .Values.splunk.hec.indexRouting .Values.global.splunk.hec.indexRouting }} diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index 0c6ab2e0..c342083c 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -97,7 +97,7 @@ data: hec_port {{ . }} {{- end }} hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" - host "#{ENV['SPLUNK_HEC_HOST']}" + host "#{ENV['NODE_NAME']}" source_key source sourcetype_key sourcetype {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} From 0a85c31828a90c89486d231eb6b9cde3e09bdc94 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 18 Jul 2019 09:53:33 -0700 Subject: [PATCH 083/119] Fix for metrics and metrics-agg host field blank (#197) --- .../templates/configMapMetricsAggregator.yaml | 2 +- .../templates/deploymentMetricsAggregator.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml index 675d06dd..f4ae4479 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMapMetricsAggregator.yaml @@ -84,7 +84,7 @@ data: hec_port {{ . }} {{- end }} hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" - host "#{ENV['MY_NODE_NAME']}" + host "#{ENV['NODE_NAME']}" {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} index {{ . }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index 508bf9df..1651800e 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -45,6 +45,10 @@ spec: secretKeyRef: name: {{ template "splunk-kubernetes-metrics.secret" . }} key: splunk_hec_token + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName resources: {{ toYaml .Values.resources.fluent | indent 12 }} volumeMounts: From 45a22ff327a753acb2a74c39df874ca9d30272d9 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 19 Jul 2019 12:02:06 -0700 Subject: [PATCH 084/119] Openshift support (#190) * Add service account to logging daemonset * Added openshift support --- .../splunk-kubernetes-logging/templates/_helpers.tpl | 11 +++++++++++ .../templates/daemonset.yaml | 9 ++++++++- .../templates/serviceAccount.yaml | 11 +++++++++++ helm-chart/splunk-kubernetes-logging/values.yaml | 10 ++++++++++ .../templates/deployment.yaml | 6 ++++++ .../templates/deploymentMetricsAggregator.yaml | 6 ++++++ helm-chart/splunk-kubernetes-metrics/values.yaml | 2 ++ .../templates/deployment.yaml | 6 ++++++ helm-chart/splunk-kubernetes-objects/values.yaml | 2 ++ 9 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index f093c5a3..d38698aa 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -116,3 +116,14 @@ def extract_container_info: .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // "kube:container:\(.container_name)") {{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "splunk-kubernetes-logging.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "splunk-kubernetes-logging.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 49c4abdc..57a7a672 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -17,6 +17,7 @@ spec: annotations: checksum/config: {{ include (print $.Template.BasePath "/configMap.yaml") . | sha256sum }} spec: + serviceAccountName: {{ template "splunk-kubernetes-logging.serviceAccountName" . }} {{- with .Values.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} @@ -33,6 +34,12 @@ spec: - name: splunk-fluentd-k8s-logs image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} + securityContext: + {{ toYaml . | indent 12 }} + privileged: true + runAsUser: 0 + {{- end }} args: - -c - /fluentd/etc/fluent.conf @@ -89,4 +96,4 @@ spec: secretName: {{ template "splunk-kubernetes-logging.secret" . }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{ toYaml . | indent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml b/helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml new file mode 100644 index 00000000..fee73b63 --- /dev/null +++ b/helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "splunk-kubernetes-logging.serviceAccountName" . }} + labels: + app: {{ template "splunk-kubernetes-logging.name" . }} + chart: {{ template "splunk-kubernetes-logging.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 8711405c..5a5f02af 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -43,6 +43,14 @@ global: logLevel: +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + + # Local splunk configurations splunk: # Configurations for HEC (HTTP Event Collector) @@ -300,3 +308,5 @@ priorityClassName: kubernetes: # The cluster name used to tag logs. Default is cluster_name clusterName: "cluster_name" + # Add privileged access to containers for openshift compatability + openshift: false diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index 2ccd30f7..cb4a198d 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -43,6 +43,12 @@ spec: - name: splunk-fluentd-k8s-metrics image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} + securityContext: + {{ toYaml . | indent 12 }} + privileged: true + runAsUser: 0 + {{- end }} env: - name: KUBERNETES_NODE_IP valueFrom: diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index 1651800e..3c8b65bd 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -39,6 +39,12 @@ spec: - name: splunk-fluentd-k8s-metrics-agg image: {{ .Values.imageAgg.name }} imagePullPolicy: {{ .Values.imageAgg.pullPolicy }} + {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} + securityContext: + {{ toYaml . | indent 12 }} + privileged: true + runAsUser: 0 + {{- end }} env: - name: SPLUNK_HEC_TOKEN valueFrom: diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index c7c9099e..b4f11db1 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -174,3 +174,5 @@ kubernetes: secretDir: # The cluster name used to tag cluster metrics from the aggregator. Default is cluster_name clusterName: "cluster_name" + # Add privileged access to containers for openshift compatability + openshift: false diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index b1c7c7f7..7d37e89a 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -41,6 +41,12 @@ spec: - name: splunk-fluentd-k8s-objects image: {{ .Values.image.name }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} + securityContext: + {{ toYaml . | indent 12 }} + privileged: true + runAsUser: 0 + {{- end }} args: - -c - /fluentd/etc/fluent.conf diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index fd1c4e83..41352df2 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -64,6 +64,8 @@ kubernetes: secretDir: # The cluster name used to tag cluster metrics from the aggregator. Default is cluster_name clusterName: "cluster_name" + # Add privileged access to containers for openshift compatability + openshift: false # = Object Lists = From 8be55f007e5049343eabf1c1200aa8441dd98ac2 Mon Sep 17 00:00:00 2001 From: Jeff Wu Date: Fri, 19 Jul 2019 12:09:01 -0700 Subject: [PATCH 085/119] resolves issue #143, adding CRI support (#151) * resolves issue #143 * cleaning up exported fields * Documenting defaults in values.yaml * Updating containers config location. * Setting sourcetype to container_name * Adding the "kube:container:" prefix to sourcetype. * Removing the dockerPath value setting. --- .../templates/configMap.yaml | 22 +++++++++++++++++++ .../templates/daemonset.yaml | 12 +++++----- .../splunk-kubernetes-logging/values.yaml | 13 ++++++----- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 13ba6eb6..8d1053e5 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -52,7 +52,12 @@ data: path_key source read_from_head true + {{- if eq .Values.containers.logFormat "cri" }} + @type regexp + expression /^(? + # extract pod_uid and container_name for CRIO runtime + {{- if eq .Values.containers.logFormat "cri" }} + + @type jq_transformer + jq '.record | . + (.source | capture("/var/log/pods/(?[^/]+)/(?[^/]+)/(?[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)' + + {{- end }} + # = filters for journald logs = {{- range $name, $logDef := .Values.logs }} {{- if and $logDef.from.journald $logDef.multiline }} @@ -232,10 +245,19 @@ data: ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} + # currently CRI does not produce log paths with all the necessary + # metadata to parse out pod, namespace, container_name, container_id. + # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031 + {{- if eq .Values.containers.logFormat "cri"}} + pod_uid + container_name + container_retry + {{- else }} pod namespace container_name container_id + {{- end }} {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{- end }} diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 57a7a672..2357f47d 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -65,9 +65,9 @@ spec: {{ toYaml .Values.resources | indent 10 }} volumeMounts: - name: varlog - mountPath: /var/log - - name: varlibdockercontainers - mountPath: {{ with .Values.dockerPath }}{{ printf "%s/containers" . | quote }}{{ end }} + mountPath: {{ .Values.containers.path }} + - name: varlogdest + mountPath: {{ .Values.containers.pathDest }} readOnly: true - name: journallogpath mountPath: {{ .Values.journalLogPath | quote }} @@ -81,10 +81,10 @@ spec: volumes: - name: varlog hostPath: - path: /var/log - - name: varlibdockercontainers + path: {{ .Values.containers.path }} + - name: varlogdest hostPath: - path: {{ with .Values.dockerPath }}{{ printf "%s/containers" . | quote }}{{ end }} + path: {{ .Values.containers.pathDest }} - name: journallogpath hostPath: path: {{ .Values.journalLogPath | quote }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 5a5f02af..ba2b6449 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -42,6 +42,14 @@ global: # * error logLevel: +# Configurations for container logs +containers: + # Path to root directory of container logs + path: /var/log + # Final volume destination of container log symlinks + pathDest: /var/lib/docker/containers + # Log format, "json" or "cri" + logFormat: json serviceAccount: # Specifies whether a ServiceAccount should be created @@ -94,11 +102,6 @@ secret: # Directory where to read journald logs. journalLogPath: /run/log/journal - -# Directory where the docker daemon's data is stored -dockerPath: /var/lib/docker - - # `logs` defines the source of logs, multiline support, and their sourcetypes. # # The scheme to define a log is: From 569324c5febf1ba537f4cb8e53553ed95f9b8e35 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 23 Jul 2019 09:12:03 -0700 Subject: [PATCH 086/119] Make CRI-O log format configurable (#199) * Make CRI-O log format configurable --- .../templates/configMap.yaml | 11 ++++++----- helm-chart/splunk-kubernetes-logging/values.yaml | 7 +++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 8d1053e5..b2753428 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -52,15 +52,16 @@ data: path_key source read_from_head true - {{- if eq .Values.containers.logFormat "cri" }} + {{- if eq .Values.containers.logFormatType "cri" }} @type regexp expression /^(? @@ -168,7 +169,7 @@ data: # extract pod_uid and container_name for CRIO runtime - {{- if eq .Values.containers.logFormat "cri" }} + {{- if eq .Values.containers.logFormatType "cri" }} @type jq_transformer jq '.record | . + (.source | capture("/var/log/pods/(?[^/]+)/(?[^/]+)/(?[0-9]+).log")) | .sourcetype = ("kube:container:" + .container_name)' @@ -248,7 +249,7 @@ data: # currently CRI does not produce log paths with all the necessary # metadata to parse out pod, namespace, container_name, container_id. # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031 - {{- if eq .Values.containers.logFormat "cri"}} + {{- if eq .Values.containers.logFormatType "cri"}} pod_uid container_name container_retry diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index ba2b6449..0abf3dc7 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -48,8 +48,11 @@ containers: path: /var/log # Final volume destination of container log symlinks pathDest: /var/lib/docker/containers - # Log format, "json" or "cri" - logFormat: json + # Log format type, "json" or "cri" + logFormatType: json + # Specify the log format for "cri" logFormatType + # It can be "%Y-%m-%dT%H:%M:%S.%N%:z" for openshift and "%Y-%m-%dT%H:%M:%S.%NZ" for IBM IKS + logFormat: serviceAccount: # Specifies whether a ServiceAccount should be created From efda8efd74ac2c2de95de5222b0c34e277a2ed43 Mon Sep 17 00:00:00 2001 From: Ryler Hockenbury Date: Tue, 23 Jul 2019 12:41:19 -0400 Subject: [PATCH 087/119] extensions/v1beta1 -> apps/v1 (#186) --- .../splunk-kubernetes-logging/templates/daemonset.yaml | 6 +++++- .../templates/deploymentMetricsAggregator.yaml | 2 +- .../splunk-kubernetes-objects/templates/deployment.yaml | 2 +- manifests/splunk-kubernetes-logging/daemonset.yaml | 6 +++++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 2357f47d..90dd5100 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: {{ template "splunk-kubernetes-logging.fullname" . }} @@ -9,6 +9,10 @@ metadata: heritage: {{ .Release.Service }} engine: fluentd spec: + selector: + matchLabels: + app: {{ template "splunk-kubernetes-logging.name" . }} + release: {{ .Release.Name }} template: metadata: labels: diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index 3c8b65bd..5c1aee0f 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ template "splunk-kubernetes-metrics.fullname" . }}-agg diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index 7d37e89a..ef1bd498 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ template "splunk-kubernetes-objects.fullname" . }} diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index 9ae20901..8149df21 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -1,5 +1,5 @@ --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: splunk-kubernetes-logging @@ -8,6 +8,10 @@ metadata: engine: fluentd version: 1.2.0 spec: + selector: + matchLabels: + app: splunk-kubernetes-logging + version: 1.2.0 template: metadata: labels: From 2addba7458d9e30a31f7bb861f9cdc71cb155dca Mon Sep 17 00:00:00 2001 From: Kousik Sundar Date: Tue, 23 Jul 2019 09:41:59 -0700 Subject: [PATCH 088/119] Add cluster-name for journald and file sources of logs (#195) --- helm-chart/splunk-kubernetes-logging/templates/configMap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index b2753428..14c5d96e 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -193,7 +193,7 @@ data: {{- if $checks.hasJournald }} @type jq_transformer - jq '.record.source = "{{ .Values.journalLogPath }}/" + .record.source | .record.sourcetype = (.tag | ltrimstr("journald.")) | .record' + jq '.record.source = "{{ .Values.journalLogPath }}/" + .record.source | .record.sourcetype = (.tag | ltrimstr("journald.")) | .record.cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" | .record' {{- end }} @@ -209,7 +209,7 @@ data: # = filters for monitor agent = @type jq_transformer - jq ".record.source = \"namespace:#{ENV['MY_NAMESPACE']}/pod:#{ENV['MY_POD_NAME']}\" | .record.sourcetype = \"fluentd:monitor-agent\" | .record" + jq ".record.source = \"namespace:#{ENV['MY_NAMESPACE']}/pod:#{ENV['MY_POD_NAME']}\" | .record.sourcetype = \"fluentd:monitor-agent\" | .record.cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" | .record" # = output = From 4bbeb829e65996076e9a0e4161c8e174d1ad8190 Mon Sep 17 00:00:00 2001 From: Ryler Hockenbury Date: Thu, 25 Jul 2019 13:15:05 -0400 Subject: [PATCH 089/119] Add linux node selector to deployments (#182) --- manifests/splunk-kubernetes-logging/daemonset.yaml | 2 ++ manifests/splunk-kubernetes-metrics/deployment.yaml | 2 ++ .../splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml | 2 ++ manifests/splunk-kubernetes-objects/deployment.yaml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index 8149df21..fc1f10db 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -22,6 +22,8 @@ spec: tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: splunk-fluentd-k8s-logs image: splunk/fluentd-hec:1.1.1 diff --git a/manifests/splunk-kubernetes-metrics/deployment.yaml b/manifests/splunk-kubernetes-metrics/deployment.yaml index 117804b4..ffbab357 100644 --- a/manifests/splunk-kubernetes-metrics/deployment.yaml +++ b/manifests/splunk-kubernetes-metrics/deployment.yaml @@ -27,6 +27,8 @@ spec: tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: splunk-fluentd-k8s-metrics image: splunk/k8s-metrics:1.1.1 diff --git a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml index 991f3974..917d42d0 100644 --- a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml @@ -21,6 +21,8 @@ spec: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: splunk-kubernetes-metrics + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: splunk-fluentd-k8s-metrics-agg image: splunk/k8s-metrics-aggr:1.1.0 diff --git a/manifests/splunk-kubernetes-objects/deployment.yaml b/manifests/splunk-kubernetes-objects/deployment.yaml index a58dfb1b..1a83e428 100644 --- a/manifests/splunk-kubernetes-objects/deployment.yaml +++ b/manifests/splunk-kubernetes-objects/deployment.yaml @@ -22,6 +22,8 @@ spec: annotations: {} spec: serviceAccountName: splunk-kubernetes-objects + nodeSelector: + beta.kubernetes.io/os: linux terminationGracePeriodSeconds: 30 containers: - name: splunk-fluentd-k8s-objects From fe35029d8be4623a03f08e1ea78cd9048a6b5595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarle=20B=C3=B8rsheim?= Date: Mon, 5 Aug 2019 18:02:42 +0200 Subject: [PATCH 090/119] Fixed problem in splunk-kubernetes-logging configMap. #208 (#213) --- helm-chart/splunk-kubernetes-logging/templates/configMap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 14c5d96e..c7c89acf 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -209,7 +209,7 @@ data: # = filters for monitor agent = @type jq_transformer - jq ".record.source = \"namespace:#{ENV['MY_NAMESPACE']}/pod:#{ENV['MY_POD_NAME']}\" | .record.sourcetype = \"fluentd:monitor-agent\" | .record.cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" | .record" + jq '.record.source = "namespace:#{ENV[\'MY_NAMESPACE\']}/pod:#{ENV[\'MY_POD_NAME\']}" | .record.sourcetype = "fluentd:monitor-agent" | .record.cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" | .record' # = output = From a1eda78a16d6fcdba2247acd09ac78b9f7d83305 Mon Sep 17 00:00:00 2001 From: Gustavo Michels Date: Mon, 5 Aug 2019 12:36:35 -0400 Subject: [PATCH 091/119] Ensure clusterName can be used from global.kubernetes.clusterName (#207) --- helm-chart/splunk-kubernetes-logging/values.yaml | 2 +- helm-chart/splunk-kubernetes-metrics/values.yaml | 2 +- helm-chart/splunk-kubernetes-objects/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 0abf3dc7..2c6bfe13 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -313,6 +313,6 @@ priorityClassName: # = Kubernetes Connection Configs = kubernetes: # The cluster name used to tag logs. Default is cluster_name - clusterName: "cluster_name" + clusterName: # Add privileged access to containers for openshift compatability openshift: false diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index b4f11db1..a089087a 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -173,6 +173,6 @@ kubernetes: # Path of the location where pod's service account's credentials are stored. Usually you don't need to care about this config, the default value should work in most cases. secretDir: # The cluster name used to tag cluster metrics from the aggregator. Default is cluster_name - clusterName: "cluster_name" + clusterName: # Add privileged access to containers for openshift compatability openshift: false diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 41352df2..0dc653b4 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -63,7 +63,7 @@ kubernetes: # Path of the location where pod's service account's credentials are stored. Usually you don't need to care about this config, the default value should work in most cases. secretDir: # The cluster name used to tag cluster metrics from the aggregator. Default is cluster_name - clusterName: "cluster_name" + clusterName: # Add privileged access to containers for openshift compatability openshift: false From e9a5e5c88ed55b0ff47ba75459adb09a466bd91b Mon Sep 17 00:00:00 2001 From: Paul Braham Date: Mon, 5 Aug 2019 17:40:24 +0100 Subject: [PATCH 092/119] removed unnecessary .toYAML from priorityClassName to allow it to be used as before (#210) --- helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml | 2 +- .../splunk-kubernetes-metrics/templates/deployment.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 90dd5100..869ffe73 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -99,5 +99,5 @@ spec: secret: secretName: {{ template "splunk-kubernetes-logging.secret" . }} {{- if .Values.priorityClassName }} - priorityClassName: {{ .Values.priorityClassName }} {{ toYaml . | indent 8 }} + priorityClassName: {{ .Values.priorityClassName }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index cb4a198d..e0602fbb 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -79,5 +79,5 @@ spec: secret: secretName: {{ template "splunk-kubernetes-metrics.secret" . }} {{- if .Values.priorityClassName }} - priorityClassName: {{ .Values.priorityClassName }} {{ toYaml . | indent 8 }} - {{- end }} \ No newline at end of file + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} From ac6283aab09c51b5fb40433d7b0b99ac66b84099 Mon Sep 17 00:00:00 2001 From: "Tony S. Wu" Date: Mon, 5 Aug 2019 15:20:59 -0700 Subject: [PATCH 093/119] Add Support for Custom Metadata Tags (#203) --- .../examples/mini_custom_metadata.yaml | 13 +++++++++++++ .../templates/_helpers.tpl | 5 +++++ .../templates/configMap.yaml | 5 +++++ helm-chart/splunk-kubernetes-logging/values.yaml | 6 ++++++ 4 files changed, 29 insertions(+) create mode 100644 helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml diff --git a/helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml b/helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml new file mode 100644 index 00000000..7f446634 --- /dev/null +++ b/helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml @@ -0,0 +1,13 @@ +splunk: + hec: + token: 11111111-2222-3333-4444-555555555555 + host: my.splunk.com + +kubernetes: + clusterName: "my-eks-cluster" + +custom_metadata: + - name: "cloud_account_id" + value: "1234567890" + - name: "cloud_account_region" + value: "us-west-2" diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index d38698aa..5ff39d77 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -112,6 +112,11 @@ def extract_container_info: | .container_name = ($cparts[:-1] | join("-")) | .container_id = ($cparts[-1] | rtrimstr(".log")) | .cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" + {{- if .Values.custom_metadata }} + {{- range .Values.custom_metadata }} + | .{{ .name }} = "{{ .value }}" + {{- end }} + {{- end }} | .; .record | extract_container_info | .sourcetype = (find_sourcetype(.pod; .container_name) // "kube:container:\(.container_name)") diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index c7c89acf..dafcb67e 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -262,6 +262,11 @@ data: {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{- end }} + {{- if .Values.custom_metadata }} + {{- range .Values.custom_metadata }} + {{ .name }} + {{- end }} + {{- end }} {{- with .Values.buffer }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 2c6bfe13..e5d265cf 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -316,3 +316,9 @@ kubernetes: clusterName: # Add privileged access to containers for openshift compatability openshift: false + +# List of key/value pairs for metadata purpse. +# Can be used to define things such as cloud_account_id, cloud_account_region, etc. +custom_metadata: +# - name: "cloud_account_id" +# value: "1234567890" From 142e11584fe1c1ab85d2654ab829dde3894d22c4 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 6 Aug 2019 14:05:42 -0700 Subject: [PATCH 094/119] index routing fix for data integrity (#215) --- README.md | 7 +++++-- .../splunk-kubernetes-logging/templates/_helpers.tpl | 9 +++++---- .../splunk-kubernetes-logging/templates/configMap.yaml | 2 +- helm-chart/splunk-kubernetes-logging/values.yaml | 8 ++++---- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3c70256c..2b451e99 100644 --- a/README.md +++ b/README.md @@ -130,12 +130,15 @@ using the two configurable parameters `indexRouting` and `indexRoutingDefaultInd `indexRoutingDefaultIndex` is the Splunk index used for the events from the default Kubernetes namespace Warning: Before enabling this feature it is essential to have Splunk indexes created which map to your Kubernetes namespaces. -Example: +For example: +Consider the following kubernetes namespace to splunk index topology. * (Namespace) -> (Splunk Index) * kube-system -> kube-system * kube-public -> kube-public -* default -> indexRoutingDefaultIndex +* default -> indexRoutingDefaultIndex +For this topology to work appropriately we have to create the splunk indexes "kube-system", "kube-public" and the value of indexRoutingDefaultIndex. + # Maintenance And Support diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index 5ff39d77..c2405172 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -96,10 +96,10 @@ elif startswith({{ list (or .from.container .name) .from.pod | join "/" | quote else empty end; -def set_namespace(value): +def set_index(value): if value == "default" then -{{- $index := or .Values.splunk.hec.indexRoutingDefaultIndex .Values.global.splunk.hec.indexRoutingDefaultIndex | default "main" | quote}} +{{- $index := or .Values.splunk.hec.indexRoutingDefaultIndex .Values.global.splunk.hec.indexRoutingDefaultIndex | default "main" | quote }} {{- printf " %s" $index -}} else value end; @@ -108,7 +108,8 @@ def extract_container_info: (.source | ltrimstr("/var/log/containers/") | split("_")) as $parts | ($parts[-1] | split("-")) as $cparts | .pod = $parts[0] - | .namespace = set_namespace($parts[1]) + | .namespace = $parts[1] + | .index = set_index($parts[1]) | .container_name = ($cparts[:-1] | join("-")) | .container_id = ($cparts[-1] | rtrimstr(".log")) | .cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" @@ -131,4 +132,4 @@ Create the name of the service account to use {{- else -}} {{ default "default" .Values.serviceAccount.name }} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index dafcb67e..e16ea4c9 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -229,7 +229,7 @@ data: source_key source sourcetype_key sourcetype {{- if or .Values.splunk.hec.indexRouting .Values.global.splunk.hec.indexRouting }} - index_key namespace + index_key index {{- else }} {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} index {{ . }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index e5d265cf..673f29ce 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -16,7 +16,7 @@ global: insecureSSL: false # indexRouting is a boolean, it indicates whether user wants to route logs to the index with name same as the namespace. Default is false. indexRouting: - # indexRoutingDefaultIndex tells which index to use for the default kubenetes namespace. Used with indexRouting. Default is main. + # indexRoutingDefaultIndex tells which index to use for the default kubernetes namespace. Used with indexRouting. This value is required when using indexRouting. Default is main if indexRoutingDefaultIndex is not specified. indexRoutingDefaultIndex: kubernetes: clusterName: "cluster_name" @@ -29,7 +29,7 @@ global: path: /var/log/containers/*.log # paths of logfiles to exclude. object type is array as per fluentd specification: # https://docs.fluentd.org/input/tail#exclude_path - exclude_path: + exclude_path: # - /var/log/containers/kube-svc-redirect*.log # - /var/log/containers/tiller*.log @@ -92,7 +92,7 @@ splunk: caFile: # indexRouting is a boolean, it indicates whether user wants to route logs to the index with name same as the namespace. Default is false. indexRouting: - # indexRoutingDefaultIndex tells which index to use for the default kubenetes namespace. Used with indexRouting. Default is main. + # indexRoutingDefaultIndex tells which index to use for the default kubernetes namespace. Used with indexRouting. This value is required when using indexRouting. indexRoutingDefaultIndex: @@ -307,7 +307,7 @@ affinity: {} # Defines priorityClassName to assign a priority class to pods. -priorityClassName: +priorityClassName: # = Kubernetes Connection Configs = From 75c81a585cbf29644d1b666b9ab24cf137f2f5c7 Mon Sep 17 00:00:00 2001 From: abbas-splunk <52936563+abbas-splunk@users.noreply.github.com> Date: Tue, 6 Aug 2019 15:22:08 -0700 Subject: [PATCH 095/119] Delete LICENSE.md This is an older pre-release license and since SCK is a released product under Apache 2.0 license, this pre-release license no longer applies, thus deleting. --- .../splunk-connect-for-kubernetes/LICENSE.md | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 helm-chart/splunk-connect-for-kubernetes/LICENSE.md diff --git a/helm-chart/splunk-connect-for-kubernetes/LICENSE.md b/helm-chart/splunk-connect-for-kubernetes/LICENSE.md deleted file mode 100644 index b4b506d2..00000000 --- a/helm-chart/splunk-connect-for-kubernetes/LICENSE.md +++ /dev/null @@ -1,53 +0,0 @@ -[SPLUNK PRE-RELEASE SOFTWARE LICENSE AGREEMENT](https://www.splunk.com/en_us/legal/splunk-pre-release-software-license-agreement.html) - - - -THIS PRE-RELEASE SOFTWARE LICENSE AGREEMENT (THE "AGREEMENT") GOVERNS YOUR USE OF THE PRE-RELEASE SOFTWARE (DEFINED BELOW) PROVIDED BY SPLUNK INC., AND/OR ITS AFFILIATES ("SPLUNK”). YOU WILL BE REQUIRED TO INDICATE YOUR AGREEMENT TO THIS AGREEMENT IN ORDER TO ACCESS OR DOWNLOAD THE PRE-RELEASE SOFTWARE OR TO COMPLETE THE INSTALLATION PROCESS FOR THE PRE-RELEASE SOFTWARE. BY CLICKING ON THE "YES" OR “I ACCEPT” BUTTON (OR OTHER BUTTON OR MECHANISM DESIGNED TO ACKNOWLEDGE AGREEMENT TO THE TERMS OF THIS AGREEMENT), OR BY DOWNLOADING, ACCESSING OR INSTALLING THE PRE-RELEASE SOFTWARE, YOU ARE CONSENTING TO BE BOUND BY THIS AGREEMENT. - -NOTE: THIS IS THE ONLY AGREEMENT BETWEEN YOU AND SPLUNK GOVERNING YOUR USE OF THE PRE-RELEASE SOFTWARE, AND THIS AGREEMENT SHALL SUPERSEDE ANY OTHER TERMS OF USE OR LICENSE AGREEMENT THAT MAY APPEAR DURING THE INSTALLATION OR DOWNLOADING OF THE PRE-RELEASE SOFTWARE. - -IF YOU AGREE TO THESE TERMS ON BEHALF OF A BUSINESS, YOU REPRESENT AND WARRANT THAT YOU HAVE THE POWER AND AUTHORITY TO BIND THAT BUSINESS TO THIS AGREEMENT, AND YOUR AGREEMENT TO THESE TERMS WILL BE TREATED AS THE AGREEMENT OF THE BUSINESS. IN THAT EVENT, "YOU" AND "YOUR" REFER HEREIN TO THAT BUSINESS. - -IF YOU ARE UNWILLING TO AGREE TO THIS AGREEMENT, OR YOU DO NOT HAVE THE RIGHT, POWER AND AUTHORITY TO ACT ON BEHALF OF AND BIND YOUR BUSINESS, DO NOT CLICK ON THE BUTTON AND DO NOT INSTALL, DOWNLOAD, ACCESS, OR OTHERWISE USE THE PRE-RELEASE SOFTWARE AND CANCEL THE LOADING OF THE PRE-RELEASE SOFTWARE. - -1. DEFINITIONS. - - “Pre-Release Software” means the pre-release version of the Splunk product, service, technology identified on the software download page or landing page or invitation message (the “Cover Page”), whether labeled as alpha, beta, pre-release, preview or otherwise, provided to you by Splunk under this Agreement. Pre-Release Software may include any enhancements, updates, upgrades, derivatives or bug fixes to such product, service or technology and any documentation, add-ons, templates, sample data sets or hardware devices as provided by Splunk. - - “Data” means the raw data you upload or submit to the Pre-Release Software and the processed result of the raw data generated by you using the Pre-Release Software. - - “Feedback” means all suggestions, comments, opinions, code, input, ideas, reports, information, know-how or other feedback provided by you (whether in oral, electronic or written form) to Splunk in connection with your use of the Pre-Release Software. Feedback does not include Data, unless submitted or communicated by you to Splunk as part of Feedback. - - “Internal Purposes” means internal business use with your systems, networks, devices and data for the purposes of internal testing and evaluation of the Pre-Release Software in order to provide Feedback to Splunk regarding the Pre-Release Software. Such use does not include use of your systems, networks or devices as part of services you provide for a third party's benefit. - -2. PRE-RELEASE SOFTWARE LICENSE. Subject to your compliance with the terms and conditions of this Agreement, Splunk grants you a non-exclusive, non-sublicensable, nontransferable, revocable, limited license during the term of the Agreement to use a single copy of the Pre-Release Software at your principal office in a secure location, only in connection with and solely for the Internal Purposes. - -3. LICENSE RESTRICTIONS. Except as expressly authorized in this Agreement or by Splunk, you will not, and will not permit any third party to: (i) access or use the Pre-Release Software for any other purposes than the Internal Purposes (including for any competitive analysis, commercial, professional, or other for-profit purposes); (ii) copy the Pre-Release Software (except as required to run the Pre-Release Software and for reasonable backup purposes); (iii) modify, adapt, or create derivative works of the Pre-Release Software; (iv) rent, lease, loan, resell, transfer, sublicense or distribute the Pre-Release Software to any third party; (v) use or offer any functionality of the Pre-Release Software on a service provider, service bureau, hosted, software as a service, or time sharing basis; (vi) decompile, disassemble or reverse-engineer the Pre-Release Software or otherwise attempt to derive the Pre-Release Software source code, algorithms, methods or techniques used or embodied in the Pre-Release Software; (vii) disclose to any third party the results of any benchmark tests or other evaluation of the Pre-Release Software, or (viii) remove, alter, obscure, cover or change any trademark, copyright or other proprietary notices, labels or markings from or on the Pre-Release Software; (ix) interfere with or disrupt servers or networks connected to any website through which the Pre-Release Software provided; or (x) use the Pre-Release Software to collect or store personal data about any person or entity. Any consultant, contractor, or agent hired to perform services for you may operate the Pre-Release Software on your behalf under these terms and conditions, provided that: (a) you are responsible for ensuring that any such third party agrees to abide by and fully comply with the terms of this Agreement on the same basis as applicable to you; (b) such use is only in connection with your Internal Purposes; (c) such use does not represent or constitute an increase in the scope of the licenses provided hereunder; and (d) you remain fully liable for any and all acts or omissions by such third parties related to this Agreement. Any violation of this Section shall be a material breach of this Agreement subject to immediate termination of this Agreement for which no notice from Splunk shall be required. - -4. CONFIDENTIALITY. You agree to hold the Pre-Release Software (including all intellectual property rights therein, such as any patents, inventions, copyrights, design rights, trade secrets and know-how) and any related information (“Confidential Information”), whether in oral or written form, confidential. Confidential Information may include information relating to features, functionalities, improvements, code, pricing, business strategies, product roadmaps, development plans, marketing materials, data sets, customer lists or other proprietary third-party information. You will hold such Confidential Information in strict confidence and not use or disclose the Confidential Information, in whole or in part, except as expressly permitted in this Agreement. You may disclose Confidential Information to your employees, but only to the extent they have a need to know to test the Pre-Release Software and you have advised them that such information is Confidential. You agree to instruct any such employees in advance who will have access to the Pre-Release Software that they must comply with the restrictions set forth in this Agreement. You shall have no obligation to maintain the confidentiality of any information which: (a) is or becomes publicly available without breach of this Agreement; (b) is rightfully received by you from a third party without an obligation of confidentiality and without breach of this Agreement; (c) is developed independently by you without access to or use of the Confidential Information; or (d) has been approved for release by written authorization of the party that owns the Confidential Information. You will notify immediately upon discovery of any unauthorized use or disclosure of Confidential Information or any other breach of this Agreement by you or your personnel, and will cooperate with Splunk in every reasonable way to help Splunk regain possession of the Confidential Information and prevent its further unauthorized use or disclosure. You acknowledge that any breach of its obligations under this Agreement with respect to the proprietary rights or Confidential Information will cause Splunk irreparable injury for which there are inadequate remedies at law, and therefore, Splunk will be entitled to equitable relief in addition to all other remedies provided by this Agreement or available at law or in equity. - -5. TERM AND TERMINATION. This Agreement will be effective from the earlier of (a) the date it is accepted by you and (b) the date on which you first installed, downloaded or accessed a copy of the Pre-Release Software and shall continue until terminated. This Agreement may be terminated at any time by either party, with or without cause, effective upon notice of termination. This Agreement will terminate automatically upon the end of the Pre-Release Software project as identified on the Cover Page or upon commercial release (if any) of the Pre-Release Software, whichever is earlier. This is time-sensitive software, so it will stop functioning on the termination date. Upon termination, you will immediately cease all use of the Pre-Release Software and destroy the Pre-Release Software, or upon request by Splunk, return to Splunk the Pre-Release Software and other Confidential Information that are in your possession or control. Upon Splunk’s request, you will certify in writing that you have returned or destroyed all copies of the Pre-Release Software and Confidential Information. Sections 1, 3, 4, 6, 7, 8, 9, 10, 11, 15, and 16 will survive termination of this Agreement. - -6. OWNERSHIP. Splunk, its suppliers and/or its licensors own all worldwide right, title and interest in and to the Pre-Release Software, including all worldwide patent rights (including patent applications and disclosures); copyright rights (including copyrights, copyright registration and copy rights with respect to computer software, software design, software code, software architecture, firmware, programming tools, graphic user interfaces, reports, dashboard, business rules, use cases, screens, alerts, notifications, drawings, specifications and databases); moral rights; trade secrets and other rights with respect to confidential or proprietary information; know-how; other rights with respect to inventions, discoveries, ideas, improvements, techniques, formulae, algorithms, processes, schematics, testing procedures, technical information and other technology; and any other intellectual and industrial property rights, whether or not subject to registration or protection; and all rights under any license or other arrangement with respect to the foregoing. Except as expressly stated in this Agreement, Splunk does not grant you any intellectual property rights in the Pre-Release Software, and all right, title, and interest in and to all copies of the Pre-Release Software not expressly granted remain with Splunk, its suppliers and/or its licensors. The Pre-Release Software is copyrighted and protected by the laws of the United States and other countries, and international treaty provisions. You acknowledge that the Pre-Release Software is licensed and not sold. - -7. FEEDBACK. Splunk, in its sole discretion, may or may not respond to your Feedback or promise to address all your Feedback in the development of future features or functionalities of the Pre-Release Software or any related or subsequent versions of such Pre-Release Software. In the event Splunk uses your Feedback, you grant Splunk an unrestricted, perpetual, worldwide, exclusive, transferable, irrevocable, sublicensable, royalty-free, fully paid-up license to use, copy, modify, create derivative works of, make, have made, distribute (through multiple tiers of distribution), publicly perform or display, import, export, sell, offer to sell, rent, or license copies of the Feedback as part of or in connection with any Splunk product, service, technology, content, material, specification or documentation. You warrant that the Feedback does not infringe any copyright or trade secret of any third party, and that you have no knowledge of any patent of any third party that may be infringed by the Feedback (including any implementation thereof recommended by you). You further warrant that your Feedback is not subject to any license terms that would purport to require Splunk to comply with any additional obligations with respect to any Splunk product or service that incorporates your Feedback. - -8. DATA. You hereby grant Splunk a perpetual, irrevocable, non-exclusive, royalty-free, paid-up, worldwide, sublicensable license to use, access, transmit, host, store, and display the Data solely for the purpose of providing and improving the Pre-Release Software. Splunk (or its sublicensees) may exercise such license for purposes of providing, maintaining, repairing, administering and improving the Pre-Release Software or in developing new products or services, including rights to extract, compile, aggregate, synthesize, use, and otherwise analyze all or any portion of the Data. You represent, warrant and agree that the Data and other materials you provide or make available to Splunk will include only information relevant to the Pre-Release Software and the use thereof and will not include any personally identifiable information or any protected health data. You acknowledge and agree that you are solely responsible for all Data you upload or submit using the Pre-Release Software and for your conduct while using the Pre-Release Software. You acknowledge and agree that: (a) you will evaluate and bear all risks associated with the use of any Data; (b) you are responsible for protecting and backing up the Data; (c) you are responsible for protecting the confidentiality of any Data; and (d) under no circumstances will Splunk be liable in any way for any Data, including, but not limited to, any errors or omissions in any Data, or any loss or damages or any kind incurred as a result of your use, deletion, modification, or correction of any Data. Splunk has no responsibility to store, protect, remove or delete any Data for you and shall have no liability for the deletion of or failure to store any Data. - -9. WARRANTY DISCLAIMER. THE PRE-RELEASE SOFTWARE IS PROVIDED “AS IS”. SPLUNK DISCLAIMS ANY AND ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, QUIET ENJOYMENT AND WARRANTIES ARISING OUT OF COURSE OF DEALING OR USAGE OF TRADE OR BY STATUTE OR IN LAW. SPLUNK SPECIFICALLY DOES NOT WARRANT THAT THE PRE-RELEASE SOFTWARE WILL MEET YOUR REQUIREMENTS, THE OPERATION OR OUTPUT OF THE PRE-RELEASE SOFTWARE WILL BE ERROR-FREE, ACCURATE, RELIABLE, COMPLETE OR UNINTERRUPTED. Splunk is not obligated to support, update or upgrade the Pre-Release Software. - -10. NO RELIANCE. Splunk has no obligations about any forward-looking statements made in connection with or in the course of providing the Pre-Release Software. Forward-looking statements are statements regarding future Splunk events, product offerings, product performance, customer uses or the expected financial performance of Splunk. Any such statements reflect current expectations and estimates based on factors currently known and that actual events or results could differ materially. Splunk does not assume any obligation to update any forward-looking statements made during the Pre-Release Software project. In addition, any information about our roadmap outlines our general product direction and is subject to change at any time without notice. It is for informational purposes only and shall not be incorporated into this Agreement or any contract or other commitment. Splunk undertakes no obligation either to develop the features or functionality described in the forward-looking statement or to include any such feature or functionality in a future release, including those you are reviewing as a part of this Pre-Release Software. You expressly acknowledge that the Pre-Release Software has not been fully tested and may contain defects or deficiencies which may not be corrected by Splunk, that the Pre-Release Software may undergo significant changes prior to release of the corresponding generally available final version. - -11. LIMITATION OF LIABILITY. IN NO EVENT WILL SPLUNK BE LIABLE TO YOU FOR ANY SPECIAL, INDIRECT, INCIDENTAL, CONSEQUENTIAL OR PUNITIVE DAMAGES (INCLUDING LOSS OF USE, DATA, OR PROFITS, BUSINESS INTERRUPTION, OR COSTS OF PROCURING SUBSTITUTE PRE-RELEASE SOFTWARE) ARISING OUT OF OR IN CONNECTION WITH THIS AGREEMENT OR THE USE OR PERFORMANCE OF THE PRE-RELEASE SOFTWARE, WHETHER SUCH LIABILITY ARISES FROM CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, AND WHETHER OR NOT SPLUNK HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH LOSS OR DAMAGE. THE PARTIES HAVE AGREED THAT THESE LIMITATIONS WILL SURVIVE AND APPLY EVEN IF ANY REMEDY IS FOUND TO HAVE FAILED OF ITS ESSENTIAL PURPOSE. WITHOUT LIMITING THE FOREGOING, SPLUNK WILL HAVE NO LIABILITY OR RESPONSIBILITY FOR ANY BUSINESS INTERRUPTION OR LOSS OF DATA ARISING FROM THE AUTOMATIC TERMINATION OF THE LICENSE RIGHTS GRANTED HEREIN AND ANY ASSOCIATED CESSATION OF THE PRE-RELEASE SOFTWARE FUNCTIONS OR ANY UNANTICIPATED OR UNSCHEDULED DOWNTIME FOR ANY REASON OR ANY DELETION, CORRUPTION OR DAMAGE OF DATA ON OR THROUGH THE PRE-RELEASE SOFTWARE. SPLUNK'S TOTAL CUMULATIVE LIABILITY TO YOU, FROM ALL CAUSES OF ACTION AND ALL THEORIES OF LIABILITY, WILL BE LIMITED TO AND WILL NOT EXCEED ONE HUNDRED DOLLARS ($100.00). BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU. - -12. PRIVACY. You acknowledge that Splunk and its providers may obtain information and data from you in connection with your registration, installation, and use of the Pre-Release Software, including, without limitation, personal information. Splunk may also collect and process technical and related environmental or performance information about your use of the Pre-Release Software (which may include, without limitation, data ingest volume, search concurrency, number of unique user log-ins, Internet protocol addresses, operating system, configuration information, application software, session duration, page view, operational use, and other such information) and use this information to support and troubleshoot issues, provide updates, analyze trends and improve Splunk’s products or services. You hereby consent to Splunk and its providers maintaining, using, storing and disclosing such information and data (including, without limitation, personal information, if any, in conformity with [Splunk Privacy Policy](https://www.splunk.com/view/SP-CAAAAAG), which may be updated from time to time). You further consent to receiving electronic communications and notifications from Splunk in connection with your use of the Pre-Release Software. - -13. EXPORT. You will comply fully with all relevant export laws and regulations of the United States and any other country (“Export Laws”) where you use any of the Pre-Release Software. You represent and warrant that you are not (a) located in, or a resident or a national of, a restricted country; or (b) on any of the relevant U.S. Government Lists of prohibited persons, including but not limited to the Treasury Department’s List of Specially Designated Nationals and the Commerce Department’s List of Denied Persons or Entity List. You further represent and warrant that you shall not export, re-export, ship, transfer the Pre-Release Software to any restricted countries or restricted end users or use the Pre-Release Software in any restricted countries or for any purposes prohibited by the Export Laws, including, but not limited to, nuclear, chemical, missile or biological weapons related end uses. You understand that the requirements and restrictions of the Export Laws may vary depending on the specific Pre-Release Software and may change over time, and that, to determine the precise controls applicable to the Pre-Release Software, it is necessary to refer to the U.S. Export Administration Regulations and the U.S. Department of Treasury, Office of Foreign Assets Control Regulations. - -14. GOVERNMENT END USER RIGHTS. You acknowledge that all Pre-Release Software and Confidential Information were developed entirely at private expense and that no part of the Pre-Release Software or Confidential Information was first produced in the performance of a Government contract. You agree that the Pre-Release Software and any derivatives thereof are "commercial items" as defined in 48 C.F.R. § 2.101, and if you are a U.S. Government agency or instrumentality or if you are providing all or any part of the Pre-Release Software or any derivatives thereof to the U.S. Government, such use, duplication, reproduction, release, modification, disclosure or transfer of this commercial product and data, is restricted in accordance with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-2, and 48 C.F.R. § 227.7202, as applicable. Consistent with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-1 through 48 C.F.R. § 227.7102-3, and 48 C.F.R. §§ 227.7202-1 through 227.7202-4, as applicable, the Pre-Release Software is licensed to U.S. Government end users (i) only as Commercial Items and (ii) with only those rights as are granted to all other users pursuant to this Agreement and any related agreement(s), as applicable. Accordingly, you will have no rights in the Pre-Release Software except as expressly agreed to in writing by you and Splunk. - -15. CHOICE OF LAW AND DISPUTES. For other than the U.S. Government as a party, this Agreement shall be governed by and construed in accordance with the laws of the State of California, as if performed wholly within the state and without giving effect to the principles of conflict of law rules of any jurisdiction or the United Nations Convention on Contracts for the International Sale of Goods, the application of which is expressly excluded. Any legal action or proceeding arising under this Agreement will be brought exclusively in the federal or state courts located in San Francisco, California and the parties hereby consent to personal jurisdiction and venue therein. - -16. GENERAL. All notices required or permitted under this Agreement hereto will be in writing and delivered in person, by confirmed facsimile transmission, by overnight delivery service, or by registered or certified mail, postage prepaid with return receipt requested, and in each instance will be deemed given upon receipt. You may not assign, delegate or transfer this Agreement, in whole or in part, by agreement, operation of law or otherwise. You acknowledge that Splunk may assign, subcontract or delegate any of its rights or obligations under this Agreement. Any attempt to assign this Agreement other than as permitted herein shall be null and void. Subject to the foregoing, this Agreement will bind and inure to the benefit of the parties’ permitted successors and assigns. This Agreement along with any additional terms incorporated herein by reference constitute the complete and exclusive understanding and agreement between the parties relating only to the subject matter of the Pre-Release Software and Confidential Information and shall supersede any and all prior or contemporaneous agreements, communications and understandings, written or oral, relating to such subject matter. This Agreement is limited to the use of Pre-Release Software and Confidential Information and as such, this Agreement is separate from and shall have no effect on any other agreement you may have with Splunk. Any waiver, modification or amendment of any provision of this Agreement will be effective only if in writing and signed by duly authorized representatives of both parties. All rights and remedies, whether conferred hereunder or by any other instrument or law, will be cumulative and may be exercised singularly or concurrently. The failure by either party to enforce any provisions of this Agreement will not constitute a waiver of any other right hereunder or of any subsequent enforcement of that or any other provisions. The terms and conditions stated herein are declared to be severable. If a court of competent jurisdiction holds any provision of this Agreement invalid or unenforceable, the remaining provisions of the Agreement will remain in full force and effect, and the provision affected will be construed so as to be enforceable to the maximum extent permissible by law. - -Updated February 16, 2017 From 1ed3c0861db4470eb06aec670629f65a0063655a Mon Sep 17 00:00:00 2001 From: abbas-splunk <52936563+abbas-splunk@users.noreply.github.com> Date: Tue, 6 Aug 2019 15:23:22 -0700 Subject: [PATCH 096/119] Delete LICENSE.md This is an older license and since SCK has been released under Apache 2.0 license this stale pre-release license no longer applies --- .../splunk-kubernetes-logging/LICENSE.md | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 helm-chart/splunk-kubernetes-logging/LICENSE.md diff --git a/helm-chart/splunk-kubernetes-logging/LICENSE.md b/helm-chart/splunk-kubernetes-logging/LICENSE.md deleted file mode 100644 index b4b506d2..00000000 --- a/helm-chart/splunk-kubernetes-logging/LICENSE.md +++ /dev/null @@ -1,53 +0,0 @@ -[SPLUNK PRE-RELEASE SOFTWARE LICENSE AGREEMENT](https://www.splunk.com/en_us/legal/splunk-pre-release-software-license-agreement.html) - - - -THIS PRE-RELEASE SOFTWARE LICENSE AGREEMENT (THE "AGREEMENT") GOVERNS YOUR USE OF THE PRE-RELEASE SOFTWARE (DEFINED BELOW) PROVIDED BY SPLUNK INC., AND/OR ITS AFFILIATES ("SPLUNK”). YOU WILL BE REQUIRED TO INDICATE YOUR AGREEMENT TO THIS AGREEMENT IN ORDER TO ACCESS OR DOWNLOAD THE PRE-RELEASE SOFTWARE OR TO COMPLETE THE INSTALLATION PROCESS FOR THE PRE-RELEASE SOFTWARE. BY CLICKING ON THE "YES" OR “I ACCEPT” BUTTON (OR OTHER BUTTON OR MECHANISM DESIGNED TO ACKNOWLEDGE AGREEMENT TO THE TERMS OF THIS AGREEMENT), OR BY DOWNLOADING, ACCESSING OR INSTALLING THE PRE-RELEASE SOFTWARE, YOU ARE CONSENTING TO BE BOUND BY THIS AGREEMENT. - -NOTE: THIS IS THE ONLY AGREEMENT BETWEEN YOU AND SPLUNK GOVERNING YOUR USE OF THE PRE-RELEASE SOFTWARE, AND THIS AGREEMENT SHALL SUPERSEDE ANY OTHER TERMS OF USE OR LICENSE AGREEMENT THAT MAY APPEAR DURING THE INSTALLATION OR DOWNLOADING OF THE PRE-RELEASE SOFTWARE. - -IF YOU AGREE TO THESE TERMS ON BEHALF OF A BUSINESS, YOU REPRESENT AND WARRANT THAT YOU HAVE THE POWER AND AUTHORITY TO BIND THAT BUSINESS TO THIS AGREEMENT, AND YOUR AGREEMENT TO THESE TERMS WILL BE TREATED AS THE AGREEMENT OF THE BUSINESS. IN THAT EVENT, "YOU" AND "YOUR" REFER HEREIN TO THAT BUSINESS. - -IF YOU ARE UNWILLING TO AGREE TO THIS AGREEMENT, OR YOU DO NOT HAVE THE RIGHT, POWER AND AUTHORITY TO ACT ON BEHALF OF AND BIND YOUR BUSINESS, DO NOT CLICK ON THE BUTTON AND DO NOT INSTALL, DOWNLOAD, ACCESS, OR OTHERWISE USE THE PRE-RELEASE SOFTWARE AND CANCEL THE LOADING OF THE PRE-RELEASE SOFTWARE. - -1. DEFINITIONS. - - “Pre-Release Software” means the pre-release version of the Splunk product, service, technology identified on the software download page or landing page or invitation message (the “Cover Page”), whether labeled as alpha, beta, pre-release, preview or otherwise, provided to you by Splunk under this Agreement. Pre-Release Software may include any enhancements, updates, upgrades, derivatives or bug fixes to such product, service or technology and any documentation, add-ons, templates, sample data sets or hardware devices as provided by Splunk. - - “Data” means the raw data you upload or submit to the Pre-Release Software and the processed result of the raw data generated by you using the Pre-Release Software. - - “Feedback” means all suggestions, comments, opinions, code, input, ideas, reports, information, know-how or other feedback provided by you (whether in oral, electronic or written form) to Splunk in connection with your use of the Pre-Release Software. Feedback does not include Data, unless submitted or communicated by you to Splunk as part of Feedback. - - “Internal Purposes” means internal business use with your systems, networks, devices and data for the purposes of internal testing and evaluation of the Pre-Release Software in order to provide Feedback to Splunk regarding the Pre-Release Software. Such use does not include use of your systems, networks or devices as part of services you provide for a third party's benefit. - -2. PRE-RELEASE SOFTWARE LICENSE. Subject to your compliance with the terms and conditions of this Agreement, Splunk grants you a non-exclusive, non-sublicensable, nontransferable, revocable, limited license during the term of the Agreement to use a single copy of the Pre-Release Software at your principal office in a secure location, only in connection with and solely for the Internal Purposes. - -3. LICENSE RESTRICTIONS. Except as expressly authorized in this Agreement or by Splunk, you will not, and will not permit any third party to: (i) access or use the Pre-Release Software for any other purposes than the Internal Purposes (including for any competitive analysis, commercial, professional, or other for-profit purposes); (ii) copy the Pre-Release Software (except as required to run the Pre-Release Software and for reasonable backup purposes); (iii) modify, adapt, or create derivative works of the Pre-Release Software; (iv) rent, lease, loan, resell, transfer, sublicense or distribute the Pre-Release Software to any third party; (v) use or offer any functionality of the Pre-Release Software on a service provider, service bureau, hosted, software as a service, or time sharing basis; (vi) decompile, disassemble or reverse-engineer the Pre-Release Software or otherwise attempt to derive the Pre-Release Software source code, algorithms, methods or techniques used or embodied in the Pre-Release Software; (vii) disclose to any third party the results of any benchmark tests or other evaluation of the Pre-Release Software, or (viii) remove, alter, obscure, cover or change any trademark, copyright or other proprietary notices, labels or markings from or on the Pre-Release Software; (ix) interfere with or disrupt servers or networks connected to any website through which the Pre-Release Software provided; or (x) use the Pre-Release Software to collect or store personal data about any person or entity. Any consultant, contractor, or agent hired to perform services for you may operate the Pre-Release Software on your behalf under these terms and conditions, provided that: (a) you are responsible for ensuring that any such third party agrees to abide by and fully comply with the terms of this Agreement on the same basis as applicable to you; (b) such use is only in connection with your Internal Purposes; (c) such use does not represent or constitute an increase in the scope of the licenses provided hereunder; and (d) you remain fully liable for any and all acts or omissions by such third parties related to this Agreement. Any violation of this Section shall be a material breach of this Agreement subject to immediate termination of this Agreement for which no notice from Splunk shall be required. - -4. CONFIDENTIALITY. You agree to hold the Pre-Release Software (including all intellectual property rights therein, such as any patents, inventions, copyrights, design rights, trade secrets and know-how) and any related information (“Confidential Information”), whether in oral or written form, confidential. Confidential Information may include information relating to features, functionalities, improvements, code, pricing, business strategies, product roadmaps, development plans, marketing materials, data sets, customer lists or other proprietary third-party information. You will hold such Confidential Information in strict confidence and not use or disclose the Confidential Information, in whole or in part, except as expressly permitted in this Agreement. You may disclose Confidential Information to your employees, but only to the extent they have a need to know to test the Pre-Release Software and you have advised them that such information is Confidential. You agree to instruct any such employees in advance who will have access to the Pre-Release Software that they must comply with the restrictions set forth in this Agreement. You shall have no obligation to maintain the confidentiality of any information which: (a) is or becomes publicly available without breach of this Agreement; (b) is rightfully received by you from a third party without an obligation of confidentiality and without breach of this Agreement; (c) is developed independently by you without access to or use of the Confidential Information; or (d) has been approved for release by written authorization of the party that owns the Confidential Information. You will notify immediately upon discovery of any unauthorized use or disclosure of Confidential Information or any other breach of this Agreement by you or your personnel, and will cooperate with Splunk in every reasonable way to help Splunk regain possession of the Confidential Information and prevent its further unauthorized use or disclosure. You acknowledge that any breach of its obligations under this Agreement with respect to the proprietary rights or Confidential Information will cause Splunk irreparable injury for which there are inadequate remedies at law, and therefore, Splunk will be entitled to equitable relief in addition to all other remedies provided by this Agreement or available at law or in equity. - -5. TERM AND TERMINATION. This Agreement will be effective from the earlier of (a) the date it is accepted by you and (b) the date on which you first installed, downloaded or accessed a copy of the Pre-Release Software and shall continue until terminated. This Agreement may be terminated at any time by either party, with or without cause, effective upon notice of termination. This Agreement will terminate automatically upon the end of the Pre-Release Software project as identified on the Cover Page or upon commercial release (if any) of the Pre-Release Software, whichever is earlier. This is time-sensitive software, so it will stop functioning on the termination date. Upon termination, you will immediately cease all use of the Pre-Release Software and destroy the Pre-Release Software, or upon request by Splunk, return to Splunk the Pre-Release Software and other Confidential Information that are in your possession or control. Upon Splunk’s request, you will certify in writing that you have returned or destroyed all copies of the Pre-Release Software and Confidential Information. Sections 1, 3, 4, 6, 7, 8, 9, 10, 11, 15, and 16 will survive termination of this Agreement. - -6. OWNERSHIP. Splunk, its suppliers and/or its licensors own all worldwide right, title and interest in and to the Pre-Release Software, including all worldwide patent rights (including patent applications and disclosures); copyright rights (including copyrights, copyright registration and copy rights with respect to computer software, software design, software code, software architecture, firmware, programming tools, graphic user interfaces, reports, dashboard, business rules, use cases, screens, alerts, notifications, drawings, specifications and databases); moral rights; trade secrets and other rights with respect to confidential or proprietary information; know-how; other rights with respect to inventions, discoveries, ideas, improvements, techniques, formulae, algorithms, processes, schematics, testing procedures, technical information and other technology; and any other intellectual and industrial property rights, whether or not subject to registration or protection; and all rights under any license or other arrangement with respect to the foregoing. Except as expressly stated in this Agreement, Splunk does not grant you any intellectual property rights in the Pre-Release Software, and all right, title, and interest in and to all copies of the Pre-Release Software not expressly granted remain with Splunk, its suppliers and/or its licensors. The Pre-Release Software is copyrighted and protected by the laws of the United States and other countries, and international treaty provisions. You acknowledge that the Pre-Release Software is licensed and not sold. - -7. FEEDBACK. Splunk, in its sole discretion, may or may not respond to your Feedback or promise to address all your Feedback in the development of future features or functionalities of the Pre-Release Software or any related or subsequent versions of such Pre-Release Software. In the event Splunk uses your Feedback, you grant Splunk an unrestricted, perpetual, worldwide, exclusive, transferable, irrevocable, sublicensable, royalty-free, fully paid-up license to use, copy, modify, create derivative works of, make, have made, distribute (through multiple tiers of distribution), publicly perform or display, import, export, sell, offer to sell, rent, or license copies of the Feedback as part of or in connection with any Splunk product, service, technology, content, material, specification or documentation. You warrant that the Feedback does not infringe any copyright or trade secret of any third party, and that you have no knowledge of any patent of any third party that may be infringed by the Feedback (including any implementation thereof recommended by you). You further warrant that your Feedback is not subject to any license terms that would purport to require Splunk to comply with any additional obligations with respect to any Splunk product or service that incorporates your Feedback. - -8. DATA. You hereby grant Splunk a perpetual, irrevocable, non-exclusive, royalty-free, paid-up, worldwide, sublicensable license to use, access, transmit, host, store, and display the Data solely for the purpose of providing and improving the Pre-Release Software. Splunk (or its sublicensees) may exercise such license for purposes of providing, maintaining, repairing, administering and improving the Pre-Release Software or in developing new products or services, including rights to extract, compile, aggregate, synthesize, use, and otherwise analyze all or any portion of the Data. You represent, warrant and agree that the Data and other materials you provide or make available to Splunk will include only information relevant to the Pre-Release Software and the use thereof and will not include any personally identifiable information or any protected health data. You acknowledge and agree that you are solely responsible for all Data you upload or submit using the Pre-Release Software and for your conduct while using the Pre-Release Software. You acknowledge and agree that: (a) you will evaluate and bear all risks associated with the use of any Data; (b) you are responsible for protecting and backing up the Data; (c) you are responsible for protecting the confidentiality of any Data; and (d) under no circumstances will Splunk be liable in any way for any Data, including, but not limited to, any errors or omissions in any Data, or any loss or damages or any kind incurred as a result of your use, deletion, modification, or correction of any Data. Splunk has no responsibility to store, protect, remove or delete any Data for you and shall have no liability for the deletion of or failure to store any Data. - -9. WARRANTY DISCLAIMER. THE PRE-RELEASE SOFTWARE IS PROVIDED “AS IS”. SPLUNK DISCLAIMS ANY AND ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, QUIET ENJOYMENT AND WARRANTIES ARISING OUT OF COURSE OF DEALING OR USAGE OF TRADE OR BY STATUTE OR IN LAW. SPLUNK SPECIFICALLY DOES NOT WARRANT THAT THE PRE-RELEASE SOFTWARE WILL MEET YOUR REQUIREMENTS, THE OPERATION OR OUTPUT OF THE PRE-RELEASE SOFTWARE WILL BE ERROR-FREE, ACCURATE, RELIABLE, COMPLETE OR UNINTERRUPTED. Splunk is not obligated to support, update or upgrade the Pre-Release Software. - -10. NO RELIANCE. Splunk has no obligations about any forward-looking statements made in connection with or in the course of providing the Pre-Release Software. Forward-looking statements are statements regarding future Splunk events, product offerings, product performance, customer uses or the expected financial performance of Splunk. Any such statements reflect current expectations and estimates based on factors currently known and that actual events or results could differ materially. Splunk does not assume any obligation to update any forward-looking statements made during the Pre-Release Software project. In addition, any information about our roadmap outlines our general product direction and is subject to change at any time without notice. It is for informational purposes only and shall not be incorporated into this Agreement or any contract or other commitment. Splunk undertakes no obligation either to develop the features or functionality described in the forward-looking statement or to include any such feature or functionality in a future release, including those you are reviewing as a part of this Pre-Release Software. You expressly acknowledge that the Pre-Release Software has not been fully tested and may contain defects or deficiencies which may not be corrected by Splunk, that the Pre-Release Software may undergo significant changes prior to release of the corresponding generally available final version. - -11. LIMITATION OF LIABILITY. IN NO EVENT WILL SPLUNK BE LIABLE TO YOU FOR ANY SPECIAL, INDIRECT, INCIDENTAL, CONSEQUENTIAL OR PUNITIVE DAMAGES (INCLUDING LOSS OF USE, DATA, OR PROFITS, BUSINESS INTERRUPTION, OR COSTS OF PROCURING SUBSTITUTE PRE-RELEASE SOFTWARE) ARISING OUT OF OR IN CONNECTION WITH THIS AGREEMENT OR THE USE OR PERFORMANCE OF THE PRE-RELEASE SOFTWARE, WHETHER SUCH LIABILITY ARISES FROM CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, AND WHETHER OR NOT SPLUNK HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH LOSS OR DAMAGE. THE PARTIES HAVE AGREED THAT THESE LIMITATIONS WILL SURVIVE AND APPLY EVEN IF ANY REMEDY IS FOUND TO HAVE FAILED OF ITS ESSENTIAL PURPOSE. WITHOUT LIMITING THE FOREGOING, SPLUNK WILL HAVE NO LIABILITY OR RESPONSIBILITY FOR ANY BUSINESS INTERRUPTION OR LOSS OF DATA ARISING FROM THE AUTOMATIC TERMINATION OF THE LICENSE RIGHTS GRANTED HEREIN AND ANY ASSOCIATED CESSATION OF THE PRE-RELEASE SOFTWARE FUNCTIONS OR ANY UNANTICIPATED OR UNSCHEDULED DOWNTIME FOR ANY REASON OR ANY DELETION, CORRUPTION OR DAMAGE OF DATA ON OR THROUGH THE PRE-RELEASE SOFTWARE. SPLUNK'S TOTAL CUMULATIVE LIABILITY TO YOU, FROM ALL CAUSES OF ACTION AND ALL THEORIES OF LIABILITY, WILL BE LIMITED TO AND WILL NOT EXCEED ONE HUNDRED DOLLARS ($100.00). BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU. - -12. PRIVACY. You acknowledge that Splunk and its providers may obtain information and data from you in connection with your registration, installation, and use of the Pre-Release Software, including, without limitation, personal information. Splunk may also collect and process technical and related environmental or performance information about your use of the Pre-Release Software (which may include, without limitation, data ingest volume, search concurrency, number of unique user log-ins, Internet protocol addresses, operating system, configuration information, application software, session duration, page view, operational use, and other such information) and use this information to support and troubleshoot issues, provide updates, analyze trends and improve Splunk’s products or services. You hereby consent to Splunk and its providers maintaining, using, storing and disclosing such information and data (including, without limitation, personal information, if any, in conformity with [Splunk Privacy Policy](https://www.splunk.com/view/SP-CAAAAAG), which may be updated from time to time). You further consent to receiving electronic communications and notifications from Splunk in connection with your use of the Pre-Release Software. - -13. EXPORT. You will comply fully with all relevant export laws and regulations of the United States and any other country (“Export Laws”) where you use any of the Pre-Release Software. You represent and warrant that you are not (a) located in, or a resident or a national of, a restricted country; or (b) on any of the relevant U.S. Government Lists of prohibited persons, including but not limited to the Treasury Department’s List of Specially Designated Nationals and the Commerce Department’s List of Denied Persons or Entity List. You further represent and warrant that you shall not export, re-export, ship, transfer the Pre-Release Software to any restricted countries or restricted end users or use the Pre-Release Software in any restricted countries or for any purposes prohibited by the Export Laws, including, but not limited to, nuclear, chemical, missile or biological weapons related end uses. You understand that the requirements and restrictions of the Export Laws may vary depending on the specific Pre-Release Software and may change over time, and that, to determine the precise controls applicable to the Pre-Release Software, it is necessary to refer to the U.S. Export Administration Regulations and the U.S. Department of Treasury, Office of Foreign Assets Control Regulations. - -14. GOVERNMENT END USER RIGHTS. You acknowledge that all Pre-Release Software and Confidential Information were developed entirely at private expense and that no part of the Pre-Release Software or Confidential Information was first produced in the performance of a Government contract. You agree that the Pre-Release Software and any derivatives thereof are "commercial items" as defined in 48 C.F.R. § 2.101, and if you are a U.S. Government agency or instrumentality or if you are providing all or any part of the Pre-Release Software or any derivatives thereof to the U.S. Government, such use, duplication, reproduction, release, modification, disclosure or transfer of this commercial product and data, is restricted in accordance with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-2, and 48 C.F.R. § 227.7202, as applicable. Consistent with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-1 through 48 C.F.R. § 227.7102-3, and 48 C.F.R. §§ 227.7202-1 through 227.7202-4, as applicable, the Pre-Release Software is licensed to U.S. Government end users (i) only as Commercial Items and (ii) with only those rights as are granted to all other users pursuant to this Agreement and any related agreement(s), as applicable. Accordingly, you will have no rights in the Pre-Release Software except as expressly agreed to in writing by you and Splunk. - -15. CHOICE OF LAW AND DISPUTES. For other than the U.S. Government as a party, this Agreement shall be governed by and construed in accordance with the laws of the State of California, as if performed wholly within the state and without giving effect to the principles of conflict of law rules of any jurisdiction or the United Nations Convention on Contracts for the International Sale of Goods, the application of which is expressly excluded. Any legal action or proceeding arising under this Agreement will be brought exclusively in the federal or state courts located in San Francisco, California and the parties hereby consent to personal jurisdiction and venue therein. - -16. GENERAL. All notices required or permitted under this Agreement hereto will be in writing and delivered in person, by confirmed facsimile transmission, by overnight delivery service, or by registered or certified mail, postage prepaid with return receipt requested, and in each instance will be deemed given upon receipt. You may not assign, delegate or transfer this Agreement, in whole or in part, by agreement, operation of law or otherwise. You acknowledge that Splunk may assign, subcontract or delegate any of its rights or obligations under this Agreement. Any attempt to assign this Agreement other than as permitted herein shall be null and void. Subject to the foregoing, this Agreement will bind and inure to the benefit of the parties’ permitted successors and assigns. This Agreement along with any additional terms incorporated herein by reference constitute the complete and exclusive understanding and agreement between the parties relating only to the subject matter of the Pre-Release Software and Confidential Information and shall supersede any and all prior or contemporaneous agreements, communications and understandings, written or oral, relating to such subject matter. This Agreement is limited to the use of Pre-Release Software and Confidential Information and as such, this Agreement is separate from and shall have no effect on any other agreement you may have with Splunk. Any waiver, modification or amendment of any provision of this Agreement will be effective only if in writing and signed by duly authorized representatives of both parties. All rights and remedies, whether conferred hereunder or by any other instrument or law, will be cumulative and may be exercised singularly or concurrently. The failure by either party to enforce any provisions of this Agreement will not constitute a waiver of any other right hereunder or of any subsequent enforcement of that or any other provisions. The terms and conditions stated herein are declared to be severable. If a court of competent jurisdiction holds any provision of this Agreement invalid or unenforceable, the remaining provisions of the Agreement will remain in full force and effect, and the provision affected will be construed so as to be enforceable to the maximum extent permissible by law. - -Updated February 16, 2017 From a61e4b63e3ec56a7342e5919f5fe4cc80308f0f0 Mon Sep 17 00:00:00 2001 From: abbas-splunk <52936563+abbas-splunk@users.noreply.github.com> Date: Tue, 6 Aug 2019 15:24:13 -0700 Subject: [PATCH 097/119] Delete LICENSE.md This is an older license and since SCK has been released under Apache 2.0 license this stale pre-release license no longer applies --- .../splunk-kubernetes-metrics/LICENSE.md | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 helm-chart/splunk-kubernetes-metrics/LICENSE.md diff --git a/helm-chart/splunk-kubernetes-metrics/LICENSE.md b/helm-chart/splunk-kubernetes-metrics/LICENSE.md deleted file mode 100644 index b4b506d2..00000000 --- a/helm-chart/splunk-kubernetes-metrics/LICENSE.md +++ /dev/null @@ -1,53 +0,0 @@ -[SPLUNK PRE-RELEASE SOFTWARE LICENSE AGREEMENT](https://www.splunk.com/en_us/legal/splunk-pre-release-software-license-agreement.html) - - - -THIS PRE-RELEASE SOFTWARE LICENSE AGREEMENT (THE "AGREEMENT") GOVERNS YOUR USE OF THE PRE-RELEASE SOFTWARE (DEFINED BELOW) PROVIDED BY SPLUNK INC., AND/OR ITS AFFILIATES ("SPLUNK”). YOU WILL BE REQUIRED TO INDICATE YOUR AGREEMENT TO THIS AGREEMENT IN ORDER TO ACCESS OR DOWNLOAD THE PRE-RELEASE SOFTWARE OR TO COMPLETE THE INSTALLATION PROCESS FOR THE PRE-RELEASE SOFTWARE. BY CLICKING ON THE "YES" OR “I ACCEPT” BUTTON (OR OTHER BUTTON OR MECHANISM DESIGNED TO ACKNOWLEDGE AGREEMENT TO THE TERMS OF THIS AGREEMENT), OR BY DOWNLOADING, ACCESSING OR INSTALLING THE PRE-RELEASE SOFTWARE, YOU ARE CONSENTING TO BE BOUND BY THIS AGREEMENT. - -NOTE: THIS IS THE ONLY AGREEMENT BETWEEN YOU AND SPLUNK GOVERNING YOUR USE OF THE PRE-RELEASE SOFTWARE, AND THIS AGREEMENT SHALL SUPERSEDE ANY OTHER TERMS OF USE OR LICENSE AGREEMENT THAT MAY APPEAR DURING THE INSTALLATION OR DOWNLOADING OF THE PRE-RELEASE SOFTWARE. - -IF YOU AGREE TO THESE TERMS ON BEHALF OF A BUSINESS, YOU REPRESENT AND WARRANT THAT YOU HAVE THE POWER AND AUTHORITY TO BIND THAT BUSINESS TO THIS AGREEMENT, AND YOUR AGREEMENT TO THESE TERMS WILL BE TREATED AS THE AGREEMENT OF THE BUSINESS. IN THAT EVENT, "YOU" AND "YOUR" REFER HEREIN TO THAT BUSINESS. - -IF YOU ARE UNWILLING TO AGREE TO THIS AGREEMENT, OR YOU DO NOT HAVE THE RIGHT, POWER AND AUTHORITY TO ACT ON BEHALF OF AND BIND YOUR BUSINESS, DO NOT CLICK ON THE BUTTON AND DO NOT INSTALL, DOWNLOAD, ACCESS, OR OTHERWISE USE THE PRE-RELEASE SOFTWARE AND CANCEL THE LOADING OF THE PRE-RELEASE SOFTWARE. - -1. DEFINITIONS. - - “Pre-Release Software” means the pre-release version of the Splunk product, service, technology identified on the software download page or landing page or invitation message (the “Cover Page”), whether labeled as alpha, beta, pre-release, preview or otherwise, provided to you by Splunk under this Agreement. Pre-Release Software may include any enhancements, updates, upgrades, derivatives or bug fixes to such product, service or technology and any documentation, add-ons, templates, sample data sets or hardware devices as provided by Splunk. - - “Data” means the raw data you upload or submit to the Pre-Release Software and the processed result of the raw data generated by you using the Pre-Release Software. - - “Feedback” means all suggestions, comments, opinions, code, input, ideas, reports, information, know-how or other feedback provided by you (whether in oral, electronic or written form) to Splunk in connection with your use of the Pre-Release Software. Feedback does not include Data, unless submitted or communicated by you to Splunk as part of Feedback. - - “Internal Purposes” means internal business use with your systems, networks, devices and data for the purposes of internal testing and evaluation of the Pre-Release Software in order to provide Feedback to Splunk regarding the Pre-Release Software. Such use does not include use of your systems, networks or devices as part of services you provide for a third party's benefit. - -2. PRE-RELEASE SOFTWARE LICENSE. Subject to your compliance with the terms and conditions of this Agreement, Splunk grants you a non-exclusive, non-sublicensable, nontransferable, revocable, limited license during the term of the Agreement to use a single copy of the Pre-Release Software at your principal office in a secure location, only in connection with and solely for the Internal Purposes. - -3. LICENSE RESTRICTIONS. Except as expressly authorized in this Agreement or by Splunk, you will not, and will not permit any third party to: (i) access or use the Pre-Release Software for any other purposes than the Internal Purposes (including for any competitive analysis, commercial, professional, or other for-profit purposes); (ii) copy the Pre-Release Software (except as required to run the Pre-Release Software and for reasonable backup purposes); (iii) modify, adapt, or create derivative works of the Pre-Release Software; (iv) rent, lease, loan, resell, transfer, sublicense or distribute the Pre-Release Software to any third party; (v) use or offer any functionality of the Pre-Release Software on a service provider, service bureau, hosted, software as a service, or time sharing basis; (vi) decompile, disassemble or reverse-engineer the Pre-Release Software or otherwise attempt to derive the Pre-Release Software source code, algorithms, methods or techniques used or embodied in the Pre-Release Software; (vii) disclose to any third party the results of any benchmark tests or other evaluation of the Pre-Release Software, or (viii) remove, alter, obscure, cover or change any trademark, copyright or other proprietary notices, labels or markings from or on the Pre-Release Software; (ix) interfere with or disrupt servers or networks connected to any website through which the Pre-Release Software provided; or (x) use the Pre-Release Software to collect or store personal data about any person or entity. Any consultant, contractor, or agent hired to perform services for you may operate the Pre-Release Software on your behalf under these terms and conditions, provided that: (a) you are responsible for ensuring that any such third party agrees to abide by and fully comply with the terms of this Agreement on the same basis as applicable to you; (b) such use is only in connection with your Internal Purposes; (c) such use does not represent or constitute an increase in the scope of the licenses provided hereunder; and (d) you remain fully liable for any and all acts or omissions by such third parties related to this Agreement. Any violation of this Section shall be a material breach of this Agreement subject to immediate termination of this Agreement for which no notice from Splunk shall be required. - -4. CONFIDENTIALITY. You agree to hold the Pre-Release Software (including all intellectual property rights therein, such as any patents, inventions, copyrights, design rights, trade secrets and know-how) and any related information (“Confidential Information”), whether in oral or written form, confidential. Confidential Information may include information relating to features, functionalities, improvements, code, pricing, business strategies, product roadmaps, development plans, marketing materials, data sets, customer lists or other proprietary third-party information. You will hold such Confidential Information in strict confidence and not use or disclose the Confidential Information, in whole or in part, except as expressly permitted in this Agreement. You may disclose Confidential Information to your employees, but only to the extent they have a need to know to test the Pre-Release Software and you have advised them that such information is Confidential. You agree to instruct any such employees in advance who will have access to the Pre-Release Software that they must comply with the restrictions set forth in this Agreement. You shall have no obligation to maintain the confidentiality of any information which: (a) is or becomes publicly available without breach of this Agreement; (b) is rightfully received by you from a third party without an obligation of confidentiality and without breach of this Agreement; (c) is developed independently by you without access to or use of the Confidential Information; or (d) has been approved for release by written authorization of the party that owns the Confidential Information. You will notify immediately upon discovery of any unauthorized use or disclosure of Confidential Information or any other breach of this Agreement by you or your personnel, and will cooperate with Splunk in every reasonable way to help Splunk regain possession of the Confidential Information and prevent its further unauthorized use or disclosure. You acknowledge that any breach of its obligations under this Agreement with respect to the proprietary rights or Confidential Information will cause Splunk irreparable injury for which there are inadequate remedies at law, and therefore, Splunk will be entitled to equitable relief in addition to all other remedies provided by this Agreement or available at law or in equity. - -5. TERM AND TERMINATION. This Agreement will be effective from the earlier of (a) the date it is accepted by you and (b) the date on which you first installed, downloaded or accessed a copy of the Pre-Release Software and shall continue until terminated. This Agreement may be terminated at any time by either party, with or without cause, effective upon notice of termination. This Agreement will terminate automatically upon the end of the Pre-Release Software project as identified on the Cover Page or upon commercial release (if any) of the Pre-Release Software, whichever is earlier. This is time-sensitive software, so it will stop functioning on the termination date. Upon termination, you will immediately cease all use of the Pre-Release Software and destroy the Pre-Release Software, or upon request by Splunk, return to Splunk the Pre-Release Software and other Confidential Information that are in your possession or control. Upon Splunk’s request, you will certify in writing that you have returned or destroyed all copies of the Pre-Release Software and Confidential Information. Sections 1, 3, 4, 6, 7, 8, 9, 10, 11, 15, and 16 will survive termination of this Agreement. - -6. OWNERSHIP. Splunk, its suppliers and/or its licensors own all worldwide right, title and interest in and to the Pre-Release Software, including all worldwide patent rights (including patent applications and disclosures); copyright rights (including copyrights, copyright registration and copy rights with respect to computer software, software design, software code, software architecture, firmware, programming tools, graphic user interfaces, reports, dashboard, business rules, use cases, screens, alerts, notifications, drawings, specifications and databases); moral rights; trade secrets and other rights with respect to confidential or proprietary information; know-how; other rights with respect to inventions, discoveries, ideas, improvements, techniques, formulae, algorithms, processes, schematics, testing procedures, technical information and other technology; and any other intellectual and industrial property rights, whether or not subject to registration or protection; and all rights under any license or other arrangement with respect to the foregoing. Except as expressly stated in this Agreement, Splunk does not grant you any intellectual property rights in the Pre-Release Software, and all right, title, and interest in and to all copies of the Pre-Release Software not expressly granted remain with Splunk, its suppliers and/or its licensors. The Pre-Release Software is copyrighted and protected by the laws of the United States and other countries, and international treaty provisions. You acknowledge that the Pre-Release Software is licensed and not sold. - -7. FEEDBACK. Splunk, in its sole discretion, may or may not respond to your Feedback or promise to address all your Feedback in the development of future features or functionalities of the Pre-Release Software or any related or subsequent versions of such Pre-Release Software. In the event Splunk uses your Feedback, you grant Splunk an unrestricted, perpetual, worldwide, exclusive, transferable, irrevocable, sublicensable, royalty-free, fully paid-up license to use, copy, modify, create derivative works of, make, have made, distribute (through multiple tiers of distribution), publicly perform or display, import, export, sell, offer to sell, rent, or license copies of the Feedback as part of or in connection with any Splunk product, service, technology, content, material, specification or documentation. You warrant that the Feedback does not infringe any copyright or trade secret of any third party, and that you have no knowledge of any patent of any third party that may be infringed by the Feedback (including any implementation thereof recommended by you). You further warrant that your Feedback is not subject to any license terms that would purport to require Splunk to comply with any additional obligations with respect to any Splunk product or service that incorporates your Feedback. - -8. DATA. You hereby grant Splunk a perpetual, irrevocable, non-exclusive, royalty-free, paid-up, worldwide, sublicensable license to use, access, transmit, host, store, and display the Data solely for the purpose of providing and improving the Pre-Release Software. Splunk (or its sublicensees) may exercise such license for purposes of providing, maintaining, repairing, administering and improving the Pre-Release Software or in developing new products or services, including rights to extract, compile, aggregate, synthesize, use, and otherwise analyze all or any portion of the Data. You represent, warrant and agree that the Data and other materials you provide or make available to Splunk will include only information relevant to the Pre-Release Software and the use thereof and will not include any personally identifiable information or any protected health data. You acknowledge and agree that you are solely responsible for all Data you upload or submit using the Pre-Release Software and for your conduct while using the Pre-Release Software. You acknowledge and agree that: (a) you will evaluate and bear all risks associated with the use of any Data; (b) you are responsible for protecting and backing up the Data; (c) you are responsible for protecting the confidentiality of any Data; and (d) under no circumstances will Splunk be liable in any way for any Data, including, but not limited to, any errors or omissions in any Data, or any loss or damages or any kind incurred as a result of your use, deletion, modification, or correction of any Data. Splunk has no responsibility to store, protect, remove or delete any Data for you and shall have no liability for the deletion of or failure to store any Data. - -9. WARRANTY DISCLAIMER. THE PRE-RELEASE SOFTWARE IS PROVIDED “AS IS”. SPLUNK DISCLAIMS ANY AND ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, QUIET ENJOYMENT AND WARRANTIES ARISING OUT OF COURSE OF DEALING OR USAGE OF TRADE OR BY STATUTE OR IN LAW. SPLUNK SPECIFICALLY DOES NOT WARRANT THAT THE PRE-RELEASE SOFTWARE WILL MEET YOUR REQUIREMENTS, THE OPERATION OR OUTPUT OF THE PRE-RELEASE SOFTWARE WILL BE ERROR-FREE, ACCURATE, RELIABLE, COMPLETE OR UNINTERRUPTED. Splunk is not obligated to support, update or upgrade the Pre-Release Software. - -10. NO RELIANCE. Splunk has no obligations about any forward-looking statements made in connection with or in the course of providing the Pre-Release Software. Forward-looking statements are statements regarding future Splunk events, product offerings, product performance, customer uses or the expected financial performance of Splunk. Any such statements reflect current expectations and estimates based on factors currently known and that actual events or results could differ materially. Splunk does not assume any obligation to update any forward-looking statements made during the Pre-Release Software project. In addition, any information about our roadmap outlines our general product direction and is subject to change at any time without notice. It is for informational purposes only and shall not be incorporated into this Agreement or any contract or other commitment. Splunk undertakes no obligation either to develop the features or functionality described in the forward-looking statement or to include any such feature or functionality in a future release, including those you are reviewing as a part of this Pre-Release Software. You expressly acknowledge that the Pre-Release Software has not been fully tested and may contain defects or deficiencies which may not be corrected by Splunk, that the Pre-Release Software may undergo significant changes prior to release of the corresponding generally available final version. - -11. LIMITATION OF LIABILITY. IN NO EVENT WILL SPLUNK BE LIABLE TO YOU FOR ANY SPECIAL, INDIRECT, INCIDENTAL, CONSEQUENTIAL OR PUNITIVE DAMAGES (INCLUDING LOSS OF USE, DATA, OR PROFITS, BUSINESS INTERRUPTION, OR COSTS OF PROCURING SUBSTITUTE PRE-RELEASE SOFTWARE) ARISING OUT OF OR IN CONNECTION WITH THIS AGREEMENT OR THE USE OR PERFORMANCE OF THE PRE-RELEASE SOFTWARE, WHETHER SUCH LIABILITY ARISES FROM CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, AND WHETHER OR NOT SPLUNK HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH LOSS OR DAMAGE. THE PARTIES HAVE AGREED THAT THESE LIMITATIONS WILL SURVIVE AND APPLY EVEN IF ANY REMEDY IS FOUND TO HAVE FAILED OF ITS ESSENTIAL PURPOSE. WITHOUT LIMITING THE FOREGOING, SPLUNK WILL HAVE NO LIABILITY OR RESPONSIBILITY FOR ANY BUSINESS INTERRUPTION OR LOSS OF DATA ARISING FROM THE AUTOMATIC TERMINATION OF THE LICENSE RIGHTS GRANTED HEREIN AND ANY ASSOCIATED CESSATION OF THE PRE-RELEASE SOFTWARE FUNCTIONS OR ANY UNANTICIPATED OR UNSCHEDULED DOWNTIME FOR ANY REASON OR ANY DELETION, CORRUPTION OR DAMAGE OF DATA ON OR THROUGH THE PRE-RELEASE SOFTWARE. SPLUNK'S TOTAL CUMULATIVE LIABILITY TO YOU, FROM ALL CAUSES OF ACTION AND ALL THEORIES OF LIABILITY, WILL BE LIMITED TO AND WILL NOT EXCEED ONE HUNDRED DOLLARS ($100.00). BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU. - -12. PRIVACY. You acknowledge that Splunk and its providers may obtain information and data from you in connection with your registration, installation, and use of the Pre-Release Software, including, without limitation, personal information. Splunk may also collect and process technical and related environmental or performance information about your use of the Pre-Release Software (which may include, without limitation, data ingest volume, search concurrency, number of unique user log-ins, Internet protocol addresses, operating system, configuration information, application software, session duration, page view, operational use, and other such information) and use this information to support and troubleshoot issues, provide updates, analyze trends and improve Splunk’s products or services. You hereby consent to Splunk and its providers maintaining, using, storing and disclosing such information and data (including, without limitation, personal information, if any, in conformity with [Splunk Privacy Policy](https://www.splunk.com/view/SP-CAAAAAG), which may be updated from time to time). You further consent to receiving electronic communications and notifications from Splunk in connection with your use of the Pre-Release Software. - -13. EXPORT. You will comply fully with all relevant export laws and regulations of the United States and any other country (“Export Laws”) where you use any of the Pre-Release Software. You represent and warrant that you are not (a) located in, or a resident or a national of, a restricted country; or (b) on any of the relevant U.S. Government Lists of prohibited persons, including but not limited to the Treasury Department’s List of Specially Designated Nationals and the Commerce Department’s List of Denied Persons or Entity List. You further represent and warrant that you shall not export, re-export, ship, transfer the Pre-Release Software to any restricted countries or restricted end users or use the Pre-Release Software in any restricted countries or for any purposes prohibited by the Export Laws, including, but not limited to, nuclear, chemical, missile or biological weapons related end uses. You understand that the requirements and restrictions of the Export Laws may vary depending on the specific Pre-Release Software and may change over time, and that, to determine the precise controls applicable to the Pre-Release Software, it is necessary to refer to the U.S. Export Administration Regulations and the U.S. Department of Treasury, Office of Foreign Assets Control Regulations. - -14. GOVERNMENT END USER RIGHTS. You acknowledge that all Pre-Release Software and Confidential Information were developed entirely at private expense and that no part of the Pre-Release Software or Confidential Information was first produced in the performance of a Government contract. You agree that the Pre-Release Software and any derivatives thereof are "commercial items" as defined in 48 C.F.R. § 2.101, and if you are a U.S. Government agency or instrumentality or if you are providing all or any part of the Pre-Release Software or any derivatives thereof to the U.S. Government, such use, duplication, reproduction, release, modification, disclosure or transfer of this commercial product and data, is restricted in accordance with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-2, and 48 C.F.R. § 227.7202, as applicable. Consistent with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-1 through 48 C.F.R. § 227.7102-3, and 48 C.F.R. §§ 227.7202-1 through 227.7202-4, as applicable, the Pre-Release Software is licensed to U.S. Government end users (i) only as Commercial Items and (ii) with only those rights as are granted to all other users pursuant to this Agreement and any related agreement(s), as applicable. Accordingly, you will have no rights in the Pre-Release Software except as expressly agreed to in writing by you and Splunk. - -15. CHOICE OF LAW AND DISPUTES. For other than the U.S. Government as a party, this Agreement shall be governed by and construed in accordance with the laws of the State of California, as if performed wholly within the state and without giving effect to the principles of conflict of law rules of any jurisdiction or the United Nations Convention on Contracts for the International Sale of Goods, the application of which is expressly excluded. Any legal action or proceeding arising under this Agreement will be brought exclusively in the federal or state courts located in San Francisco, California and the parties hereby consent to personal jurisdiction and venue therein. - -16. GENERAL. All notices required or permitted under this Agreement hereto will be in writing and delivered in person, by confirmed facsimile transmission, by overnight delivery service, or by registered or certified mail, postage prepaid with return receipt requested, and in each instance will be deemed given upon receipt. You may not assign, delegate or transfer this Agreement, in whole or in part, by agreement, operation of law or otherwise. You acknowledge that Splunk may assign, subcontract or delegate any of its rights or obligations under this Agreement. Any attempt to assign this Agreement other than as permitted herein shall be null and void. Subject to the foregoing, this Agreement will bind and inure to the benefit of the parties’ permitted successors and assigns. This Agreement along with any additional terms incorporated herein by reference constitute the complete and exclusive understanding and agreement between the parties relating only to the subject matter of the Pre-Release Software and Confidential Information and shall supersede any and all prior or contemporaneous agreements, communications and understandings, written or oral, relating to such subject matter. This Agreement is limited to the use of Pre-Release Software and Confidential Information and as such, this Agreement is separate from and shall have no effect on any other agreement you may have with Splunk. Any waiver, modification or amendment of any provision of this Agreement will be effective only if in writing and signed by duly authorized representatives of both parties. All rights and remedies, whether conferred hereunder or by any other instrument or law, will be cumulative and may be exercised singularly or concurrently. The failure by either party to enforce any provisions of this Agreement will not constitute a waiver of any other right hereunder or of any subsequent enforcement of that or any other provisions. The terms and conditions stated herein are declared to be severable. If a court of competent jurisdiction holds any provision of this Agreement invalid or unenforceable, the remaining provisions of the Agreement will remain in full force and effect, and the provision affected will be construed so as to be enforceable to the maximum extent permissible by law. - -Updated February 16, 2017 From 2b4d40575b528bb75d054bebc4fbbfdecdae2cbd Mon Sep 17 00:00:00 2001 From: abbas-splunk <52936563+abbas-splunk@users.noreply.github.com> Date: Tue, 6 Aug 2019 15:24:51 -0700 Subject: [PATCH 098/119] Delete LICENSE.md This is an older license and since SCK has been released under Apache 2.0 license this stale pre-release license no longer applies --- .../splunk-kubernetes-objects/LICENSE.md | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 helm-chart/splunk-kubernetes-objects/LICENSE.md diff --git a/helm-chart/splunk-kubernetes-objects/LICENSE.md b/helm-chart/splunk-kubernetes-objects/LICENSE.md deleted file mode 100644 index b4b506d2..00000000 --- a/helm-chart/splunk-kubernetes-objects/LICENSE.md +++ /dev/null @@ -1,53 +0,0 @@ -[SPLUNK PRE-RELEASE SOFTWARE LICENSE AGREEMENT](https://www.splunk.com/en_us/legal/splunk-pre-release-software-license-agreement.html) - - - -THIS PRE-RELEASE SOFTWARE LICENSE AGREEMENT (THE "AGREEMENT") GOVERNS YOUR USE OF THE PRE-RELEASE SOFTWARE (DEFINED BELOW) PROVIDED BY SPLUNK INC., AND/OR ITS AFFILIATES ("SPLUNK”). YOU WILL BE REQUIRED TO INDICATE YOUR AGREEMENT TO THIS AGREEMENT IN ORDER TO ACCESS OR DOWNLOAD THE PRE-RELEASE SOFTWARE OR TO COMPLETE THE INSTALLATION PROCESS FOR THE PRE-RELEASE SOFTWARE. BY CLICKING ON THE "YES" OR “I ACCEPT” BUTTON (OR OTHER BUTTON OR MECHANISM DESIGNED TO ACKNOWLEDGE AGREEMENT TO THE TERMS OF THIS AGREEMENT), OR BY DOWNLOADING, ACCESSING OR INSTALLING THE PRE-RELEASE SOFTWARE, YOU ARE CONSENTING TO BE BOUND BY THIS AGREEMENT. - -NOTE: THIS IS THE ONLY AGREEMENT BETWEEN YOU AND SPLUNK GOVERNING YOUR USE OF THE PRE-RELEASE SOFTWARE, AND THIS AGREEMENT SHALL SUPERSEDE ANY OTHER TERMS OF USE OR LICENSE AGREEMENT THAT MAY APPEAR DURING THE INSTALLATION OR DOWNLOADING OF THE PRE-RELEASE SOFTWARE. - -IF YOU AGREE TO THESE TERMS ON BEHALF OF A BUSINESS, YOU REPRESENT AND WARRANT THAT YOU HAVE THE POWER AND AUTHORITY TO BIND THAT BUSINESS TO THIS AGREEMENT, AND YOUR AGREEMENT TO THESE TERMS WILL BE TREATED AS THE AGREEMENT OF THE BUSINESS. IN THAT EVENT, "YOU" AND "YOUR" REFER HEREIN TO THAT BUSINESS. - -IF YOU ARE UNWILLING TO AGREE TO THIS AGREEMENT, OR YOU DO NOT HAVE THE RIGHT, POWER AND AUTHORITY TO ACT ON BEHALF OF AND BIND YOUR BUSINESS, DO NOT CLICK ON THE BUTTON AND DO NOT INSTALL, DOWNLOAD, ACCESS, OR OTHERWISE USE THE PRE-RELEASE SOFTWARE AND CANCEL THE LOADING OF THE PRE-RELEASE SOFTWARE. - -1. DEFINITIONS. - - “Pre-Release Software” means the pre-release version of the Splunk product, service, technology identified on the software download page or landing page or invitation message (the “Cover Page”), whether labeled as alpha, beta, pre-release, preview or otherwise, provided to you by Splunk under this Agreement. Pre-Release Software may include any enhancements, updates, upgrades, derivatives or bug fixes to such product, service or technology and any documentation, add-ons, templates, sample data sets or hardware devices as provided by Splunk. - - “Data” means the raw data you upload or submit to the Pre-Release Software and the processed result of the raw data generated by you using the Pre-Release Software. - - “Feedback” means all suggestions, comments, opinions, code, input, ideas, reports, information, know-how or other feedback provided by you (whether in oral, electronic or written form) to Splunk in connection with your use of the Pre-Release Software. Feedback does not include Data, unless submitted or communicated by you to Splunk as part of Feedback. - - “Internal Purposes” means internal business use with your systems, networks, devices and data for the purposes of internal testing and evaluation of the Pre-Release Software in order to provide Feedback to Splunk regarding the Pre-Release Software. Such use does not include use of your systems, networks or devices as part of services you provide for a third party's benefit. - -2. PRE-RELEASE SOFTWARE LICENSE. Subject to your compliance with the terms and conditions of this Agreement, Splunk grants you a non-exclusive, non-sublicensable, nontransferable, revocable, limited license during the term of the Agreement to use a single copy of the Pre-Release Software at your principal office in a secure location, only in connection with and solely for the Internal Purposes. - -3. LICENSE RESTRICTIONS. Except as expressly authorized in this Agreement or by Splunk, you will not, and will not permit any third party to: (i) access or use the Pre-Release Software for any other purposes than the Internal Purposes (including for any competitive analysis, commercial, professional, or other for-profit purposes); (ii) copy the Pre-Release Software (except as required to run the Pre-Release Software and for reasonable backup purposes); (iii) modify, adapt, or create derivative works of the Pre-Release Software; (iv) rent, lease, loan, resell, transfer, sublicense or distribute the Pre-Release Software to any third party; (v) use or offer any functionality of the Pre-Release Software on a service provider, service bureau, hosted, software as a service, or time sharing basis; (vi) decompile, disassemble or reverse-engineer the Pre-Release Software or otherwise attempt to derive the Pre-Release Software source code, algorithms, methods or techniques used or embodied in the Pre-Release Software; (vii) disclose to any third party the results of any benchmark tests or other evaluation of the Pre-Release Software, or (viii) remove, alter, obscure, cover or change any trademark, copyright or other proprietary notices, labels or markings from or on the Pre-Release Software; (ix) interfere with or disrupt servers or networks connected to any website through which the Pre-Release Software provided; or (x) use the Pre-Release Software to collect or store personal data about any person or entity. Any consultant, contractor, or agent hired to perform services for you may operate the Pre-Release Software on your behalf under these terms and conditions, provided that: (a) you are responsible for ensuring that any such third party agrees to abide by and fully comply with the terms of this Agreement on the same basis as applicable to you; (b) such use is only in connection with your Internal Purposes; (c) such use does not represent or constitute an increase in the scope of the licenses provided hereunder; and (d) you remain fully liable for any and all acts or omissions by such third parties related to this Agreement. Any violation of this Section shall be a material breach of this Agreement subject to immediate termination of this Agreement for which no notice from Splunk shall be required. - -4. CONFIDENTIALITY. You agree to hold the Pre-Release Software (including all intellectual property rights therein, such as any patents, inventions, copyrights, design rights, trade secrets and know-how) and any related information (“Confidential Information”), whether in oral or written form, confidential. Confidential Information may include information relating to features, functionalities, improvements, code, pricing, business strategies, product roadmaps, development plans, marketing materials, data sets, customer lists or other proprietary third-party information. You will hold such Confidential Information in strict confidence and not use or disclose the Confidential Information, in whole or in part, except as expressly permitted in this Agreement. You may disclose Confidential Information to your employees, but only to the extent they have a need to know to test the Pre-Release Software and you have advised them that such information is Confidential. You agree to instruct any such employees in advance who will have access to the Pre-Release Software that they must comply with the restrictions set forth in this Agreement. You shall have no obligation to maintain the confidentiality of any information which: (a) is or becomes publicly available without breach of this Agreement; (b) is rightfully received by you from a third party without an obligation of confidentiality and without breach of this Agreement; (c) is developed independently by you without access to or use of the Confidential Information; or (d) has been approved for release by written authorization of the party that owns the Confidential Information. You will notify immediately upon discovery of any unauthorized use or disclosure of Confidential Information or any other breach of this Agreement by you or your personnel, and will cooperate with Splunk in every reasonable way to help Splunk regain possession of the Confidential Information and prevent its further unauthorized use or disclosure. You acknowledge that any breach of its obligations under this Agreement with respect to the proprietary rights or Confidential Information will cause Splunk irreparable injury for which there are inadequate remedies at law, and therefore, Splunk will be entitled to equitable relief in addition to all other remedies provided by this Agreement or available at law or in equity. - -5. TERM AND TERMINATION. This Agreement will be effective from the earlier of (a) the date it is accepted by you and (b) the date on which you first installed, downloaded or accessed a copy of the Pre-Release Software and shall continue until terminated. This Agreement may be terminated at any time by either party, with or without cause, effective upon notice of termination. This Agreement will terminate automatically upon the end of the Pre-Release Software project as identified on the Cover Page or upon commercial release (if any) of the Pre-Release Software, whichever is earlier. This is time-sensitive software, so it will stop functioning on the termination date. Upon termination, you will immediately cease all use of the Pre-Release Software and destroy the Pre-Release Software, or upon request by Splunk, return to Splunk the Pre-Release Software and other Confidential Information that are in your possession or control. Upon Splunk’s request, you will certify in writing that you have returned or destroyed all copies of the Pre-Release Software and Confidential Information. Sections 1, 3, 4, 6, 7, 8, 9, 10, 11, 15, and 16 will survive termination of this Agreement. - -6. OWNERSHIP. Splunk, its suppliers and/or its licensors own all worldwide right, title and interest in and to the Pre-Release Software, including all worldwide patent rights (including patent applications and disclosures); copyright rights (including copyrights, copyright registration and copy rights with respect to computer software, software design, software code, software architecture, firmware, programming tools, graphic user interfaces, reports, dashboard, business rules, use cases, screens, alerts, notifications, drawings, specifications and databases); moral rights; trade secrets and other rights with respect to confidential or proprietary information; know-how; other rights with respect to inventions, discoveries, ideas, improvements, techniques, formulae, algorithms, processes, schematics, testing procedures, technical information and other technology; and any other intellectual and industrial property rights, whether or not subject to registration or protection; and all rights under any license or other arrangement with respect to the foregoing. Except as expressly stated in this Agreement, Splunk does not grant you any intellectual property rights in the Pre-Release Software, and all right, title, and interest in and to all copies of the Pre-Release Software not expressly granted remain with Splunk, its suppliers and/or its licensors. The Pre-Release Software is copyrighted and protected by the laws of the United States and other countries, and international treaty provisions. You acknowledge that the Pre-Release Software is licensed and not sold. - -7. FEEDBACK. Splunk, in its sole discretion, may or may not respond to your Feedback or promise to address all your Feedback in the development of future features or functionalities of the Pre-Release Software or any related or subsequent versions of such Pre-Release Software. In the event Splunk uses your Feedback, you grant Splunk an unrestricted, perpetual, worldwide, exclusive, transferable, irrevocable, sublicensable, royalty-free, fully paid-up license to use, copy, modify, create derivative works of, make, have made, distribute (through multiple tiers of distribution), publicly perform or display, import, export, sell, offer to sell, rent, or license copies of the Feedback as part of or in connection with any Splunk product, service, technology, content, material, specification or documentation. You warrant that the Feedback does not infringe any copyright or trade secret of any third party, and that you have no knowledge of any patent of any third party that may be infringed by the Feedback (including any implementation thereof recommended by you). You further warrant that your Feedback is not subject to any license terms that would purport to require Splunk to comply with any additional obligations with respect to any Splunk product or service that incorporates your Feedback. - -8. DATA. You hereby grant Splunk a perpetual, irrevocable, non-exclusive, royalty-free, paid-up, worldwide, sublicensable license to use, access, transmit, host, store, and display the Data solely for the purpose of providing and improving the Pre-Release Software. Splunk (or its sublicensees) may exercise such license for purposes of providing, maintaining, repairing, administering and improving the Pre-Release Software or in developing new products or services, including rights to extract, compile, aggregate, synthesize, use, and otherwise analyze all or any portion of the Data. You represent, warrant and agree that the Data and other materials you provide or make available to Splunk will include only information relevant to the Pre-Release Software and the use thereof and will not include any personally identifiable information or any protected health data. You acknowledge and agree that you are solely responsible for all Data you upload or submit using the Pre-Release Software and for your conduct while using the Pre-Release Software. You acknowledge and agree that: (a) you will evaluate and bear all risks associated with the use of any Data; (b) you are responsible for protecting and backing up the Data; (c) you are responsible for protecting the confidentiality of any Data; and (d) under no circumstances will Splunk be liable in any way for any Data, including, but not limited to, any errors or omissions in any Data, or any loss or damages or any kind incurred as a result of your use, deletion, modification, or correction of any Data. Splunk has no responsibility to store, protect, remove or delete any Data for you and shall have no liability for the deletion of or failure to store any Data. - -9. WARRANTY DISCLAIMER. THE PRE-RELEASE SOFTWARE IS PROVIDED “AS IS”. SPLUNK DISCLAIMS ANY AND ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, QUIET ENJOYMENT AND WARRANTIES ARISING OUT OF COURSE OF DEALING OR USAGE OF TRADE OR BY STATUTE OR IN LAW. SPLUNK SPECIFICALLY DOES NOT WARRANT THAT THE PRE-RELEASE SOFTWARE WILL MEET YOUR REQUIREMENTS, THE OPERATION OR OUTPUT OF THE PRE-RELEASE SOFTWARE WILL BE ERROR-FREE, ACCURATE, RELIABLE, COMPLETE OR UNINTERRUPTED. Splunk is not obligated to support, update or upgrade the Pre-Release Software. - -10. NO RELIANCE. Splunk has no obligations about any forward-looking statements made in connection with or in the course of providing the Pre-Release Software. Forward-looking statements are statements regarding future Splunk events, product offerings, product performance, customer uses or the expected financial performance of Splunk. Any such statements reflect current expectations and estimates based on factors currently known and that actual events or results could differ materially. Splunk does not assume any obligation to update any forward-looking statements made during the Pre-Release Software project. In addition, any information about our roadmap outlines our general product direction and is subject to change at any time without notice. It is for informational purposes only and shall not be incorporated into this Agreement or any contract or other commitment. Splunk undertakes no obligation either to develop the features or functionality described in the forward-looking statement or to include any such feature or functionality in a future release, including those you are reviewing as a part of this Pre-Release Software. You expressly acknowledge that the Pre-Release Software has not been fully tested and may contain defects or deficiencies which may not be corrected by Splunk, that the Pre-Release Software may undergo significant changes prior to release of the corresponding generally available final version. - -11. LIMITATION OF LIABILITY. IN NO EVENT WILL SPLUNK BE LIABLE TO YOU FOR ANY SPECIAL, INDIRECT, INCIDENTAL, CONSEQUENTIAL OR PUNITIVE DAMAGES (INCLUDING LOSS OF USE, DATA, OR PROFITS, BUSINESS INTERRUPTION, OR COSTS OF PROCURING SUBSTITUTE PRE-RELEASE SOFTWARE) ARISING OUT OF OR IN CONNECTION WITH THIS AGREEMENT OR THE USE OR PERFORMANCE OF THE PRE-RELEASE SOFTWARE, WHETHER SUCH LIABILITY ARISES FROM CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, AND WHETHER OR NOT SPLUNK HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH LOSS OR DAMAGE. THE PARTIES HAVE AGREED THAT THESE LIMITATIONS WILL SURVIVE AND APPLY EVEN IF ANY REMEDY IS FOUND TO HAVE FAILED OF ITS ESSENTIAL PURPOSE. WITHOUT LIMITING THE FOREGOING, SPLUNK WILL HAVE NO LIABILITY OR RESPONSIBILITY FOR ANY BUSINESS INTERRUPTION OR LOSS OF DATA ARISING FROM THE AUTOMATIC TERMINATION OF THE LICENSE RIGHTS GRANTED HEREIN AND ANY ASSOCIATED CESSATION OF THE PRE-RELEASE SOFTWARE FUNCTIONS OR ANY UNANTICIPATED OR UNSCHEDULED DOWNTIME FOR ANY REASON OR ANY DELETION, CORRUPTION OR DAMAGE OF DATA ON OR THROUGH THE PRE-RELEASE SOFTWARE. SPLUNK'S TOTAL CUMULATIVE LIABILITY TO YOU, FROM ALL CAUSES OF ACTION AND ALL THEORIES OF LIABILITY, WILL BE LIMITED TO AND WILL NOT EXCEED ONE HUNDRED DOLLARS ($100.00). BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU. - -12. PRIVACY. You acknowledge that Splunk and its providers may obtain information and data from you in connection with your registration, installation, and use of the Pre-Release Software, including, without limitation, personal information. Splunk may also collect and process technical and related environmental or performance information about your use of the Pre-Release Software (which may include, without limitation, data ingest volume, search concurrency, number of unique user log-ins, Internet protocol addresses, operating system, configuration information, application software, session duration, page view, operational use, and other such information) and use this information to support and troubleshoot issues, provide updates, analyze trends and improve Splunk’s products or services. You hereby consent to Splunk and its providers maintaining, using, storing and disclosing such information and data (including, without limitation, personal information, if any, in conformity with [Splunk Privacy Policy](https://www.splunk.com/view/SP-CAAAAAG), which may be updated from time to time). You further consent to receiving electronic communications and notifications from Splunk in connection with your use of the Pre-Release Software. - -13. EXPORT. You will comply fully with all relevant export laws and regulations of the United States and any other country (“Export Laws”) where you use any of the Pre-Release Software. You represent and warrant that you are not (a) located in, or a resident or a national of, a restricted country; or (b) on any of the relevant U.S. Government Lists of prohibited persons, including but not limited to the Treasury Department’s List of Specially Designated Nationals and the Commerce Department’s List of Denied Persons or Entity List. You further represent and warrant that you shall not export, re-export, ship, transfer the Pre-Release Software to any restricted countries or restricted end users or use the Pre-Release Software in any restricted countries or for any purposes prohibited by the Export Laws, including, but not limited to, nuclear, chemical, missile or biological weapons related end uses. You understand that the requirements and restrictions of the Export Laws may vary depending on the specific Pre-Release Software and may change over time, and that, to determine the precise controls applicable to the Pre-Release Software, it is necessary to refer to the U.S. Export Administration Regulations and the U.S. Department of Treasury, Office of Foreign Assets Control Regulations. - -14. GOVERNMENT END USER RIGHTS. You acknowledge that all Pre-Release Software and Confidential Information were developed entirely at private expense and that no part of the Pre-Release Software or Confidential Information was first produced in the performance of a Government contract. You agree that the Pre-Release Software and any derivatives thereof are "commercial items" as defined in 48 C.F.R. § 2.101, and if you are a U.S. Government agency or instrumentality or if you are providing all or any part of the Pre-Release Software or any derivatives thereof to the U.S. Government, such use, duplication, reproduction, release, modification, disclosure or transfer of this commercial product and data, is restricted in accordance with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-2, and 48 C.F.R. § 227.7202, as applicable. Consistent with 48 C.F.R. § 12.211, 48 C.F.R. § 12.212, 48 C.F.R. § 227.7102-1 through 48 C.F.R. § 227.7102-3, and 48 C.F.R. §§ 227.7202-1 through 227.7202-4, as applicable, the Pre-Release Software is licensed to U.S. Government end users (i) only as Commercial Items and (ii) with only those rights as are granted to all other users pursuant to this Agreement and any related agreement(s), as applicable. Accordingly, you will have no rights in the Pre-Release Software except as expressly agreed to in writing by you and Splunk. - -15. CHOICE OF LAW AND DISPUTES. For other than the U.S. Government as a party, this Agreement shall be governed by and construed in accordance with the laws of the State of California, as if performed wholly within the state and without giving effect to the principles of conflict of law rules of any jurisdiction or the United Nations Convention on Contracts for the International Sale of Goods, the application of which is expressly excluded. Any legal action or proceeding arising under this Agreement will be brought exclusively in the federal or state courts located in San Francisco, California and the parties hereby consent to personal jurisdiction and venue therein. - -16. GENERAL. All notices required or permitted under this Agreement hereto will be in writing and delivered in person, by confirmed facsimile transmission, by overnight delivery service, or by registered or certified mail, postage prepaid with return receipt requested, and in each instance will be deemed given upon receipt. You may not assign, delegate or transfer this Agreement, in whole or in part, by agreement, operation of law or otherwise. You acknowledge that Splunk may assign, subcontract or delegate any of its rights or obligations under this Agreement. Any attempt to assign this Agreement other than as permitted herein shall be null and void. Subject to the foregoing, this Agreement will bind and inure to the benefit of the parties’ permitted successors and assigns. This Agreement along with any additional terms incorporated herein by reference constitute the complete and exclusive understanding and agreement between the parties relating only to the subject matter of the Pre-Release Software and Confidential Information and shall supersede any and all prior or contemporaneous agreements, communications and understandings, written or oral, relating to such subject matter. This Agreement is limited to the use of Pre-Release Software and Confidential Information and as such, this Agreement is separate from and shall have no effect on any other agreement you may have with Splunk. Any waiver, modification or amendment of any provision of this Agreement will be effective only if in writing and signed by duly authorized representatives of both parties. All rights and remedies, whether conferred hereunder or by any other instrument or law, will be cumulative and may be exercised singularly or concurrently. The failure by either party to enforce any provisions of this Agreement will not constitute a waiver of any other right hereunder or of any subsequent enforcement of that or any other provisions. The terms and conditions stated herein are declared to be severable. If a court of competent jurisdiction holds any provision of this Agreement invalid or unenforceable, the remaining provisions of the Agreement will remain in full force and effect, and the provision affected will be construed so as to be enforceable to the maximum extent permissible by law. - -Updated February 16, 2017 From 98e3709cf775012b6ddfe0b7808bb271db0683e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarle=20B=C3=B8rsheim?= Date: Wed, 7 Aug 2019 18:13:23 +0200 Subject: [PATCH 099/119] Fixed problem in splunk-kubernetes-logging configMap. #217 (#218) * Fixed problem in splunk-kubernetes-logging configMap. Did not parse. (#208) * Fixed problem with resolving source in configMap. (#217) --- helm-chart/splunk-kubernetes-logging/templates/configMap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index e16ea4c9..6fbbc68b 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -209,7 +209,7 @@ data: # = filters for monitor agent = @type jq_transformer - jq '.record.source = "namespace:#{ENV[\'MY_NAMESPACE\']}/pod:#{ENV[\'MY_POD_NAME\']}" | .record.sourcetype = "fluentd:monitor-agent" | .record.cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" | .record' + jq ".record.source = \"namespace:#{ENV['MY_NAMESPACE']}/pod:#{ENV['MY_POD_NAME']}\" | .record.sourcetype = \"fluentd:monitor-agent\" | .record.cluster_name = \"{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}\" | .record" # = output = From 1c40aa8ec9f3f8c83b97dbe0dd3d5920f0f83682 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 8 Aug 2019 13:40:18 -0700 Subject: [PATCH 100/119] Custom filters (#216) --- .../templates/configMap.yaml | 12 ++++++ .../splunk-kubernetes-logging/values.yaml | 37 ++++++++++++++++ .../templates/configMap.yaml | 11 +++++ .../splunk-kubernetes-metrics/values.yaml | 43 +++++++++++++++++-- .../templates/configMap.yaml | 12 ++++++ .../splunk-kubernetes-objects/values.yaml | 38 ++++++++++++++++ 6 files changed, 150 insertions(+), 3 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 6fbbc68b..043dfc97 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -212,6 +212,18 @@ data: jq ".record.source = \"namespace:#{ENV['MY_NAMESPACE']}/pod:#{ENV['MY_POD_NAME']}\" | .record.sourcetype = \"fluentd:monitor-agent\" | .record.cluster_name = \"{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}\" | .record" + # = custom filters specified by users = + {{- range $name, $filterDef := .Values.customFilters }} + {{- if and $filterDef.tag $filterDef.type }} + + @type {{ $filterDef.type }} + {{- if $filterDef.body }} + {{ $filterDef.body }} + {{- end }} + + {{- end }} + {{- end }} + # = output = @type splunk_hec diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 673f29ce..371f363a 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -322,3 +322,40 @@ kubernetes: custom_metadata: # - name: "cloud_account_id" # value: "1234567890" + +# `customFilters` defines the custom filters to be used. +# This section can be used to define custom filters using plugins like https://github.com/splunk/fluent-plugin-jq +# Its also possible to use other filters like https://www.fluentd.org/plugins#filter +# +# The scheme to define a custom filter is: +# +# ``` +# : +# tag: +# type: +# body: +# ``` +# +# = fluentd tag for the filter = +# This is the fluentd tag for the record +# +# = fluentd filter type = +# This is the fluentd filter that the user wants to use for record manipulation. +# +# = definition of the fluentd filter = +# This defines the body/logic for using the filter for record manipulation. +# +# For example if you want to define a filter which sets cluster_name field to "my_awesome_cluster" you would the following filter +# +# @type jq_transformer +# jq '.record.cluster_name = "my_awesome_cluster" | .record' +# +# This can be defined in the customFilters section as follows: +# ``` +# customFilters: +# NamespaceSourcetypeFilter: +# tag: tail.containers.** +# type: jq_transformer +# body: jq '.record.cluster_name = "my_awesome_cluster" | .record' +# ``` +customFilters: {} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml index 8e61bbb2..fa3f4f8d 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/configMap.yaml @@ -76,6 +76,17 @@ data: source ${record['node']}/${record['pod-name']}/${record['container-name']} + # = custom filters specified by users = + {{- range $name, $filterDef := .Values.customFilters }} + {{- if and $filterDef.tag $filterDef.type }} + + @type {{ $filterDef.type }} + {{- if $filterDef.body }} + {{ $filterDef.body }} + {{- end }} + + {{- end }} + {{- end }} @type splunk_hec data_type metric diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index a089087a..7695820e 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -127,8 +127,8 @@ aggregatorBuffer: flush_thread_count: 1 overflow_action: block retry_max_times: 3 - - + + # Configure how often SCK pulls metrics for its kubenetes sources. 15s is the default where 's' is seconds. metricsInterval: 15s @@ -144,7 +144,7 @@ tolerations: effect: NoSchedule -# Tolerations for the aggregator pod. We do not really want this running on the master nodes, so we leave this +# Tolerations for the aggregator pod. We do not really want this running on the master nodes, so we leave this # blank by default. aggregatorTolerations: {} @@ -176,3 +176,40 @@ kubernetes: clusterName: # Add privileged access to containers for openshift compatability openshift: false + + # `customFilters` defines the custom filters to be used. + # This section can be used to define custom filters using plugins like https://github.com/splunk/fluent-plugin-jq + # Its also possible to use other filters like https://www.fluentd.org/plugins#filter + # + # The scheme to define a custom filter is: + # + # ``` + # : + # tag: + # type: + # body: + # ``` + # + # = fluentd tag for the filter = + # This is the fluentd tag for the record + # + # = fluentd filter type = + # This is the fluentd filter that the user wants to use for record manipulation. + # + # = definition of the fluentd filter = + # This defines the body/logic for using the filter for record manipulation. + # + # For example if you want to define a filter which sets cluster_name field to "my_awesome_cluster" you would the following filter + # + # @type jq_transformer + # jq '.record.cluster_name = "my_awesome_cluster" | .record' + # + # This can be defined in the customFilters section as follows: + # ``` + # customFilters: + # NamespaceSourcetypeFilter: + # tag: tail.containers.** + # type: jq_transformer + # body: jq '.record.cluster_name = "my_awesome_cluster" | .record' + # ``` + customFilters: {} diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index c342083c..4788daeb 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -87,6 +87,18 @@ data: jq '.record.cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }}" | .record' + # = custom filters specified by users = + {{- range $name, $filterDef := .Values.customFilters }} + {{- if and $filterDef.tag $filterDef.type }} + + @type {{ $filterDef.type }} + {{- if $filterDef.body }} + {{ $filterDef.body }} + {{- end }} + + {{- end }} + {{- end }} + @type splunk_hec protocol {{ or .Values.splunk.hec.protocol .Values.global.splunk.hec.protocol | default "https" }} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 0dc653b4..721dfde4 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -213,3 +213,41 @@ tolerations: [] affinity: {} + + +# `customFilters` defines the custom filters to be used. +# This section can be used to define custom filters using plugins like https://github.com/splunk/fluent-plugin-jq +# Its also possible to use other filters like https://www.fluentd.org/plugins#filter +# +# The scheme to define a custom filter is: +# +# ``` +# : +# tag: +# type: +# body: +# ``` +# +# = fluentd tag for the filter = +# This is the fluentd tag for the record +# +# = fluentd filter type = +# This is the fluentd filter that the user wants to use for record manipulation. +# +# = definition of the fluentd filter = +# This defines the body/logic for using the filter for record manipulation. +# +# For example if you want to define a filter which sets cluster_name field to "my_awesome_cluster" you would the following filter +# +# @type jq_transformer +# jq '.record.cluster_name = "my_awesome_cluster" | .record' +# +# This can be defined in the customFilters section as follows: +# ``` +# customFilters: +# NamespaceSourcetypeFilter: +# tag: tail.containers.** +# type: jq_transformer +# body: jq '.record.cluster_name = "my_awesome_cluster" | .record' +# ``` +customFilters: {} From c18ccff22ed7a024ed3858bb48b272eedd25c986 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 19 Aug 2019 13:31:58 -0700 Subject: [PATCH 101/119] Update README to reflect support policy (#227) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b451e99..2b43cc56 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,8 @@ For this topology to work appropriately we have to create the splunk indexes "ku # Maintenance And Support - +Splunk Connect For Kubernetes is supported through Splunk Support assuming the customer has a current Splunk support entitlement. For customers that do not have a current Splunk support entitlement, please file an issue at create a new issue at [Create a new issue in splunk connect for kubernetes project](https://github.com/splunk/splunk-connect-for-kubernetes/issues/new) The current maintainers of this project are the DataEdge team at Splunk. You can reach us at [DataEdge@splunk.com](mailto:DataEdge@splunk.com). -If you have any issues with the software, please file an issue at [Create a new issue in splunk connect for kubernetes project](https://github.com/splunk/splunk-connect-for-kubernetes/issues/new) # License From 3ad8cb10239fc9286fa9fcec7afdebe409d2e92b Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 20 Aug 2019 10:41:54 -0700 Subject: [PATCH 102/119] Docker private images (#222) --- .../templates/_helpers.tpl | 5 ++ .../templates/daemonset.yaml | 2 +- .../templates/serviceAccount.yaml | 6 ++- .../splunk-kubernetes-logging/values.yaml | 15 +++++- .../templates/_helpers.tpl | 13 ++++++ .../templates/deployment.yaml | 2 +- .../deploymentMetricsAggregator.yaml | 2 +- .../templates/serviceAccount.yaml | 4 ++ .../splunk-kubernetes-metrics/values.yaml | 46 +++++++++++++++---- .../templates/_helpers.tpl | 7 +++ .../templates/deployment.yaml | 2 +- .../templates/serviceAccount.yaml | 4 ++ .../splunk-kubernetes-objects/values.yaml | 24 ++++++++-- 13 files changed, 115 insertions(+), 17 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index c2405172..3fef8e02 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -133,3 +133,8 @@ Create the name of the service account to use {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} +Create the image name +*/}} +{{- define "splunk-kubernetes-logging.image" -}} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.name .Values.image.tag -}} +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 869ffe73..0b96e1ce 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -36,7 +36,7 @@ spec: {{- end }} containers: - name: splunk-fluentd-k8s-logs - image: {{ .Values.image.name }} + image: {{ template "splunk-kubernetes-logging.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: diff --git a/helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml b/helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml index fee73b63..f722adda 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/serviceAccount.yaml @@ -1,5 +1,9 @@ {{- if .Values.serviceAccount.create -}} apiVersion: v1 +{{ if .Values.image.usePullSecret }} +imagePullSecrets: +- name: {{ .Values.image.pullSecretName }} +{{ end }} kind: ServiceAccount metadata: name: {{ template "splunk-kubernetes-logging.serviceAccountName" . }} @@ -8,4 +12,4 @@ metadata: chart: {{ template "splunk-kubernetes-logging.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 371f363a..9479ccfa 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -60,6 +60,9 @@ serviceAccount: # The name of the ServiceAccount to use. # If not set and create is true, a name is generated using the fullname template name: + # This flag specifies if the user wants to use a secret for creating the serviceAccount, + # which will be used to get the images from a private registry + usePullSecrets: false # Local splunk configurations @@ -259,8 +262,18 @@ logs: # Defines which version of image to use, and how it should be pulled. image: - name: splunk/fluentd-hec:1.1.1 + # The domain of the registry to pull the image from + registry: docker.io + # The name of the image to pull + name: splunk/fluentd-hec + # The tag of the image to pull + tag: 1.1.1 + # The policy that specifies when the user wants the images to be pulled pullPolicy: Always + # Indicates if the image should be pulled using authentication from a secret + usePullSecret: false + # The name of the pull secret to attach to the respective serviceaccount used to pull the image + pullsecretName: # Controls the resources used by the fluentd daemonset diff --git a/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl index fe7c5a6c..98489637 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-metrics/templates/_helpers.tpl @@ -52,3 +52,16 @@ Create the name of the service account to use {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Create the image name +*/}} +{{- define "splunk-kubernetes-metrics.image" -}} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.name .Values.image.tag -}} +{{- end -}} +{{/* +Create the aggregate image name +*/}} +{{- define "splunk-kubernetes-metrics.imageAgg" -}} +{{- printf "%s/%s:%s" .Values.imageAgg.registry .Values.imageAgg.name .Values.imageAgg.tag -}} +{{- end -}} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index e0602fbb..d089f1c2 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -41,7 +41,7 @@ spec: {{- end }} containers: - name: splunk-fluentd-k8s-metrics - image: {{ .Values.image.name }} + image: {{ template "splunk-kubernetes-metrics.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index 5c1aee0f..0403cdd8 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -37,7 +37,7 @@ spec: {{- end }} containers: - name: splunk-fluentd-k8s-metrics-agg - image: {{ .Values.imageAgg.name }} + image: {{ template "splunk-kubernetes-metrics.imageAgg" . }} imagePullPolicy: {{ .Values.imageAgg.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: diff --git a/helm-chart/splunk-kubernetes-metrics/templates/serviceAccount.yaml b/helm-chart/splunk-kubernetes-metrics/templates/serviceAccount.yaml index 5a189b61..3ec69a23 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/serviceAccount.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/serviceAccount.yaml @@ -1,5 +1,9 @@ {{- if .Values.serviceAccount.create -}} apiVersion: v1 +{{ if .Values.image.usePullSecret }} +imagePullSecrets: +- name: {{ .Values.image.pullSecretName }} +{{ end }} kind: ServiceAccount metadata: name: {{ template "splunk-kubernetes-metrics.serviceAccountName" . }} diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index 7695820e..dd3d6046 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -1,15 +1,16 @@ -# Default values for splunk-kubernetes-logging. +# Default values for splunk-kubernetes-metrics. # This is a YAML-formatted file. # Declare variables to be passed into your templates. -# = Global Configs = -# Allow to pass config values from parent chart using `global.*`. -# Global values have lower priority. -# Values defined here are the default values. +# Global configurations +# These configurations will be used if the corresponding local configurations are not set. +# For example, if `global.logLevel` is set and `logLevel` is not set, `global.logLevel` will be used; if `logLevel` is set, it will be used regardless `global.logLevel` is set or not. global: logLevel: info + # If local splunk configurations are not present, the global ones will be used (if available) splunk: + # It has exactly the same configs as splunk.hec does hec: port: 8088 protocol: https @@ -42,6 +43,9 @@ serviceAccount: # The name of the ServiceAccount to use. # If not set and create is true, a name is generated using the fullname template name: + # This flag specifies if the user wants to use a secret for creating the serviceAccount, + # which will be used to get the images from a private registry + usePullSecrets: false # = Splunk HEC Connection = @@ -80,16 +84,39 @@ secret: name: +# Defines which version of image to use, and how it should be pulled. image: - name: splunk/k8s-metrics:1.1.1 + # The domain of the registry to pull the image from + registry: docker.io + # The name of the image to pull + name: splunk/k8s-metrics + # The tag of the image to pull + tag: 1.1.1 + # The policy that specifies when the user wants the images to be pulled pullPolicy: Always + # Indicates if the image should be pulled using authentication from a secret + usePullSecret: false + # The name of the pull secret to attach to the respective serviceaccount used to pull the image + pullsecretName: +# Defines which version of image to use, and how it should be pulled. imageAgg: - name: splunk/k8s-metrics-aggr:1.1.0 + # The domain of the registry to pull the image from + registry: docker.io + # The name of the image to pull + name: splunk/k8s-metrics-aggr + # The tag of the image to pull + tag: 1.1.0 + # The policy that specifies when the user wants the images to be pulled pullPolicy: Always + # Indicates if the image should be pulled using authentication from a secret + usePullSecret: false + # The name of the pull secret to attach to the respective serviceaccount used to pull the image + pullsecretName: +# Controls the resources used by the fluentd daemonset resources: fluent: limits: @@ -100,7 +127,7 @@ resources: memory: 300Mi - # Controls the output buffer for fluentd. + # Controls the output buffer for fluentd for the metrics pod # Note that, for memory buffer, if `resources.sidecar.limits.memory` is set, # the total buffer size should not bigger than the memory limit, it should also # consider the basic memory usage by fluentd itself. @@ -118,6 +145,7 @@ buffer: retry_max_times: 3 + # Controls the output buffer for fluentd for the metrics aggregator pod aggregatorBuffer: "@type": memory total_limit_size: 400m @@ -133,6 +161,7 @@ aggregatorBuffer: metricsInterval: 15s +# Defines which nodes should be selected to deploy the fluentd daemonset. nodeSelector: {} # kubernetes.io/role: master @@ -153,6 +182,7 @@ aggregatorTolerations: {} priorityClassName: +# Defines node affinity to restrict pod deployment. affinity: {} # = Kubernetes Connection Configs = diff --git a/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl index 63b98795..770b5f79 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-objects/templates/_helpers.tpl @@ -81,3 +81,10 @@ Rules: {{- $mem -}} {{- end -}} {{- end -}} + +{{/* +Create the image name +*/}} +{{- define "splunk-kubernetes-objects.image" -}} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.name .Values.image.tag -}} +{{- end -}} \ No newline at end of file diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index ef1bd498..65a1bd54 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -39,7 +39,7 @@ spec: {{- end }} containers: - name: splunk-fluentd-k8s-objects - image: {{ .Values.image.name }} + image: {{ template "splunk-kubernetes-objects.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: diff --git a/helm-chart/splunk-kubernetes-objects/templates/serviceAccount.yaml b/helm-chart/splunk-kubernetes-objects/templates/serviceAccount.yaml index cd5d7174..2ebb5e55 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/serviceAccount.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/serviceAccount.yaml @@ -1,5 +1,9 @@ {{- if .Values.serviceAccount.create -}} apiVersion: v1 +{{ if .Values.image.usePullSecret }} +imagePullSecrets: +- name: {{ .Values.image.pullSecretName }} +{{ end }} kind: ServiceAccount metadata: name: {{ template "splunk-kubernetes-objects.serviceAccountName" . }} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 721dfde4..e944750a 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -1,4 +1,4 @@ -# Default values for splunk-kubernetes-logging. +# Default values for splunk-kubernetes-objects. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -44,6 +44,9 @@ serviceAccount: # The name of the ServiceAccount to use. # If not set and create is true, a name is generated using the fullname template name: + # This flag specifies if the user wants to use a secret for creating the serviceAccount, + # which will be used to get the images from a private registry + usePullSecrets: false # = Kubernetes Connection Configs = @@ -173,9 +176,20 @@ secret: name: +# Defines which version of image to use, and how it should be pulled. image: - name: splunk/kube-objects:1.1.0 - pullPolicy: IfNotPresent + # The domain of the registry to pull the image from + registry: docker.io + # The name of the image to pull + name: splunk/kube-objects + # The tag of the image to pull + tag: 1.1.2 + # The policy that specifies when the user wants the images to be pulled + pullPolicy: Always + # Indicates if the image should be pulled using authentication from a secret + usePullSecret: false + # The name of the pull secret to attach to the respective serviceaccount used to pull the image + pullSecretName: # = Resoruce Limitation Configs = @@ -206,12 +220,16 @@ buffer: retry_max_times: 3 +# Defines which nodes should be selected to deploy the fluentd daemonset. nodeSelector: {} +# This default tolerations allow the daemonset to be deployed on master nodes, +# so that we can also collect metrics from those nodes. tolerations: [] +# Defines node affinity to restrict pod deployment. affinity: {} From dbbf0b67414d7008cd14e323e16b31e5fa45929e Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 20 Aug 2019 10:42:08 -0700 Subject: [PATCH 103/119] openshift fix (#226) --- helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml | 1 - helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml | 1 - .../templates/deploymentMetricsAggregator.yaml | 1 - helm-chart/splunk-kubernetes-objects/templates/deployment.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 0b96e1ce..4938a113 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -40,7 +40,6 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: - {{ toYaml . | indent 12 }} privileged: true runAsUser: 0 {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml index d089f1c2..90211898 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deployment.yaml @@ -45,7 +45,6 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: - {{ toYaml . | indent 12 }} privileged: true runAsUser: 0 {{- end }} diff --git a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index 0403cdd8..52668077 100644 --- a/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -41,7 +41,6 @@ spec: imagePullPolicy: {{ .Values.imageAgg.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: - {{ toYaml . | indent 12 }} privileged: true runAsUser: 0 {{- end }} diff --git a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml index 65a1bd54..8a7d8367 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/deployment.yaml @@ -43,7 +43,6 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if or .Values.global.kubernetes.openshift .Values.kubernetes.openshift }} securityContext: - {{ toYaml . | indent 12 }} privileged: true runAsUser: 0 {{- end }} From f715eb64afb5687d4d16f5ac4c7bace5e0b2eb6f Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 30 Aug 2019 09:26:56 -0700 Subject: [PATCH 104/119] index-routing-objects (#235) --- .../templates/configMap.yaml | 3 +++ .../splunk-kubernetes-objects/values.yaml | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index 4788daeb..053cd20b 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -112,6 +112,9 @@ data: host "#{ENV['NODE_NAME']}" source_key source sourcetype_key sourcetype + {{- if or .Values.splunk.hec.indexRouting .Values.global.splunk.hec.indexRouting }} + index_key index + {{- else }} {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} index {{ . }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index e944750a..330da0dc 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -15,6 +15,15 @@ global: protocol: https port: 8088 insecureSSL: false + # indexRouting is a boolean, it indicates whether user wants to route logs to the index with name specified by the user using customFilters. Default is false. + # If you want to use this feature you will have to set the index key for the record using customFilters. + # For example, + # customFilters: + # SetIndexFilter: + # tag: tail.containers.** + # type: jq_transformer + # body: jq '.record.index = "my_awesome_index" | .record' + indexRouting: kubernetes: clusterName: "cluster_name" @@ -168,6 +177,15 @@ splunk: caFile: # The path to a directory containing CA certificates which are in PEM format. caPath: + # indexRouting is a boolean, it indicates whether user wants to route logs to the index with name specified by the user using customFilters. Default is false. + # If you want to use this feature you will have to set the index key for the record using customFilters. + # For example, + # customFilters: + # SetIndexFilter: + # tag: tail.containers.** + # type: jq_transformer + # body: jq '.record.index = "my_awesome_index" | .record' + indexRouting: # Create or use existing secret if name is empty default name is used From dab24ffdf325dcda81323b66175b7e14e7ee3e5a Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 30 Aug 2019 09:27:04 -0700 Subject: [PATCH 105/119] =?UTF-8?q?updated=20configuration=20for=20systemd?= =?UTF-8?q?=20plugin=20to=20use=20matches=20as=20filte=E2=80=A6=20(#233)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helm-chart/splunk-kubernetes-logging/templates/configMap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 043dfc97..045188e0 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -121,7 +121,7 @@ data: @label @SPLUNK tag journald.{{ or $logDef.sourcetype $name }} path {{ $.Values.journalLogPath | quote }} - filters [{ "_SYSTEMD_UNIT": {{ $logDef.from.journald.unit | quote }} }] + matches [{ "_SYSTEMD_UNIT": {{ $logDef.from.journald.unit | quote }} }] read_from_head true @type local From 6a879dad74a426c964283ba16189de5bda514d1e Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 30 Aug 2019 09:27:11 -0700 Subject: [PATCH 106/119] Add support for kube apiserver audit logs (#232) --- helm-chart/splunk-kubernetes-logging/values.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 9479ccfa..c48e28f9 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -258,6 +258,13 @@ logs: pod: kube-dns-autoscaler container: autoscaler sourcetype: kube:kube-dns-autoscaler + kube-audit: + from: + file: + path: /var/log/kube-apiserver-audit.log + timestampExtraction: + format: "%Y-%m-%dT%H:%M:%SZ" + sourcetype: kube:apiserver-audit # Defines which version of image to use, and how it should be pulled. From 6424ac3e1a654709ede88a6b09c4c86313f30d27 Mon Sep 17 00:00:00 2001 From: Stu Charlton Date: Thu, 5 Sep 2019 15:25:33 -0600 Subject: [PATCH 107/119] Missing end clause (#238) --- helm-chart/splunk-kubernetes-objects/templates/configMap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index 053cd20b..e6824b78 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -118,6 +118,7 @@ data: {{- with or .Values.splunk.hec.indexName .Values.global.splunk.hec.indexName }} index {{ . }} {{- end }} + {{- end }} insecure_ssl {{ or .Values.splunk.hec.insecureSSL .Values.global.splunk.hec.insecureSSL }} {{- if or .Values.splunk.hec.clientCert .Values.global.splunk.hec.clientCert }} client_cert /fluentd/etc/splunk/hec_client_cert From 097175c71fcbb23f501b6dca238f289e77e9432d Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 10 Sep 2019 10:31:02 -0700 Subject: [PATCH 108/119] added indexed fields to logging and objects (#231) --- .../examples/mini_custom_metadata.yaml | 2 +- .../templates/_helpers.tpl | 4 ++-- .../templates/configMap.yaml | 7 ++++-- .../splunk-kubernetes-logging/values.yaml | 23 ++++++++++++++++++- .../templates/configMap.yaml | 3 +++ .../splunk-kubernetes-objects/values.yaml | 19 +++++++++++++++ 6 files changed, 52 insertions(+), 6 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml b/helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml index 7f446634..094851da 100644 --- a/helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml +++ b/helm-chart/splunk-kubernetes-logging/examples/mini_custom_metadata.yaml @@ -6,7 +6,7 @@ splunk: kubernetes: clusterName: "my-eks-cluster" -custom_metadata: +customMetadata: - name: "cloud_account_id" value: "1234567890" - name: "cloud_account_region" diff --git a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl index 3fef8e02..27a75279 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl +++ b/helm-chart/splunk-kubernetes-logging/templates/_helpers.tpl @@ -113,8 +113,8 @@ def extract_container_info: | .container_name = ($cparts[:-1] | join("-")) | .container_id = ($cparts[-1] | rtrimstr(".log")) | .cluster_name = "{{ or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName | default "cluster_name" }}" - {{- if .Values.custom_metadata }} - {{- range .Values.custom_metadata }} + {{- if .Values.customMetadata }} + {{- range .Values.customMetadata }} | .{{ .name }} = "{{ .value }}" {{- end }} {{- end }} diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 045188e0..ff9581fa 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -274,11 +274,14 @@ data: {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{- end }} - {{- if .Values.custom_metadata }} - {{- range .Values.custom_metadata }} + {{- if .Values.customMetadata }} + {{- range .Values.customMetadata }} {{ .name }} {{- end }} {{- end }} + {{- range .Values.indexFields }} + {{ . }} + {{- end }} {{- with .Values.buffer }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index c48e28f9..d157a512 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -339,7 +339,7 @@ kubernetes: # List of key/value pairs for metadata purpse. # Can be used to define things such as cloud_account_id, cloud_account_region, etc. -custom_metadata: +customMetadata: # - name: "cloud_account_id" # value: "1234567890" @@ -379,3 +379,24 @@ custom_metadata: # body: jq '.record.cluster_name = "my_awesome_cluster" | .record' # ``` customFilters: {} + + +# +# You can find more information on indexed fields here - http://dev.splunk.com/view/event-collector/SP-CAAAFB6 +# The scheme to define an indexed field is: +# +# ``` +# ["field_1", "field_2"] +# ``` +# +# `indexFields` defines the fields from the fluentd record to be indexed. +# You can find more information on indexed fields here - http://dev.splunk.com/view/event-collector/SP-CAAAFB6 +# The input is in the form of an array(comma separated list) of the values you want to use as indexed fields. +# +# For example if you want to define indexed fields for "field_1" and "field_2" +# you will have to define an indexFields section as follows in values.yaml file. +# ``` +# indexFields: ["field_1", "field_2"] +# ``` +# WARNING: The fields being used here must be available inside the fluentd record. +indexFields: [] diff --git a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml index e6824b78..5155d013 100644 --- a/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-objects/templates/configMap.yaml @@ -133,6 +133,9 @@ data: {{- if or .Values.kubernetes.clusterName .Values.global.kubernetes.clusterName }} cluster_name {{- end }} + {{- range .Values.indexFields }} + {{ . }} + {{- end }} {{- with .Values.buffer }} diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index 330da0dc..f280f7e0 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -287,3 +287,22 @@ affinity: {} # body: jq '.record.cluster_name = "my_awesome_cluster" | .record' # ``` customFilters: {} +# +# You can find more information on indexed fields here - http://dev.splunk.com/view/event-collector/SP-CAAAFB6 +# The scheme to define an indexed field is: +# +# ``` +# ["field_1", "field_2"] +# ``` +# +# `indexFields` defines the fields from the fluentd record to be indexed. +# You can find more information on indexed fields here - http://dev.splunk.com/view/event-collector/SP-CAAAFB6 +# The input is in the form of an array(comma separated list) of the values you want to use as indexed fields. +# +# For example if you want to define indexed fields for "field_1" and "field_2" +# you will have to define an indexFields section as follows in values.yaml file. +# ``` +# indexFields: ["field_1", "field_2"] +# ``` +# WARNING: The fields being used here must be available inside the fluentd record. +indexFields: [] From 5983bb36fad613c9ba62fed6fdf5c9eb34ce8ea4 Mon Sep 17 00:00:00 2001 From: Darren Foo Date: Mon, 7 Oct 2019 08:44:45 -0700 Subject: [PATCH 109/119] remove version label from selector in manifests (#229) * remove version label from selector in manifests * add linux node selector to helm templates for #182 --- helm-chart/splunk-kubernetes-logging/values.yaml | 3 ++- helm-chart/splunk-kubernetes-metrics/values.yaml | 4 ++-- helm-chart/splunk-kubernetes-objects/values.yaml | 3 ++- manifests/splunk-kubernetes-logging/daemonset.yaml | 1 - manifests/splunk-kubernetes-metrics/deployment.yaml | 1 - manifests/splunk-kubernetes-objects/deployment.yaml | 1 - tools/gen_manifest.rb | 1 - 7 files changed, 6 insertions(+), 8 deletions(-) diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index d157a512..6f19d19a 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -319,7 +319,8 @@ tolerations: # Defines which nodes should be selected to deploy the fluentd daemonset. -nodeSelector: {} +nodeSelector: + beta.kubernetes.io/os: linux # Defines node affinity to restrict pod deployment. diff --git a/helm-chart/splunk-kubernetes-metrics/values.yaml b/helm-chart/splunk-kubernetes-metrics/values.yaml index dd3d6046..3863ad17 100644 --- a/helm-chart/splunk-kubernetes-metrics/values.yaml +++ b/helm-chart/splunk-kubernetes-metrics/values.yaml @@ -162,8 +162,8 @@ metricsInterval: 15s # Defines which nodes should be selected to deploy the fluentd daemonset. -nodeSelector: {} - # kubernetes.io/role: master +nodeSelector: + beta.kubernetes.io/os: linux # This default tolerations allow the daemonset to be deployed on master nodes, diff --git a/helm-chart/splunk-kubernetes-objects/values.yaml b/helm-chart/splunk-kubernetes-objects/values.yaml index f280f7e0..2e0f3c75 100644 --- a/helm-chart/splunk-kubernetes-objects/values.yaml +++ b/helm-chart/splunk-kubernetes-objects/values.yaml @@ -239,7 +239,8 @@ buffer: # Defines which nodes should be selected to deploy the fluentd daemonset. -nodeSelector: {} +nodeSelector: + beta.kubernetes.io/os: linux # This default tolerations allow the daemonset to be deployed on master nodes, diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index fc1f10db..26f207c4 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -11,7 +11,6 @@ spec: selector: matchLabels: app: splunk-kubernetes-logging - version: 1.2.0 template: metadata: labels: diff --git a/manifests/splunk-kubernetes-metrics/deployment.yaml b/manifests/splunk-kubernetes-metrics/deployment.yaml index ffbab357..27f8f45b 100644 --- a/manifests/splunk-kubernetes-metrics/deployment.yaml +++ b/manifests/splunk-kubernetes-metrics/deployment.yaml @@ -11,7 +11,6 @@ spec: selector: matchLabels: name: splunk-kubernetes-metrics - version: 1.2.0 template: metadata: name: splunk-kubernetes-metrics diff --git a/manifests/splunk-kubernetes-objects/deployment.yaml b/manifests/splunk-kubernetes-objects/deployment.yaml index 1a83e428..4bc20fc7 100644 --- a/manifests/splunk-kubernetes-objects/deployment.yaml +++ b/manifests/splunk-kubernetes-objects/deployment.yaml @@ -11,7 +11,6 @@ spec: matchLabels: app: splunk-kubernetes-objects engine: fluentd - version: 1.2.0 replicas: 1 template: metadata: diff --git a/tools/gen_manifest.rb b/tools/gen_manifest.rb index b575b1d7..db9bb7c8 100644 --- a/tools/gen_manifest.rb +++ b/tools/gen_manifest.rb @@ -21,7 +21,6 @@ def sanitize_yaml(yaml) yaml['metadata']['labels']['version'] = @version yaml.fetch('spec', {}).tap { |spec| - spec.fetch('selector', {}).fetch('matchLabels', {})['version'] = @version spec.fetch('template', {}).fetch('metadata', {}).fetch('labels', {})['version'] = @version } From 96d5c4985e30a9b20a5ba9c4b33ea03ecd82e7b3 Mon Sep 17 00:00:00 2001 From: nokesc <50254537+nokesc@users.noreply.github.com> Date: Mon, 7 Oct 2019 11:23:52 -0500 Subject: [PATCH 110/119] =?UTF-8?q?For=20multiline=20events,=20use=20the?= =?UTF-8?q?=20timestamp=20of=20the=20first=20event=20and=20d=E2=80=A6=20(#?= =?UTF-8?q?244)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helm-chart/splunk-kubernetes-logging/templates/configMap.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index ff9581fa..5ff654d2 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -158,6 +158,8 @@ data: stream_identity_key stream multiline_start_regexp {{ $logDef.multiline.firstline }} flush_interval {{ $logDef.multiline.flushInterval | default "5s" }} + separator "" + use_first_timestamp true {{- end }} {{- end }} From 0fbf61753109f15c76d498cdf116c5b6f510f74c Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 8 Oct 2019 07:29:22 -0700 Subject: [PATCH 111/119] abstract HEC info (#254) --- .../splunk-connect-for-kubernetes/examples/insecure.yaml | 8 ++++---- .../splunk-connect-for-kubernetes/examples/mini.yaml | 6 +++--- .../splunk-kubernetes-logging/examples/use_logs.yaml | 2 +- helm-chart/splunk-kubernetes-metrics/examples/mini.yaml | 2 +- helm-chart/splunk-kubernetes-objects/examples/full.yaml | 4 ++-- helm-chart/splunk-kubernetes-objects/examples/mini.yaml | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml b/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml index 54b6a789..38d24a10 100644 --- a/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/examples/insecure.yaml @@ -2,13 +2,13 @@ global: splunk: hec: - token: 9bca7e2a-0a4b-5b75-b63f-900fac026807 - host: 64.129.124.21 - port: 8088 + token: my-hec-token + host: my-hec-host + port: my-hec-port indexName: main # connection to splunk is insecure insecureSSL: true - kubernetes: + kubernetes: clusterName: "cluster_name" splunk-kubernetes-objects: diff --git a/helm-chart/splunk-connect-for-kubernetes/examples/mini.yaml b/helm-chart/splunk-connect-for-kubernetes/examples/mini.yaml index 42eb6d70..52284bcd 100644 --- a/helm-chart/splunk-connect-for-kubernetes/examples/mini.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/examples/mini.yaml @@ -1,9 +1,9 @@ global: splunk: hec: - token: 7fbefa0b-8c06-4a62-a54b-0fa69c016988 - host: 45.209.241.112 - port: 8088 + token: my-hec-token + host: my-hec-host + port: my-hec-port indexName: main diff --git a/helm-chart/splunk-kubernetes-logging/examples/use_logs.yaml b/helm-chart/splunk-kubernetes-logging/examples/use_logs.yaml index 3b2ae186..530d08e3 100644 --- a/helm-chart/splunk-kubernetes-logging/examples/use_logs.yaml +++ b/helm-chart/splunk-kubernetes-logging/examples/use_logs.yaml @@ -1,7 +1,7 @@ splunk: hec: token: 11111111-2222-3333-4444-555555555555 - host: 12.34.56.78 + host: my-hec-host logs: # we want to read logs for `kube-apiserver` from a log file other then from the container logs diff --git a/helm-chart/splunk-kubernetes-metrics/examples/mini.yaml b/helm-chart/splunk-kubernetes-metrics/examples/mini.yaml index d2fd97ae..88f24793 100644 --- a/helm-chart/splunk-kubernetes-metrics/examples/mini.yaml +++ b/helm-chart/splunk-kubernetes-metrics/examples/mini.yaml @@ -1,4 +1,4 @@ splunk: hec: token: 11111111-2222-3333-4444-555555555555 - host: 12.34.56.78 + host: my-hec-host diff --git a/helm-chart/splunk-kubernetes-objects/examples/full.yaml b/helm-chart/splunk-kubernetes-objects/examples/full.yaml index 32deb546..3fcf904c 100644 --- a/helm-chart/splunk-kubernetes-objects/examples/full.yaml +++ b/helm-chart/splunk-kubernetes-objects/examples/full.yaml @@ -6,8 +6,8 @@ rbac: splunk: hec: token: 11111111-2222-3333-4444-555555555555 - host: 12.34.56.78 - ndexName: myIndex + host: my-hec-host + indexName: myIndex insecureSSL: false diff --git a/helm-chart/splunk-kubernetes-objects/examples/mini.yaml b/helm-chart/splunk-kubernetes-objects/examples/mini.yaml index d2fd97ae..88f24793 100644 --- a/helm-chart/splunk-kubernetes-objects/examples/mini.yaml +++ b/helm-chart/splunk-kubernetes-objects/examples/mini.yaml @@ -1,4 +1,4 @@ splunk: hec: token: 11111111-2222-3333-4444-555555555555 - host: 12.34.56.78 + host: my-hec-host From 3470aa431ae875fd3bd581de8ac9db21c7065eba Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 14 Oct 2019 10:16:19 -0700 Subject: [PATCH 112/119] ingest api support (#253) * ingest api support * add docs --- README.md | 9 ++++++ .../templates/configMap.yaml | 31 +++++++++++++++++-- .../splunk-kubernetes-logging/values.yaml | 16 ++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2b43cc56..6795f210 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,15 @@ Consider the following kubernetes namespace to splunk index topology. * default -> indexRoutingDefaultIndex For this topology to work appropriately we have to create the splunk indexes "kube-system", "kube-public" and the value of indexRoutingDefaultIndex. +# Sending logs to ingest API +Splunk Connect for Kubernetes can be used to send events to [Splunk Ingest API](https://sdc.splunkbeta.com/reference/api/ingest/v1beta2). In the ingest_api section of the yaml file you are using to deploy, the following configuration options have to be configured:
+* serviceClientIdentifier - Splunk Connect for Kubernetes uses the client identifier to make authorized requests to the ingest API. +* serviceClientSecretKey - Splunk Connect for Kubernetes uses the client secret key to make authorized requests to the ingest API. +* tokenEndpoint - This value indicates which endpoint Splunk Connect for Kubernetes should look to for the authorization token necessary for making requests to the ingest API. +* ingestAPIHost - Indicates which url/hostname to use for requests to the ingest API. +* tenant - Indicates which tenant Splunk Connect for Kubernetes should use for requests to the ingest API. +* eventsEndpoint - Indicates which endpoint to use for requests to the ingest API. +* debugIngestAPI - Set to True if you want to debug requests and responses to ingest API. # Maintenance And Support Splunk Connect For Kubernetes is supported through Splunk Support assuming the customer has a current Splunk support entitlement. For customers that do not have a current Splunk support entitlement, please file an issue at create a new issue at [Create a new issue in splunk connect for kubernetes project](https://github.com/splunk/splunk-connect-for-kubernetes/issues/new) diff --git a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml index 5ff654d2..4b8b3882 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/configMap.yaml @@ -228,6 +228,7 @@ data: # = output = + {{- if .Values.splunk.hec }} @type splunk_hec {{- with or .Values.splunk.hec.protocol .Values.global.splunk.hec.protocol }} protocol {{ . }} @@ -239,9 +240,6 @@ data: hec_port {{ . }} {{- end }} hec_token "#{ENV['SPLUNK_HEC_TOKEN']}" - host "#{ENV['NODE_NAME']}" - source_key source - sourcetype_key sourcetype {{- if or .Values.splunk.hec.indexRouting .Values.global.splunk.hec.indexRouting }} index_key index {{- else }} @@ -259,6 +257,33 @@ data: {{- if or .Values.splunk.hec.caFile .Values.global.splunk.hec.caFile }} ca_file /fluentd/etc/splunk/hec_ca_file {{- end }} + {{ else }} + @type splunk_ingest_api + {{- with or .Values.splunk.ingest_api.serviceClientIdentifier }} + service_client_identifier {{ . }} + {{- end }} + {{- with or .Values.splunk.ingest_api.serviceClientSecretKey }} + service_client_secret_key {{ . }} + {{- end }} + {{- with or .Values.splunk.ingest_api.tokenEndpoint }} + token_endpoint {{ . }} + {{- end }} + {{- with or .Values.splunk.ingest_api.ingestAPIHost }} + ingest_api_host {{ . }} + {{- end }} + {{- with or .Values.splunk.ingest_api.tenant }} + ingest_api_tenant {{ . }} + {{- end }} + {{- with or .Values.splunk.ingest_api.eventsEndpoint }} + ingest_api_events_endpoint {{ . }} + {{- end }} + {{- with or .Values.splunk.ingest_api.debugIngestAPI }} + debug_http {{ . }} + {{- end }} + {{- end }} + host "#{ENV['NODE_NAME']}" + source_key source + sourcetype_key sourcetype # currently CRI does not produce log paths with all the necessary # metadata to parse out pod, namespace, container_name, container_id. diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 6f19d19a..023914dc 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -97,6 +97,22 @@ splunk: indexRouting: # indexRoutingDefaultIndex tells which index to use for the default kubernetes namespace. Used with indexRouting. This value is required when using indexRouting. indexRoutingDefaultIndex: + # Configurations for Ingest API + ingest_api: + # serviceClientIdentifier is a string, the client identifier is used to make requests to the ingest API with authorization. + serviceClientIdentifier: + # serviceClientSecretKey is a string, the client identifier is used to make requests to the ingest API with authorization. + serviceClientSecretKey: + # tokenEndpoint is a string, it indicates which endpoint should be used to get the authorization token used to make requests to the ingest API. + tokenEndpoint: + # ingestAPIHost is a string, it indicates which url/hostname should be used to make requests to the ingest API. + ingestAPIHost: + # tenant is a string, it indicates which tenant should be used to make requests to the ingest API. + tenant: + # eventsEndpoint is a string, it indicates which endpoint should be used to make requests to the ingest API. + eventsEndpoint: + # debugIngestAPI is a boolean, it indicates whether user wants to debug requests and responses to ingest API. Default is false. + debugIngestAPI: false # Create or use existing secret if name is empty default name is used From 1a7d0bf1acfb4bb465017a7acce32c7d3c51e39b Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak <5203740+chaitanyaphalak@users.noreply.github.com> Date: Tue, 5 Nov 2019 09:22:29 -0800 Subject: [PATCH 113/119] Firelens (#260) * add firelens usage info * add firelens to main readme --- README.md | 2 +- firelens/README.md | 156 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 firelens/README.md diff --git a/README.md b/README.md index 6795f210..aae6934a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/splunk-connect-for-kubernetes) # What does Splunk Connect for Kubernetes do? -Splunk Connect for Kubernetes provides a way to import and search your Kubernetes logging, object, and metrics data in Splunk. Splunk is a proud contributor to Cloud Native Computing Foundation (CNCF) and Splunk Connect for Kubernetes utilizes and supports multiple CNCF components in the development of these tools to get data into Splunk. +Splunk Connect for Kubernetes provides a way to import and search your Kubernetes logging, object, and metrics data in Splunk. Now, Splunk Connect for Kubenetes also supports [importing and searching your container logs on AWS ECS and AWS Fargate using firelens.](https://github.com/splunk/splunk-connect-for-kubernetes/tree/develop/firelens) Splunk is a proud contributor to Cloud Native Computing Foundation (CNCF) and Splunk Connect for Kubernetes utilizes and supports multiple CNCF components in the development of these tools to get data into Splunk. ## Prerequisites diff --git a/firelens/README.md b/firelens/README.md new file mode 100644 index 00000000..4fddd6ce --- /dev/null +++ b/firelens/README.md @@ -0,0 +1,156 @@ +# Splunk Connect for AWS ECS and AWS Fargate + +Splunk Connect for Kubernetes provides an integration for sending logs to Splunk from AWS ECS and AWS Fargate using [firelens.](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html)
+The fluentd logging component from Splunk Connect for Kubernetes is used to achieve this goal using [Splunk's fluentd HEC plugin.](https://github.com/splunk/fluent-plugin-splunk-hec) + +# Enabling Firelens for Splunk Connect for AWS ECS and AWS Fargate + +### 1. Configure the fluent.conf file to be used with Splunk's firelens log router container +``` + + log_level info + + + + @type splunk_hec + protocol https + hec_host my-hec-host + hec_port my-hec-port + hec_token my-hec-token + index my-index + host_key ec2_instance_id + source_key ecs_cluster + sourcetype_key ecs_task_definition + insecure_ssl true + + container_id + container_name + ecs_task_arn + source + + + @type single_value + message_key log + add_newline false + + +``` +The fluent.conf file is used to specify the configuration for [Splunk's HEC REST endpoint.](http://dev.splunk.com/view/event-collector/SP-CAAAE7F) Configure it as follows:
+* `protocol` - The protocol to be used (HTTP/HTTPS) +* `hec_host` - The host exposing Splunk's HEC endpoint +* `hec_token` - The HEC token used for authorizing requests being made to Splunk's HEC endpoint +* `hec_port` - The port configured for use with the above `hec_token` +* `index` - Target Splunk indexes to send data to + +Store this `fluent.conf` in an S3 bucket to be used in the next step. + +### 2. Configure Splunk's firelens log router container +```json +{ + "essential":true, + "image":"splunk/fluentd-hec:firelens-1.0", + "name":"log_router", + "firelensConfiguration":{ + "type":"fluentd", + "options":{ + "config-file-type":"s3", + "config-file-value":"arn:aws:s3:::my-aws-bucket/fluent.conf" + } + } +} +``` +Add the above container definition alongside your application container definition. The following options should be configured and updated appropriately:
+* Set the `image` to the relevant Splunk's firelens log router container +* Set the `config-file-value` to the location of your fluent.conf file in s3 + + +### 3. Configure the application container +In your application container definition add the following log configuration section so that your application can use firelens and Splunk's firelens log router. +```json +"logConfiguration":{ + "logDriver":"awsfirelens" +} +``` + +### 4. Configure Cloudwatch logs for Splunk's firelens log router container (Optional step/Useful for troubleshooting) +To monitor the logs of Splunk's firelens log router container you can configure a log configuration in Splunk's firelens log router container definition as follows: +```json +"logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "my-log-group", + "awslogs-region": "my-aws-region", + "awslogs-stream-prefix": "my-stream-prefix" + } +} +``` +Configure the options for `awslogs-group`, `awslogs-region` and `awslogs-stream-prefix` appropriately. The logDriver can be [awslogs](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_awslogs.html) or [awsfirelens.](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html) +This results in a container definition as follows: +```json +{ + "essential":true, + "image":"splunk/fluentd-hec:firelens-1.0", + "name":"log_router", + "firelensConfiguration":{ + "type":"fluentd", + "options":{ + "config-file-type":"s3", + "config-file-value":"arn:aws:s3:::my-aws-bucket/fluent.conf" + } + }, + "logConfiguration":{ + "logDriver":"awslogs", + "options":{ + "awslogs-group":"my-log-group", + "awslogs-region":"my-aws-region", + "awslogs-stream-prefix":"my-stream-prefix" + } + } +} +``` + +### 5. Example Task Definition for reference +Following is an example task definition with an nginx web server accessible on port 80. All the logs from this nginx web server will be sent to the Splunk instance configured in the fluent.conf file. +```json +{ + "family":"firelens-ec2-demo-s3", + "containerDefinitions":[ + { + "essential":true, + "image":"splunk/fluentd-hec:firelens-1.0", + "name":"log_router", + "firelensConfiguration":{ + "type":"fluentd", + "options":{ + "config-file-type":"s3", + "config-file-value":"arn:aws:s3:::my-aws-bucket/fluent.conf" + } + }, + "logConfiguration":{ + "logDriver":"awslogs", + "options":{ + "awslogs-group":"debug-fluentd", + "awslogs-region":"us-west-2", + "awslogs-stream-prefix":"test" + } + } + }, + { + "essential":true, + "name":"nginx-example", + "image":"nginx:1.17", + "essential":true, + "portMappings":[ + { + "hostPort":80, + "protocol":"tcp", + "containerPort":80 + } + ], + "logConfiguration":{ + "logDriver":"awsfirelens" + } + } + ] +} +``` \ No newline at end of file From b099aaa53ca41a98bc4ab1084f13d049add8a115 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 5 Nov 2019 10:02:25 -0800 Subject: [PATCH 114/119] add latest hec image to firelens readme (#280) --- firelens/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/firelens/README.md b/firelens/README.md index 4fddd6ce..b6cef129 100644 --- a/firelens/README.md +++ b/firelens/README.md @@ -48,7 +48,7 @@ Store this `fluent.conf` in an S3 bucket to be used in the next step. ```json { "essential":true, - "image":"splunk/fluentd-hec:firelens-1.0", + "image":"splunk/fluentd-hec:1.2.0", "name":"log_router", "firelensConfiguration":{ "type":"fluentd", @@ -89,7 +89,7 @@ This results in a container definition as follows: ```json { "essential":true, - "image":"splunk/fluentd-hec:firelens-1.0", + "image":"splunk/fluentd-hec:1.2.0", "name":"log_router", "firelensConfiguration":{ "type":"fluentd", @@ -117,7 +117,7 @@ Following is an example task definition with an nginx web server accessible on p "containerDefinitions":[ { "essential":true, - "image":"splunk/fluentd-hec:firelens-1.0", + "image":"splunk/fluentd-hec:1.2.0", "name":"log_router", "firelensConfiguration":{ "type":"fluentd", @@ -153,4 +153,4 @@ Following is an example task definition with an nginx web server accessible on p } ] } -``` \ No newline at end of file +``` From 32bcc6fbcb642012bdf1f8e3d7a609cdad8fd7de Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 5 Nov 2019 10:23:02 -0800 Subject: [PATCH 115/119] add latest hec image to firelens readme (#280) From 839df5da6572f1c8b78ed5f93372defc448eaf99 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Wed, 6 Nov 2019 09:15:50 -0800 Subject: [PATCH 116/119] trigger functional test from sub plugin repos (#279) --- .circleci/build_sck_artifacts.sh | 3 +- .circleci/build_sck_artifacts_for_release.sh | 20 ++- .circleci/check_version_trigger_release.sh | 24 +++ .circleci/config.yml | 164 ++++++++++++++---- .circleci/deploy_connector.sh | 19 +- .../performance/perf_test_sck_values.yml | 4 +- .circleci/run_functional_tests.sh | 9 +- .circleci/sck_values.yml | 4 +- .circleci/update_sck_config.sh | 78 ++++++--- .circleci/update_sck_config_for_release.sh | 26 --- Makefile | 8 + PLUGIN_VERSIONS.sh | 5 + .../templates/daemonset.yaml | 2 + .../splunk-kubernetes-logging/values.yaml | 13 ++ test/k8s_logging_tests/test_config_logging.py | 6 +- tools/update_charts_version.sh | 29 ++++ 16 files changed, 297 insertions(+), 117 deletions(-) create mode 100755 .circleci/check_version_trigger_release.sh delete mode 100755 .circleci/update_sck_config_for_release.sh create mode 100755 PLUGIN_VERSIONS.sh create mode 100755 tools/update_charts_version.sh diff --git a/.circleci/build_sck_artifacts.sh b/.circleci/build_sck_artifacts.sh index 683f9eec..4d0e3853 100755 --- a/.circleci/build_sck_artifacts.sh +++ b/.circleci/build_sck_artifacts.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -e -TAG="1.1.0.Alpha-${CIRCLE_SHA1}" +VERSION=`cat VERSION` +TAG="$VERSION-${CIRCLE_SHA1}" function replace_generic_version () { diff --git a/.circleci/build_sck_artifacts_for_release.sh b/.circleci/build_sck_artifacts_for_release.sh index d852e754..36b7cdba 100755 --- a/.circleci/build_sck_artifacts_for_release.sh +++ b/.circleci/build_sck_artifacts_for_release.sh @@ -1,16 +1,20 @@ #!/usr/bin/env bash set -e -TAG=`cat VERSION` +VERSION=`cat VERSION` + +source PLUGIN_VERSIONS.sh # Install yq yaml parser wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 sudo chmod +x yq_linux_amd64 sudo mv yq_linux_amd64 /usr/local/bin/yq -yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.name splunk/fluentd-hec:1.1.1 -yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.name splunk/k8s-metrics:1.1.1 -yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.name splunk/k8s-metrics-aggr:1.1.0 -yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.name splunk/kube-objects:1.1.0 +# Before release, "make manifests" call should have been run. +# then there are nothing else to change. but keeping them here just in case +yq w -i helm-chart/splunk-kubernetes-logging/values.yaml image.tag $FLUENTD_HEC_VERSION +yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml image.tag $K8S_METRICS_VERISION +yq w -i helm-chart/splunk-kubernetes-metrics/values.yaml imageAgg.tag $K8S_METRICS_AGGR_VERSION +yq w -i helm-chart/splunk-kubernetes-objects/values.yaml image.tag $KUBE_OBJECT_VERSION yq w -i helm-chart/splunk-connect-for-kubernetes/Chart.yaml version $VERSION yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml version $VERSION @@ -22,9 +26,9 @@ yq w -i helm-chart/splunk-kubernetes-logging/Chart.yaml appVersion $VERSION yq w -i helm-chart/splunk-kubernetes-metrics/Chart.yaml appVersion $VERSION yq w -i helm-chart/splunk-kubernetes-objects/Chart.yaml appVersion $VERSION -yq w -i helm-chart/splunk-kubernetes-logging/requirements.yaml dependencies.splunk-kubernetes-logging.version $VERSION -yq w -i helm-chart/splunk-kubernetes-metrics/requirements.yaml dependencies.splunk-kubernetes-metrics.version $VERSION -yq w -i helm-chart/splunk-kubernetes-objects/requirements.yaml dependencies.splunk-kubernetes-objects.version $VERSION +yq w -i helm-chart/splunk-connect-for-kubernetes/requirements.yaml dependencies[0].version $VERSION +yq w -i helm-chart/splunk-connect-for-kubernetes/requirements.yaml dependencies[1].version $VERSION +yq w -i helm-chart/splunk-connect-for-kubernetes/requirements.yaml dependencies[2].version $VERSION mkdir helm-artifacts-release if [[ -d "helm-chart/splunk-connect-for-kubernetes/charts" ]]; then diff --git a/.circleci/check_version_trigger_release.sh b/.circleci/check_version_trigger_release.sh new file mode 100755 index 00000000..77bfbdd2 --- /dev/null +++ b/.circleci/check_version_trigger_release.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e +LATEST_COMMIT=$(git rev-parse HEAD) +VERSION_COMMIT=$(git log -1 --format=format:%H VERSION) +if [ $VERSION_COMMIT = $LATEST_COMMIT ]; + then + if [ -s VERSION ] # Check if content is empty + then + VERSION=`cat VERSION` + echo "VERSION is changed to $VERSION" + else + echo "[ERROR] VERSION file is empty." + exit 1 + fi + git config user.email "splunk-oss-admin@splunk.com" + git config user.name "splunk-oss-admin" + git checkout develop + git pull origin develop + git checkout -b release/$VERSION origin/develop + git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/splunk-connect-for-kubernetes.git release/$VERSION + git checkout master + git merge --no-edit release/$VERSION + git push https://$RELEASE_GITHUB_USER:$RELEASE_GITHUB_PASS@github.com/splunk/splunk-connect-for-kubernetes.git master +fi diff --git a/.circleci/config.yml b/.circleci/config.yml index 0117824b..a07bb34e 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,93 @@ version: 2 jobs: - build: - docker: - - image: circleci/ruby:2.6.1-node-browsers + build_test: + resource_class: large + machine: + image: ubuntu-1604:201903-01 + environment: + CHANGE_MINIKUBE_NONE_USER: true + GOPATH: /home/circleci/go + KUBECONFIG: /home/circleci/.kube/config + KUBERNETES_VERSION: v1.15.2 + MINIKUBE_HOME: /home/circleci + MINIKUBE_VERSION: v1.2.0 + MINIKUBE_WANTUPDATENOTIFICATION: false + MINIKUBE_WANTREPORTERRORPROMPT: false + SKAFFOLD_VERSION: v0.33.0 + GO111MODULE: "on" + CI_SPLUNK_VERSION: "7.3.2" + CI_SPLUNK_FILENAME: splunk-7.3.2-c60db69f8e32-linux-2.6-amd64.deb + CI_SPLUNK_HOST: 127.0.0.1 + CI_SPLUNK_PORT: 8089 + CI_SPLUNK_USERNAME: admin + CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959 + CI_SPLUNK_HEC_TOKEN_PERF: a6b5e77f-d5f6-415a-bd43-930cecb12959 + CI_SPLUNK_PASSWORD: helloworld + CI_HEC_PROTOCOL: http + CI_INDEX_EVENTS: circleci_events + CI_INDEX_OBJECTS: circleci_objects + CI_INDEX_METRICS: circleci_metrics + CI_INDEX_NAMESPACE_ROUTING: test-namespace-routing + CI_KAFKA_DATAGEN_IMAGE: chaitanyaphalak/kafkadatagen:1.0-4-gca7f6d8 working_directory: ~/repo steps: - checkout - - setup_remote_docker: - reusable: true + - run: + name: Install Splunk + command: | + cd /tmp && wget -O $CI_SPLUNK_FILENAME 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version='$CI_SPLUNK_VERSION'&product=splunk&filename='$CI_SPLUNK_FILENAME'&wget=true' + sudo dpkg -i $CI_SPLUNK_FILENAME + # Set user seed + hashed_pwd=$(sudo /opt/splunk/bin/splunk hash-passwd $CI_SPLUNK_PASSWORD) + sudo tee /opt/splunk/etc/system/local/user-seed.conf > /dev/null << EOF + [user_info] + USERNAME = $CI_SPLUNK_USERNAME + HASHED_PASSWORD = $hashed_pwd + EOF + # Add delete capability to admin role + sudo tee -a /opt/splunk/etc/system/local/authorize.conf > /dev/null << EOF + [role_admin] + delete_by_keyword = enabled + EOF + # start Splunk + sudo /opt/splunk/bin/splunk start --accept-license --answer-yes --no-prompt + # Enable HEC services + curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable + # Create new HEC token + curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=$CI_SPLUNK_HEC_TOKEN" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http + # Enable HEC new-token + sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + # Disable SSL for HEC new-token + sudo /opt/splunk/bin/splunk http-event-collector update -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -enable-ssl 0 -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + # Setup Indexes + sudo /opt/splunk/bin/splunk add index $CI_INDEX_EVENTS -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + sudo /opt/splunk/bin/splunk add index $CI_INDEX_OBJECTS -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + sudo /opt/splunk/bin/splunk add index $CI_INDEX_METRICS -datatype metric -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + sudo /opt/splunk/bin/splunk add index kube-system -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + sudo /opt/splunk/bin/splunk add index kube-public -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + sudo /opt/splunk/bin/splunk add index $CI_INDEX_NAMESPACE_ROUTING -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD + sudo /opt/splunk/bin/splunk http-event-collector update -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -name splunk_hec_token -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -index $CI_INDEX_EVENTS -indexes $CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,$CI_INDEX_NAMESPACE_ROUTING,kube-system,kube-public + # Restart Splunk + sudo /opt/splunk/bin/splunk restart --accept-license --answer-yes --no-prompt + - run: + name: Install and Start minikube + command: | + # Install Kubectl + curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + mkdir -p ${HOME}/.kube + touch ${HOME}/.kube/config + # Install Minikube + curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 + chmod +x minikube + sudo mv minikube /usr/local/bin/ + # Start Minikube and Wait + sudo -E minikube start --vm-driver=none --cpus 2 --memory 4098 --kubernetes-version=${KUBERNETES_VERSION} + export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' + until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do + sleep 1; + done - run: name: Install dependencies command: | @@ -17,24 +97,24 @@ jobs: sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash -s -- --version v2.12.3 helm init -c - echo "Installing aws cli..." - sudo pip install awscli > /dev/null 2>&1 + docker pull $CI_KAFKA_DATAGEN_IMAGE + pyenv global 3.6.5 + pip install --upgrade pip + pip install -r test/requirements.txt - run: name: Lint helm charts command: | .circleci/lint.sh - - run: - name: Update configurables to build artifacts - command: | - .circleci/update_sck_config.sh - run: name: Build SCK artifacts command: | .circleci/build_sck_artifacts.sh - run: - name: Push SCK artifacts to s3 + name: Update configurables for unit test command: | - .circleci/push_sck_artifacts.sh + export CI_SPLUNK_HEC_HOST=$(hostname -i) + export CI_SPLUNK_HEC_HOST_PERF=$(hostname -i) + .circleci/update_sck_config.sh - run: name: Deploy k8s connector command: | @@ -42,11 +122,10 @@ jobs: - run: name: Run functional tests command: | + export CI_SPLUNK_HEC_HOST=$(hostname -i) + pyenv global 3.6.5 + export PYTHONWARNINGS="ignore:Unverified HTTPS request" .circleci/run_functional_tests.sh - - run: - name: Teardown deployment - command: | - helm delete --purge ci-$CIRCLE_SHA1 perf-test: docker: @@ -97,10 +176,6 @@ jobs: sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1 curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash helm init -c - - run: - name: Update configurables to build artifacts - command: | - .circleci/update_sck_config_for_release.sh - run: name: Build SCK artifacts command: | @@ -109,32 +184,49 @@ jobs: name: Push SCK artifacts to Github Release command: | .circleci/push_sck_release.sh + + check_version: + docker: + - image: circleci/ruby:2.6.1-node-browsers + working_directory: ~/repo + steps: + - attach_workspace: + at: /tmp + - checkout + - setup_remote_docker: + reusable: true - run: - name: Deploy k8s connector - command: | - .circleci/deploy_connector_release.sh - - run: - name: Run functional tests + name: Check VERSION file for change command: | - .circleci/run_functional_tests.sh - -# Need to add teardown step + .circleci/check_version_trigger_release.sh workflows: version: 2 - build_perf-test: + build_test: jobs: - - build: + - build_test: filters: branches: - ignore: perf-testing + ignore: + - perf-testing + - /^release\/.*/ + - master + - check_version: + requires: + - build_test + perf-test: + jobs: - perf-test: filters: branches: only: perf-testing - - release: - requires: - - build + release: + jobs: + - build_test: filters: branches: - only: release-testing + only: + - master + - release: + requires: + - build_test \ No newline at end of file diff --git a/.circleci/deploy_connector.sh b/.circleci/deploy_connector.sh index 96d29cf2..c435cdad 100755 --- a/.circleci/deploy_connector.sh +++ b/.circleci/deploy_connector.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash set -e -echo "Setup kube client..." -curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl -chmod +x ./kubectl -sudo mv ./kubectl /usr/local/bin/kubectl -sudo apt-get -y install gnupg -sudo mkdir ~/.kube -echo $GPG_KEY | gpg --output config --passphrase-fd 0 --decrypt --batch .circleci/kubeconfig.gpg -sudo mv config ~/.kube/config #Update helm server version helm init --force-upgrade +# Wait for helm to be ready +until kubectl get pod --all-namespaces | grep tiller | grep -q "1\/1"; do + sleep 1; +done #Make sure to check and clean previously failed deployment echo "Checking if previous deployment exist..." if [ "`helm ls`" == "" ]; then @@ -19,6 +15,7 @@ else fi echo "Deploying k8s-connect with latest changes" helm install --name=ci-$CIRCLE_SHA1 -f .circleci/sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz -#Todo-need to add test cases and run them before cleaning the deployment -#wait for deployment to finish, sleep longer for metrics-aggr to start sending metrics -sleep 90 \ No newline at end of file +#wait for deployment to finish +until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do + sleep 1; +done \ No newline at end of file diff --git a/.circleci/performance/perf_test_sck_values.yml b/.circleci/performance/perf_test_sck_values.yml index 317a0e74..376113fd 100755 --- a/.circleci/performance/perf_test_sck_values.yml +++ b/.circleci/performance/perf_test_sck_values.yml @@ -2,10 +2,10 @@ global: logLevel: info splunk: hec: - host: $SPLUNK_HEC_HOST_PERF + host: $CI_SPLUNK_HEC_HOST_PERF insecureSSL: true protocol: http - token: $SPLUNK_HEC_TOKEN_PERF + token: $CI_SPLUNK_HEC_TOKEN_PERF splunk-kubernetes-logging: image: pullPolicy: Always diff --git a/.circleci/run_functional_tests.sh b/.circleci/run_functional_tests.sh index e1c31318..e2b7bce6 100755 --- a/.circleci/run_functional_tests.sh +++ b/.circleci/run_functional_tests.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash set -e +pyenv global 3.6.5 cd test -sudo pip3 install --upgrade pip -sudo pip3 install -r requirements.txt #Run pytests echo "Running functional tests....." -python3 -m pytest \ - --splunkd-url https://$SPLUNK_HEC_HOST:8089 \ +python -m pytest \ + --splunkd-url https://$CI_SPLUNK_HEC_HOST:8089 \ --splunk-user admin \ - --splunk-password $SPLUNK_PASSWORD \ + --splunk-password $CI_SPLUNK_PASSWORD \ -p no:warnings -s diff --git a/.circleci/sck_values.yml b/.circleci/sck_values.yml index 740d550d..b6d9d90b 100755 --- a/.circleci/sck_values.yml +++ b/.circleci/sck_values.yml @@ -4,8 +4,8 @@ global: hec: protocol: https insecureSSL: true - host: $SPLUNK_HEC_HOST - token: $SPLUNK_HEC_TOKEN + host: $CI_SPLUNK_HEC_HOST + token: $CI_SPLUNK_HEC_TOKEN splunk-kubernetes-logging: kubernetes: diff --git a/.circleci/update_sck_config.sh b/.circleci/update_sck_config.sh index a7bb37c0..ed96aa6a 100755 --- a/.circleci/update_sck_config.sh +++ b/.circleci/update_sck_config.sh @@ -1,37 +1,69 @@ #!/usr/bin/env bash set -e +source PLUGIN_VERSIONS.sh + # Install yq yaml parser wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 sudo chmod +x yq_linux_amd64 sudo mv yq_linux_amd64 /usr/local/bin/yq # Modify splunk environment values -yq w -i .circleci/sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST -yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN -yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST_PERF -yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN_PERF - -# Pull docker images locally -aws ecr get-login --region $AWS_REGION --no-include-email | bash -docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-logging:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' -docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-metrics:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' -docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-metrics-agg:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' -docker pull $AWS_ACCOUNT_ID.dkr.ecr.us-west-1.amazonaws.com/k8s-ci-objects:latest | awk 'END{print}' | sed -E 's/[0-9]+.dkr/$AWS_ACCOUNT_ID/g' +yq w -i .circleci/sck_values.yml global.splunk.hec.host $CI_SPLUNK_HEC_HOST +yq w -i .circleci/sck_values.yml global.splunk.hec.token $CI_SPLUNK_HEC_TOKEN +yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.host $CI_SPLUNK_HEC_HOST_PERF +yq w -i .circleci/performance/perf_test_sck_values.yml global.splunk.hec.token $CI_SPLUNK_HEC_TOKEN_PERF +yq w -i .circleci/sck_values.yml global.splunk.hec.protocol ${CI_HEC_PROTOCOL:-https} # Modify docker images to be used -image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-logging:latest") -yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name $image_name -yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-logging.image.name $image_name +yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec +yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.tag $FLUENTD_HEC_VERSION +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-logging.image.tag $FLUENTD_HEC_VERSION + +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.tag $K8S_METRICS_VERISION +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-metrics.image.tag $K8S_METRICS_VERISION + +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name splunk/k8s-metrics-aggr +yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.tag $K8S_METRICS_AGGR_VERSION +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-metrics.imageAgg.name splunk/k8s-metrics-aggr +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-metrics.imageAgg.tag $K8S_METRICS_AGGR_VERSION + +yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name splunk/kube-objects +yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.tag $KUBE_OBJECT_VERSION +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-objects.image.name splunk/kube-objects +yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-objects.image.tag $KUBE_OBJECT_VERSION -image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics:latest") -yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name $image_name -yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-metrics.image.name $image_name -image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-metrics-agg:latest") -yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name $image_name -yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-metrics.imageAgg.name $image_name +# locally build image for triggered functional test +if [ ! -z "$TRIG_PROJECT" ] +then + cd /tmp + TRIG_REPO="$(echo $TRIG_REPO | sed 's/git\@github\.com\:/https\:\/\/github.com\//g')" + git clone $TRIG_REPO + cd $TRIG_PROJECT + git checkout $TRIG_BRANCH + source docker/build.sh $TRIG_BRANCH + cd ~/repo + case $TRIG_PROJECT in + "fluent-plugin-kubernetes-metrics") + yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.tag $TRIG_BRANCH + yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.pullPolicy IfNotPresent + ;; + "fluent-plugin-splunk-hec") + yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.tag $TRIG_BRANCH + yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.pullPolicy IfNotPresent + ;; + "fluent-plugin-kubernetes-objects") + yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.tag $TRIG_BRANCH + yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.pullPolicy IfNotPresent + ;; + "fluent-plugin-k8s-metrics-agg") + yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.tag $TRIG_BRANCH + yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.pullPolicy IfNotPresent + ;; + esac +fi -image_name=$(echo "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/k8s-ci-objects:latest") -yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name $image_name -yq w -i .circleci/performance/perf_test_sck_values.yml splunk-kubernetes-objects.image.name $image_name \ No newline at end of file diff --git a/.circleci/update_sck_config_for_release.sh b/.circleci/update_sck_config_for_release.sh deleted file mode 100755 index e802c7e1..00000000 --- a/.circleci/update_sck_config_for_release.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e -VERSION=`cat VERSION` - -# Install yq yaml parser -wget https://github.com/mikefarah/yq/releases/download/2.2.1/yq_linux_amd64 -sudo chmod +x yq_linux_amd64 -sudo mv yq_linux_amd64 /usr/local/bin/yq - -# setup necessary for functional tests -# Modify splunk environment values -yq w -i .circleci/sck_values.yml global.splunk.hec.host $SPLUNK_HEC_HOST -yq w -i .circleci/sck_values.yml global.splunk.hec.token $SPLUNK_HEC_TOKEN - -# Pull docker images locally -docker pull splunk/fluentd-hec:1.1.1 -docker pull splunk/k8s-metrics:1.1.1 -docker pull splunk/k8s-metrics-aggr:1.1.0 -docker pull splunk/kube-objects:1.1.0 - -# Modify docker images to be used -yq w -i .circleci/sck_values.yml splunk-kubernetes-logging.image.name splunk/fluentd-hec:1.1.1 -yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.image.name splunk/k8s-metrics:1.1.1 -yq w -i .circleci/sck_values.yml splunk-kubernetes-metrics.imageAgg.name splunk/k8s-metrics-aggr:1.1.0 -yq w -i .circleci/sck_values.yml splunk-kubernetes-objects.image.name splunk/kube-objects:1.1.0 diff --git a/Makefile b/Makefile index 15d99995..67d23f96 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +include PLUGIN_VERSIONS.sh +export $(shell sed 's/=.*//' PLUGIN_VERSIONS.sh) + connect-chart: logging-chart objects-chart metrics-chart @sed -i.bak -E -e 's/^([[:space:]]+repository:[[:space:]]+).+$$/\1local/' helm-chart/splunk-connect-for-kubernetes/requirements.yaml @mv helm-chart/splunk-connect-for-kubernetes/requirements.yaml.bak . @@ -21,6 +24,7 @@ build: .PHONY: manifests manifests: connect-chart + @./tools/update_charts_version.sh @helm template \ --set global.splunk.hec.host=MY-SPLUNK-HOST \ --set global.splunk.hec.token=MY-SPLUNK-TOKEN \ @@ -29,6 +33,10 @@ manifests: connect-chart --set splunk-kubernetes-metrics.fullnameOverride="splunk-kubernetes-metrics" \ --set splunk-kubernetes-objects.fullnameOverride="splunk-kubernetes-objects" \ --set splunk-kubernetes-objects.kubernetes.insecureSSL=true \ + --set splunk-kubernetes-objects.image.tag=$(KUBE_OBJECT_VERSION) \ + --set splunk-kubernetes-logging.image.tag=$(FLUENTD_HEC_VERSION) \ + --set splunk-kubernetes-metrics.image.tag=$(K8S_METRICS_VERISION) \ + --set splunk-kubernetes-metrics.imageAgg.tag=$(K8S_METRICS_AGGR_VERSION) \ $$(ls build/splunk-connect-for-kubernetes-*.tgz) \ | ruby tools/gen_manifest.rb manifests diff --git a/PLUGIN_VERSIONS.sh b/PLUGIN_VERSIONS.sh new file mode 100755 index 00000000..a25142a8 --- /dev/null +++ b/PLUGIN_VERSIONS.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +FLUENTD_HEC_VERSION=1.1.1 +K8S_METRICS_VERISION=1.1.1 +K8S_METRICS_AGGR_VERSION=1.1.0 +KUBE_OBJECT_VERSION=1.1.2 diff --git a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml index 4938a113..b6427787 100644 --- a/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml +++ b/helm-chart/splunk-kubernetes-logging/templates/daemonset.yaml @@ -9,6 +9,8 @@ metadata: heritage: {{ .Release.Service }} engine: fluentd spec: + updateStrategy: + type: RollingUpdate selector: matchLabels: app: {{ template "splunk-kubernetes-logging.name" . }} diff --git a/helm-chart/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-kubernetes-logging/values.yaml index 023914dc..c489735e 100644 --- a/helm-chart/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-kubernetes-logging/values.yaml @@ -219,6 +219,14 @@ logs: regexp: (?