Support pushing span/trace through to splunk HEC (signalfx#629)

I wanted to export spans to our splunk setup from the gateway but the way the chart is currently set up made it difficult to get all the pieces to match up (particularly port declarations across deployment/service/pods) without manually patching each one of them. So this commit adds tracesIndex/tracesEnabled alongside metricsIndex/metricsEnabled etc so the traces are exported as expected.
omrozowicz-splunk · Jan 26, 2023 · daab50e · daab50e
1 parent be3f7f6
commit daab50e
Show file tree

Hide file tree

Showing 7 changed files with 84 additions and 8 deletions.
diff --git a/helm-charts/splunk-otel-collector/templates/_helpers.tpl b/helm-charts/splunk-otel-collector/templates/_helpers.tpl
@@ -107,6 +107,13 @@ Whether metrics enabled for Splunk Platform.
 {{- and (eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true") .Values.splunkPlatform.metricsEnabled }}
 {{- end -}}
 
+{{/*
+Whether traces enabled for Splunk Platform.
+*/}}
+{{- define "splunk-otel-collector.platformTracesEnabled" -}}
+{{- and (eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true") .Values.splunkPlatform.tracesEnabled }}
+{{- end -}}
+
 {{/*
 Whether metrics enabled for any destination.
 */}}
@@ -115,10 +122,10 @@ Whether metrics enabled for any destination.
 {{- end -}}
 
 {{/*
-Whether traces enabled for any destination. (currently applicable to Splunk Observability only).
+Whether traces enabled for any destination.
 */}}
 {{- define "splunk-otel-collector.tracesEnabled" -}}
-{{- include "splunk-otel-collector.o11yTracesEnabled" . }}
+{{- or (eq (include "splunk-otel-collector.o11yTracesEnabled" .) "true") (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
 {{- end -}}
 
 {{/*

diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl
@@ -211,7 +211,7 @@ splunk_hec/platform_logs:
 {{- end }}
 
 {{/*
-Splunk Platform Logs exporter
+Splunk Platform Metrics exporter
 */}}
 {{- define "splunk-otel-collector.splunkPlatformMetricsExporter" -}}
 splunk_hec/platform_metrics:
@@ -246,6 +246,42 @@ splunk_hec/platform_metrics:
     queue_size: {{ .Values.splunkPlatform.sendingQueue.queueSize }}
 {{- end }}
 
+{{/*
+Splunk Platform Traces exporter
+*/}}
+{{- define "splunk-otel-collector.splunkPlatformTracesExporter" -}}
+splunk_hec/platform_traces:
+  endpoint: {{ .Values.splunkPlatform.endpoint | quote }}
+  token: "${SPLUNK_PLATFORM_HEC_TOKEN}"
+  index: {{ .Values.splunkPlatform.tracesIndex | quote }}
+  source: {{ .Values.splunkPlatform.source | quote }}
+  max_connections: {{ .Values.splunkPlatform.maxConnections }}
+  disable_compression: {{ .Values.splunkPlatform.disableCompression }}
+  timeout: {{ .Values.splunkPlatform.timeout }}
+  splunk_app_name: {{ .Chart.Name }}
+  splunk_app_version: {{ .Chart.Version }}
+  tls:
+    insecure_skip_verify: {{ .Values.splunkPlatform.insecureSkipVerify }}
+    {{- if .Values.splunkPlatform.clientCert }}
+    cert_file: /otel/etc/splunk_platform_hec_client_cert
+    {{- end }}
+    {{- if .Values.splunkPlatform.clientKey  }}
+    key_file: /otel/etc/splunk_platform_hec_client_key
+    {{- end }}
+    {{- if .Values.splunkPlatform.caFile }}
+    ca_file: /otel/etc/splunk_platform_hec_ca_file
+    {{- end }}
+  retry_on_failure:
+    enabled: {{ .Values.splunkPlatform.retryOnFailure.enabled }}
+    initial_interval: {{ .Values.splunkPlatform.retryOnFailure.initialInterval }}
+    max_interval: {{ .Values.splunkPlatform.retryOnFailure.maxInterval }}
+    max_elapsed_time: {{ .Values.splunkPlatform.retryOnFailure.maxElapsedTime }}
+  sending_queue:
+    enabled:  {{ .Values.splunkPlatform.sendingQueue.enabled }}
+    num_consumers: {{ .Values.splunkPlatform.sendingQueue.numConsumers }}
+    queue_size: {{ .Values.splunkPlatform.sendingQueue.queueSize }}
+{{- end }}
+
 {{/*
 Add Extra Labels
 */}}

diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
@@ -587,6 +587,9 @@ exporters:
   {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" .) "true") }}
   {{- include "splunk-otel-collector.splunkPlatformMetricsExporter" . | nindent 2 }}
   {{- end }}
+  {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.splunkPlatformTracesExporter" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }}
@@ -698,7 +701,13 @@ service:
     {{- if (eq (include "splunk-otel-collector.tracesEnabled" .) "true") }}
     # Default traces pipeline.
     traces:
-      receivers: [otlp, jaeger, smartagent/signalfx-forwarder, zipkin]
+      receivers:
+        - otlp
+        - jaeger
+        {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" $) "true") }}
+        - smartagent/signalfx-forwarder
+        {{- end }}
+        - zipkin
       processors:
         - memory_limiter
         - k8sattributes
@@ -712,8 +721,13 @@ service:
         {{- if $gatewayEnabled }}
         - otlp
         {{- else }}
+        {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" .) "true") }}
         - sapm
         {{- end }}
+        {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+        - splunk_hec/platform_traces
+        {{- end }}
+        {{- end }}
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" $) "true") }}
         # For trace/metric correlation.
         - signalfx

diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl
@@ -164,6 +164,9 @@ exporters:
   {{- include "splunk-otel-collector.splunkPlatformMetricsExporter" . | nindent 2 }}
   {{- end }}
 
+  {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.splunkPlatformTracesExporter" . | nindent 2 }}
+  {{- end }}
 service:
   telemetry:
     metrics:
@@ -179,7 +182,7 @@ service:
   # The default pipelines should not need to be changed. You can add any custom pipeline instead.
   # In order to disable a default pipeline just set it to `null` in gateway.config overrides.
   pipelines:
-    {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" $) "true") }}
+    {{- if (eq (include "splunk-otel-collector.tracesEnabled" $) "true") }}
     # default traces pipeline
     traces:
       receivers: [otlp, jaeger, zipkin]
@@ -194,7 +197,13 @@ service:
         {{- if .Values.environment }}
         - resource/add_environment
         {{- end }}
-      exporters: [sapm]
+      exporters:
+        {{- if (eq (include "splunk-otel-collector.o11yTracesEnabled" .) "true") }}
+        - sapm
+        {{- end }}
+        {{- if (eq (include "splunk-otel-collector.platformTracesEnabled" .) "true") }}
+        - splunk_hec/platform_traces
+        {{- end }}
     {{- end }}
 
     {{- if (eq (include "splunk-otel-collector.metricsEnabled" .) "true") }}

diff --git a/helm-charts/splunk-otel-collector/templates/daemonset.yaml b/helm-charts/splunk-otel-collector/templates/daemonset.yaml
@@ -232,7 +232,7 @@ spec:
         {{- end }}
         ports:
         {{- range $key, $port := $agent.ports }}
-        {{- if eq true (and (eq (include "splunk-otel-collector.metricsEnabled" $) "true") (has "metrics" $port.enabled_for)) (and (eq (include "splunk-otel-collector.o11yTracesEnabled" $) "true") (has "traces" $port.enabled_for)) (and (eq (include "splunk-otel-collector.logsEnabled" $) "true") (has "logs" $port.enabled_for)) (and (eq (include "splunk-otel-collector.profilingEnabled" $) "true") (has "profiling" $port.enabled_for)) }}
+        {{- if eq true (and (eq (include "splunk-otel-collector.metricsEnabled" $) "true") (has "metrics" $port.enabled_for)) (and (eq (include "splunk-otel-collector.tracesEnabled" $) "true") (has "traces" $port.enabled_for)) (and (eq (include "splunk-otel-collector.logsEnabled" $) "true") (has "logs" $port.enabled_for)) (and (eq (include "splunk-otel-collector.profilingEnabled" $) "true") (has "profiling" $port.enabled_for)) }}
         - name: {{ $key }}
           {{- omit $port "enabled_for" | toYaml | trim | nindent 10 }}
         {{- end }}

diff --git a/helm-charts/splunk-otel-collector/values.schema.json b/helm-charts/splunk-otel-collector/values.schema.json
@@ -46,6 +46,9 @@
         "metricsIndex": {
           "type": "string"
         },
+        "tracesIndex": {
+          "type": "string"
+        },
         "source": {
           "type": "string"
         },
@@ -81,6 +84,10 @@
           "description": "Send Metrics to Splunk Platform",
           "type": "boolean"
         },
+        "tracesEnabled": {
+          "description": "Send Traces to Splunk Platform",
+          "type": "boolean"
+        },
         "fieldNameConvention": {
           "type": "object",
           "additionalProperties": false,

diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml
@@ -35,6 +35,8 @@ splunkPlatform:
   index: "main"
   # Name of the Splunk metric type index targeted. Required when ingesting metrics to Splunk Platform.
   metricsIndex: ""
+  # Name of the Splunk event type index targeted. Required when ingesting traces to Splunk Platform.
+  tracesIndex: ""
   # Optional. Default value for `source` field.
   source: "kubernetes"
   # Optional. Default value for `sourcetype` field. For container logs, it will
@@ -67,7 +69,8 @@ splunkPlatform:
   logsEnabled: true
   # If you enable metrics collection, make sure that `metricsIndex` is provided as well.
   metricsEnabled: false
-
+  # If you enable traces collection, make sure that `tracesIndex` is provided as well.
+  tracesEnabled: false
   # Field name conventions to use. (Only for those who are migrating from Splunk Connect for Kubernetes helm chart)
   fieldNameConvention:
     # Boolean for renaming pod metadata fields to match to Splunk Connect for Kubernetes helm chart.