From 43ee64ef5ffe951fe47a9a7124034a6d5a44d059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Wysoglad?= Date: Thu, 3 Oct 2024 12:26:20 +0200 Subject: [PATCH] feat: TLS support for the Thanos web endpoint (#496) * feat: add Thanos Web endpoint TLS support * test: add testcase for Querier with TLS * feat: watch Querier TLS resources --------- Co-authored-by: Jan Fajerski --- .../monitoring.rhobs_thanosqueriers.yaml | 61 +++++++ .../monitoring.rhobs_thanosqueriers.yaml | 61 +++++++ docs/api.md | 150 ++++++++++++++++++ pkg/apis/monitoring/v1alpha1/types.go | 3 + .../v1alpha1/zz_generated.deepcopy.go | 5 + .../monitoring/thanos-querier/components.go | 122 +++++++++++++- .../monitoring/thanos-querier/controller.go | 123 +++++++++++++- test/e2e/thanos_querier_controller_test.go | 113 +++++++++++++ 8 files changed, 630 insertions(+), 8 deletions(-) diff --git a/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml b/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml index cc5275d0..c41580a1 100644 --- a/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml +++ b/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml @@ -110,6 +110,67 @@ spec: type: object type: object x-kubernetes-map-type: atomic + webTLSConfig: + description: Configure TLS options for the Thanos web server. + properties: + certificate: + description: Reference to the TLS public certificate for the web + server. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + minLength: 1 + type: string + name: + description: The name of the secret in the object's namespace + to select from. + minLength: 1 + type: string + required: + - key + - name + type: object + certificateAuthority: + description: Reference to the root Certificate Authority used + to verify the web server's certificate. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + minLength: 1 + type: string + name: + description: The name of the secret in the object's namespace + to select from. + minLength: 1 + type: string + required: + - key + - name + type: object + privateKey: + description: Reference to the TLS private key for the web server. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + minLength: 1 + type: string + name: + description: The name of the secret in the object's namespace + to select from. + minLength: 1 + type: string + required: + - key + - name + type: object + required: + - certificate + - certificateAuthority + - privateKey + type: object required: - selector type: object diff --git a/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml b/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml index d708ea93..e64f4ded 100644 --- a/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml +++ b/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml @@ -110,6 +110,67 @@ spec: type: object type: object x-kubernetes-map-type: atomic + webTLSConfig: + description: Configure TLS options for the Thanos web server. + properties: + certificate: + description: Reference to the TLS public certificate for the web + server. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + minLength: 1 + type: string + name: + description: The name of the secret in the object's namespace + to select from. + minLength: 1 + type: string + required: + - key + - name + type: object + certificateAuthority: + description: Reference to the root Certificate Authority used + to verify the web server's certificate. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + minLength: 1 + type: string + name: + description: The name of the secret in the object's namespace + to select from. + minLength: 1 + type: string + required: + - key + - name + type: object + privateKey: + description: Reference to the TLS private key for the web server. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + minLength: 1 + type: string + name: + description: The name of the secret in the object's namespace + to select from. + minLength: 1 + type: string + required: + - key + - name + type: object + required: + - certificate + - certificateAuthority + - privateKey + type: object required: - selector type: object diff --git a/docs/api.md b/docs/api.md index 9b51d358..51a8419c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -3690,6 +3690,13 @@ deduplicate.
false + + webTLSConfig + object + + Configure TLS options for the Thanos web server.
+ + false @@ -3810,6 +3817,149 @@ list restricting them.
+ +### ThanosQuerier.spec.webTLSConfig +[↩ Parent](#thanosquerierspec) + + + +Configure TLS options for the Thanos web server. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
certificateobject + Reference to the TLS public certificate for the web server.
+ 		true
certificateAuthorityobject + Reference to the root Certificate Authority used to verify the web server's certificate.
+ 		true
privateKeyobject + Reference to the TLS private key for the web server.
+ 		true
+ + +### ThanosQuerier.spec.webTLSConfig.certificate +[↩ Parent](#thanosquerierspecwebtlsconfig) + + + +Reference to the TLS public certificate for the web server. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key of the secret to select from. Must be a valid secret key.
+ 		true
namestring + The name of the secret in the object's namespace to select from.
+ 		true
+ + +### ThanosQuerier.spec.webTLSConfig.certificateAuthority +[↩ Parent](#thanosquerierspecwebtlsconfig) + + + +Reference to the root Certificate Authority used to verify the web server's certificate. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key of the secret to select from. Must be a valid secret key.
+ 		true
namestring + The name of the secret in the object's namespace to select from.
+ 		true
+ + +### ThanosQuerier.spec.webTLSConfig.privateKey +[↩ Parent](#thanosquerierspecwebtlsconfig) + + + +Reference to the TLS private key for the web server. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key of the secret to select from. Must be a valid secret key.
+ 		true
namestring + The name of the secret in the object's namespace to select from.
+ 		true
+ # observability.openshift.io/v1alpha1 Resource Types: diff --git a/pkg/apis/monitoring/v1alpha1/types.go b/pkg/apis/monitoring/v1alpha1/types.go index 6ed6205a..18eb00df 100644 --- a/pkg/apis/monitoring/v1alpha1/types.go +++ b/pkg/apis/monitoring/v1alpha1/types.go @@ -279,6 +279,9 @@ type ThanosQuerierSpec struct { // Selector to select which namespaces the Monitoring Stack objects are discovered from. NamespaceSelector NamespaceSelector `json:"namespaceSelector,omitempty"` ReplicaLabels []string `json:"replicaLabels,omitempty"` + // Configure TLS options for the Thanos web server. + // +optional + WebTLSConfig *WebTLSConfig `json:"webTLSConfig,omitempty"` } // ThanosQuerierStatus defines the observed state of ThanosQuerier. diff --git a/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go index ca88fb31..c62674f8 100644 --- a/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go @@ -348,6 +348,11 @@ func (in *ThanosQuerierSpec) DeepCopyInto(out *ThanosQuerierSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.WebTLSConfig != nil { + in, out := &in.WebTLSConfig, &out.WebTLSConfig + *out = new(WebTLSConfig) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ThanosQuerierSpec. diff --git a/pkg/controllers/monitoring/thanos-querier/components.go b/pkg/controllers/monitoring/thanos-querier/components.go index 90553995..160684a7 100644 --- a/pkg/controllers/monitoring/thanos-querier/components.go +++ b/pkg/controllers/monitoring/thanos-querier/components.go @@ -13,17 +13,54 @@ import ( "github.com/rhobs/observability-operator/pkg/reconciler" ) -func thanosComponentReconcilers(thanos *msoapi.ThanosQuerier, sidecarUrls []string, thanosCfg ThanosConfiguration) []reconciler.Reconciler { +func thanosComponentReconcilers( + thanos *msoapi.ThanosQuerier, + sidecarUrls []string, + thanosCfg ThanosConfiguration, + tlsHashes map[string]string, +) []reconciler.Reconciler { name := "thanos-querier-" + thanos.Name return []reconciler.Reconciler{ reconciler.NewUpdater(newServiceAccount(name, thanos.Namespace), thanos), - reconciler.NewUpdater(newThanosQuerierDeployment(name, thanos, sidecarUrls, thanosCfg), thanos), + reconciler.NewUpdater(newThanosQuerierDeployment(name, thanos, sidecarUrls, thanosCfg, tlsHashes), thanos), reconciler.NewUpdater(newService(name, thanos.Namespace), thanos), - reconciler.NewUpdater(newServiceMonitor(name, thanos.Namespace), thanos), + reconciler.NewUpdater(newServiceMonitor(name, thanos.Namespace, thanos), thanos), + reconciler.NewOptionalUpdater(newHttpConfConfigMap(name, thanos), thanos, thanos.Spec.WebTLSConfig != nil), } } -func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecarUrls []string, thanosCfg ThanosConfiguration) *appsv1.Deployment { +func newHttpConfConfigMap(name string, thanos *msoapi.ThanosQuerier) *corev1.ConfigMap { + httpConf := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-http-conf", name), + Namespace: thanos.Namespace, + }, + } + if thanos.Spec.WebTLSConfig != nil { + httpConf.Data = map[string]string{ + "http.conf": ` +tls_server_config: + cert_file: /etc/thanos/tls-assets/web-cert-secret/` + thanos.Spec.WebTLSConfig.Certificate.Key + ` + key_file: /etc/thanos/tls-assets/web-key-secret/` + thanos.Spec.WebTLSConfig.PrivateKey.Key, + } + } + + return httpConf +} + +func newThanosQuerierDeployment( + name string, + spec *msoapi.ThanosQuerier, + sidecarUrls []string, + thanosCfg ThanosConfiguration, + tlsHashes map[string]string, +) *appsv1.Deployment { + httpConfCMName := fmt.Sprintf("%s-http-conf", name) + args := []string{ "query", "--log.format=logfmt", @@ -38,6 +75,10 @@ func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecar args = append(args, fmt.Sprintf("--query.replica-label=%s", rl)) } + if spec.Spec.WebTLSConfig != nil { + args = append(args, "--http.config=/etc/thanos/tls-assets/web-http-conf-cm/http.conf") + } + thanos := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: appsv1.SchemeGroupVersion.String(), @@ -102,6 +143,58 @@ func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecar ProgressDeadlineSeconds: ptr.To(int32(300)), }, } + if spec.Spec.WebTLSConfig != nil { + thanos.Spec.Template.Spec.Volumes = append(thanos.Spec.Template.Spec.Volumes, []corev1.Volume{ + { + Name: "thanos-web-tls-key", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: spec.Spec.WebTLSConfig.PrivateKey.Name, + }, + }, + }, + { + Name: "thanos-web-tls-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: spec.Spec.WebTLSConfig.Certificate.Name, + }, + }, + }, + { + Name: "thanos-web-http-conf", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: httpConfCMName, + }, + }, + }, + }, + }...) + thanos.Spec.Template.Spec.Containers[0].VolumeMounts = append(thanos.Spec.Template.Spec.Containers[0].VolumeMounts, []corev1.VolumeMount{ + { + Name: "thanos-web-tls-key", + MountPath: "/etc/thanos/tls-assets/web-cert-secret", + ReadOnly: true, + }, + { + Name: "thanos-web-tls-cert", + MountPath: "/etc/thanos/tls-assets/web-key-secret", + ReadOnly: true, + }, + { + Name: "thanos-web-http-conf", + MountPath: "/etc/thanos/tls-assets/web-http-conf-cm", + ReadOnly: true, + }, + }...) + tlsAnnotations := map[string]string{} + for name, hash := range tlsHashes { + tlsAnnotations[fmt.Sprintf("monitoring.openshift.io/%s-hash", name)] = hash + } + thanos.ObjectMeta.Annotations = tlsAnnotations + } return thanos } @@ -144,8 +237,8 @@ func newService(name string, namespace string) *corev1.Service { } } -func newServiceMonitor(name string, namespace string) *monv1.ServiceMonitor { - return &monv1.ServiceMonitor{ +func newServiceMonitor(name string, namespace string, thanos *msoapi.ThanosQuerier) *monv1.ServiceMonitor { + serviceMonitor := &monv1.ServiceMonitor{ TypeMeta: metav1.TypeMeta{ APIVersion: monv1.SchemeGroupVersion.String(), Kind: "ServiceMonitor", @@ -169,6 +262,23 @@ func newServiceMonitor(name string, namespace string) *monv1.ServiceMonitor { }, }, } + if thanos.Spec.WebTLSConfig != nil { + serviceMonitor.Spec.Endpoints[0].Scheme = "https" + serviceMonitor.Spec.Endpoints[0].TLSConfig = &monv1.TLSConfig{ + SafeTLSConfig: monv1.SafeTLSConfig{ + CA: monv1.SecretOrConfigMap{ + Secret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: thanos.Spec.WebTLSConfig.CertificateAuthority.Name, + }, + Key: thanos.Spec.WebTLSConfig.CertificateAuthority.Key, + }, + }, + ServerName: ptr.To(name), + }, + } + } + return serviceMonitor } func componentLabels(querierName string) map[string]string { diff --git a/pkg/controllers/monitoring/thanos-querier/controller.go b/pkg/controllers/monitoring/thanos-querier/controller.go index 87275a99..b9285fe7 100644 --- a/pkg/controllers/monitoring/thanos-querier/controller.go +++ b/pkg/controllers/monitoring/thanos-querier/controller.go @@ -14,6 +14,7 @@ package thanos_querier import ( "context" + "crypto/sha256" "fmt" "time" @@ -22,9 +23,11 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -51,6 +54,12 @@ type Options struct { Thanos ThanosConfiguration } +const ( + thanosTLSPrivateKeySecretNameField = ".spec.webTLSConfig.privateKey.name" + thanosTLSCertificateSecretNameField = ".spec.webTLSConfig.certificate.name" + thanosTLSCertificateAuthoritySecretNameField = ".spec.webTLSConfig.certificateAuthority.name" +) + // RBAC for watching monitoring stacks //+kubebuilder:rbac:groups=monitoring.rhobs,resources=monitoringstacks,verbs=list;watch @@ -63,7 +72,8 @@ type Options struct { //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=list;watch;create;update;patch;delete // RBAC for managing core resources -//+kubebuilder:rbac:groups=core,resources=services;serviceaccounts,verbs=list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=services;serviceaccounts;configmaps,verbs=list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=secrets,verbs=list;watch // RBAC for managing Prometheus Operator CRs //+kubebuilder:rbac:groups=monitoring.rhobs,resources=servicemonitors,verbs=list;watch;create;update;patch;delete @@ -79,17 +89,56 @@ func RegisterWithManager(mgr ctrl.Manager, opts Options) error { thanos: opts.Thanos, } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSPrivateKeySecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*msoapi.ThanosQuerier) + if cr.Spec.WebTLSConfig == nil { + return nil + } + return []string{cr.Spec.WebTLSConfig.PrivateKey.Name} + }); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSCertificateSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*msoapi.ThanosQuerier) + if cr.Spec.WebTLSConfig == nil { + return nil + } + return []string{cr.Spec.WebTLSConfig.Certificate.Name} + }); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSCertificateAuthoritySecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*msoapi.ThanosQuerier) + if cr.Spec.WebTLSConfig == nil { + return nil + } + return []string{cr.Spec.WebTLSConfig.CertificateAuthority.Name} + }); err != nil { + return err + } + p := predicate.GenerationChangedPredicate{} return ctrl.NewControllerManagedBy(mgr). For(&msoapi.ThanosQuerier{}). Owns(&appsv1.Deployment{}).WithEventFilter(p). Owns(&corev1.ServiceAccount{}).WithEventFilter(p). Owns(&corev1.Service{}).WithEventFilter(p). + Owns(&corev1.ConfigMap{}).WithEventFilter(p). Watches( &msoapi.MonitoringStack{}, handler.EnqueueRequestsFromMapFunc(rm.findQueriersForMonitoringStack), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(rm.findQueriersForTLSSecrets), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). Complete(rm) } @@ -113,7 +162,23 @@ func (rm resourceManager) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{RequeueAfter: 10 * time.Second}, err } - reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos) + tlsHashes := map[string]string{} + if querier.Spec.WebTLSConfig != nil { + secretSelectors := []msoapi.SecretKeySelector{ + querier.Spec.WebTLSConfig.CertificateAuthority, + querier.Spec.WebTLSConfig.Certificate, + querier.Spec.WebTLSConfig.PrivateKey, + } + for _, secretSelector := range secretSelectors { + hash, err := rm.hashOfTLSSecret(secretSelector, querier.Namespace) + if err != nil { + return ctrl.Result{}, err + } + tlsHashes[fmt.Sprintf("%s-%s", secretSelector.Name, secretSelector.Key)] = hash + } + } + + reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos, tlsHashes) for _, reconciler := range reconcilers { err := reconciler.Reconcile(ctx, rm, rm.scheme) // handle creation / updation errors that can happen due to a stale cache by @@ -156,6 +221,20 @@ func (rm resourceManager) findSidecarServices(ctx context.Context, tQuerier *mso return sidecarUrls, nil } +func (rm resourceManager) hashOfTLSSecret(selector msoapi.SecretKeySelector, namespace string) (string, error) { + var secret corev1.Secret + err := rm.Get(context.Background(), types.NamespacedName{ + Name: selector.Name, + Namespace: namespace, + }, &secret) + if err != nil { + return "", fmt.Errorf("Couldn't get TLS secret %s: %s", selector.Name, err) + } + + hash := sha256.Sum256(secret.Data[selector.Key]) + return rand.SafeEncodeString(fmt.Sprint(hash)), nil +} + // Given a Service object, return a url to use as value for --store/--endpoint. func getEndpointUrl(serviceName string, namespace string) string { return fmt.Sprintf("dnssrv+_grpc._tcp.%s.%s.svc.cluster.local", serviceName, namespace) @@ -191,3 +270,43 @@ func (rm resourceManager) findQueriersForMonitoringStack(ctx context.Context, ms } return requests } + +// Find all ThanosQueriers, whose TLS secrets fit the given Secret and +// return a list of reconcile requests, one for each ThanosQuerier. +func (rm resourceManager) findQueriersForTLSSecrets(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + logger := rm.logger.WithValues("Secret", src.GetNamespace()+"/"+src.GetName()) + logger.Info("watched Secret changed, checking for matching querier") + + thanosWatchFields := []string{ + thanosTLSCertificateAuthoritySecretNameField, + thanosTLSCertificateSecretNameField, + thanosTLSPrivateKeySecretNameField, + } + + for _, field := range thanosWatchFields { + crList := &msoapi.ThanosQuerierList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := rm.Client.List(ctx, crList, listOps) + if err != nil { + logger.Error(err, "Failed to list Thanosqueriers") + return []reconcile.Request{} + } + + for _, item := range crList.Items { + logger.Info("Found querier, scheduling sync") + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }) + } + } + + return requests +} diff --git a/test/e2e/thanos_querier_controller_test.go b/test/e2e/thanos_querier_controller_test.go index f42255e0..b5a5c350 100644 --- a/test/e2e/thanos_querier_controller_test.go +++ b/test/e2e/thanos_querier_controller_test.go @@ -3,6 +3,8 @@ package e2e import ( "context" "fmt" + "net" + "strings" "testing" "time" @@ -13,6 +15,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/cert" "sigs.k8s.io/controller-runtime/pkg/client" msov1 "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1" @@ -31,6 +34,10 @@ func TestThanosQuerierController(t *testing.T) { name: "Delete resources if matched monitoring stack is deleted", scenario: stackWithSidecarGetsDeleted, }, + { + name: "Create resources for single monitoring stack with web endpoint TLS", + scenario: singleStackWithSidecarTLS, + }, } for _, tc := range ts { @@ -115,6 +122,112 @@ func singleStackWithSidecar(t *testing.T) { } } +func singleStackWithSidecarTLS(t *testing.T) { + comboName := "tq-ms-combo-tls" + querierName := "thanos-querier-" + comboName + + certs, key, err := cert.GenerateSelfSignedCertKey(querierName, []net.IP{}, []string{}) + assert.NilError(t, err) + + thanosKey := string(key) + thanosCerts := strings.SplitAfter(string(certs), "-----END CERTIFICATE-----") + + tlsSecretName := "thanos-test-tls-secret" + + thanosTLSSecret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: tlsSecretName, + Namespace: e2eTestNamespace, + }, + StringData: map[string]string{ + "tls.key": thanosKey, + "tls.crt": thanosCerts[0], + "ca.crt": thanosCerts[1], + }, + } + err = f.K8sClient.Create(context.Background(), &thanosTLSSecret) + assert.NilError(t, err) + + tq, ms := newThanosStackCombo(t, comboName) + tq.Spec.WebTLSConfig = &msov1.WebTLSConfig{ + PrivateKey: msov1.SecretKeySelector{ + Name: tlsSecretName, + Key: "tls.key", + }, + Certificate: msov1.SecretKeySelector{ + Name: tlsSecretName, + Key: "tls.crt", + }, + CertificateAuthority: msov1.SecretKeySelector{ + Name: tlsSecretName, + Key: "ca.crt", + }, + } + err = f.K8sClient.Create(context.Background(), tq) + assert.NilError(t, err, "failed to create a thanos querier") + err = f.K8sClient.Create(context.Background(), ms) + assert.NilError(t, err, "failed to create a monitoring stack") + + // Creating a basic combo must create a thanos deployment and a service + thanosDeployment := appsv1.Deployment{} + f.GetResourceWithRetry(t, querierName, tq.Namespace, &thanosDeployment) + + thanosService := corev1.Service{} + f.GetResourceWithRetry(t, querierName, tq.Namespace, &thanosService) + + f.AssertDeploymentReady(querierName, tq.Namespace, framework.WithTimeout(5*time.Minute))(t) + // Assert prometheus instance can be queried + stopChan := make(chan struct{}) + defer close(stopChan) + if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + err = f.StartServicePortForward(querierName, e2eTestNamespace, "10902", stopChan) + return err == nil, nil + }); wait.Interrupted(err) { + t.Fatal("timeout waiting for port-forward") + } + + promClient, err := framework.NewTLSPrometheusClient("https://localhost:10902", thanosCerts[1], querierName) + if err != nil { + t.Fatal(fmt.Errorf("Failed to create prometheus client: %s", err)) + } + expectedResults := map[string]int{ + "prometheus_build_info": 2, // must return from both prometheus pods + } + var lastErr error + if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + correct := 0 + for query, value := range expectedResults { + result, err := promClient.Query(query) + if err != nil { + return false, nil + } + + if len(result.Data.Result) == 0 { + return false, nil + } + + if len(result.Data.Result) > value { + lastErr = fmt.Errorf("invalid result for query %s, got %d, want %d", query, len(result.Data.Result), value) + return true, lastErr + } + + if len(result.Data.Result) != value { + return false, nil + } + + correct++ + } + + return correct == len(expectedResults), nil + }); wait.Interrupted(err) { + t.Fatal(fmt.Errorf("querying thanos did not yield expected results: %w", lastErr)) + } +} + func newThanosQuerier(t *testing.T, name string, selector map[string]string) *msov1.ThanosQuerier { tq := &msov1.ThanosQuerier{ ObjectMeta: metav1.ObjectMeta{