Merge pull request #2358 from amvanbaren/configure-tracing-12

Configure monitoring and tracing

Dockerfile

@@ -27,7 +27,8 @@ RUN /usr/bin/yarn --cwd website \
   && /usr/bin/yarn --cwd website build
 
 # Main image derived from openvsx-server 
-FROM ghcr.io/eclipse/openvsx-server:v0.14.3
+FROM docker.io/amvanbaren/openvsx-server:1b712aa
 
 COPY --from=builder --chown=openvsx:openvsx /workdir/website/static/ BOOT-INF/classes/static/
 COPY --from=builder --chown=openvsx:openvsx /workdir/configuration/ config/
+COPY --from=builder --chown=openvsx:openvsx /workdir/logging/logback-spring.xml BOOT-INF/classes/

configuration/application.yml

@@ -3,6 +3,9 @@
 # for database access and ovsx.storage.azure for file storage access.
 # Additional properties are loaded from the file provided with the
 # DEPLOYMENT_CONFIG environment variable.
+logging:
+  pattern:
+    level: '%5p [${spring.application.name:},%X{traceId:-},%X{spanId:-}]'
 server:
   address: 0.0.0.0
   port: 8080
@@ -12,6 +15,8 @@ server:
     mime-types: text/html,text/plain,text/css,application/javascript,application/json,text/xml,application/xml,application/xml+rss,text/javascript
     min-response-size: 1024
 spring:
+  application:
+    name: openvsx-server
   config:
     import: file:${DEPLOYMENT_CONFIG}
   cache:
@@ -57,6 +62,15 @@ management:
   health:
     probes:
       enabled: true
+  endpoints:
+    web:
+      exposure:
+        include:
+          - health
+          - prometheus
+  tracing:
+    sampling:
+      probability: 1.0 # TODO lower to 0.1 for production
 org:
   jobrunr:
     job-scheduler:

kubernetes/open-vsx.org.libsonnet

@@ -40,6 +40,71 @@ local newEnvironment(envName) = {
   },
 };
 
+local newGrafanaAgentConfigMap() = {
+  apiVersion: "v1",
+  kind: "ConfigMap",
+  metadata: {
+    name: "grafana-agent-configmap"
+  },
+  data: {
+    "agent.yml": |||
+      server:
+        log_level: debug
+      integrations:
+        prometheus_remote_write:
+        - basic_auth:
+            password: ${PROMETHEUS_PASSWORD}
+            username: ${PROMETHEUS_USERNAME}
+          url: ${PROMETHEUS_URL}
+        agent:
+          enabled: true
+          relabel_configs:
+          - action: replace
+            source_labels:
+            - agent_hostname
+            target_label: instance
+          - action: replace
+            target_label: job
+            replacement: "integrations/agent-check"
+          metric_relabel_configs:
+          - action: keep
+            regex: (prometheus_target_sync_length_seconds_sum|prometheus_target_scrapes_.*|prometheus_target_interval.*|prometheus_sd_discovered_targets|agent_build.*|agent_wal_samples_appended_total|process_start_time_seconds)
+            source_labels:
+            - __name__
+      metrics:
+        configs:
+        - name: integrations
+          remote_write:
+          - basic_auth:
+              password: ${PROMETHEUS_PASSWORD}
+              username: ${PROMETHEUS_USERNAME}
+            url: ${PROMETHEUS_URL}
+          scrape_configs:
+          - job_name: integrations/spring-boot
+            relabel_configs:
+              - replacement: ${HOSTNAME}
+                target_label: instance
+            static_configs:
+              - targets: ['localhost:8081']
+            metrics_path: /actuator/prometheus
+        global:
+          scrape_interval: 60s
+      traces:
+        configs:
+        - name: default
+          remote_write:
+          - endpoint: ${TEMPO_URL}
+            basic_auth:
+              username: ${TEMPO_USERNAME}
+              password: ${TEMPO_PASSWORD}
+          receivers:
+            zipkin:
+              endpoint: localhost:9411
+              parse_string_tags: false
+    |||
+  }
+};
+
 local newDeployment(env, dockerImage) = {
 
   local elasticsearchCertsVolumeName = "elastic-internal-http-certificates",
@@ -86,6 +151,33 @@ local newDeployment(env, dockerImage) = {
         ],
         containers: utils.namedObjectList(self._containers),
         _containers:: {
+          "grafana-agent": {
+            name: "grafana-agent",
+            image: "docker.io/grafana/agent:v0.39.1",
+            command: ["/bin/grafana-agent"],
+            args: [
+              "--config.file=$(CONFIG_FILE_PATH)",
+              "--metrics.wal-directory=$(DATA_FILE_PATH)",
+              "--config.expand-env=true"
+            ],
+            env: utils.pairList(self._env),
+            _env:: {
+              CONFIG_FILE_PATH: "/etc/grafana-agent/agent.yml",
+              DATA_FILE_PATH: "/etc/grafana-agent/data"
+            },
+            envFrom: [
+              {
+                secretRef: {
+                  name: "grafana-cloud-secret-%s" % env.envName
+                }
+              }
+            ],
+            volumeMounts: utils.pairList(self._volumeMounts, vfield="mountPath"),
+            _volumeMounts:: {
+              "grafana-agent-config-volume": "/etc/grafana-agent",
+              "grafana-agent-data-volume": "/etc/grafana-agent/data"
+            }
+          },
           [env.appName]: {
             local thisContainer = self,
             name: env.appName,
@@ -96,6 +188,13 @@ local newDeployment(env, dockerImage) = {
               JVM_ARGS: (if (env.envName == "staging") then "-Dspring.datasource.hikari.maximum-pool-size=5 -Xms512M -Xmx1536M" else "-Xms4G -Xmx6G") + jvmPerfOptions,
               DEPLOYMENT_CONFIG: "%s/%s" % [ env.deploymentConfig.path, env.deploymentConfig.filename, ],
             },
+            envFrom: [
+              {
+                secretRef: {
+                  name: "grafana-cloud-secret-%s" % env.envName
+                }
+              }
+            ],
             ports: utils.pairList(self._ports, vfield="containerPort"),
             _ports:: {
               http: 8080,
@@ -153,6 +252,22 @@ local newDeployment(env, dockerImage) = {
         },
         volumes: utils.namedObjectList(self._volumes),
         _volumes:: {
+          "grafana-agent-config-volume": {
+            configMap: {
+              name: "grafana-agent-configmap",
+              items: [
+                {
+                  key: "agent.yml",
+                  path: "agent.yml"
+                }
+              ]
+            }
+          },
+          "grafana-agent-data-volume": {
+            emptyDir: {
+              medium: "Memory"
+            }
+          },
           [deploymentConfigurationVolumeName]: {
             local thisVolume = self,
             secret: {
@@ -351,11 +466,13 @@ local newElasticSearchCluster(env) = {
 };
 
 local _newKubernetesResources(envName, image) = {
+  local configMap = newGrafanaAgentConfigMap(),
   local environment = newEnvironment(envName),
   local deployment = newDeployment(environment, image),
   local service = newService(environment, deployment),
 
   arr: [
+    configMap,
     deployment,
     service,
     newRoute(environment, service),

logging/logback-spring.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+    <include resource="org/springframework/boot/logging/logback/base.xml" />
+    <springProperty scope="context" name="appName" source="spring.application.name"/>
+
+    <appender name="LOKI" class="com.github.loki4j.logback.Loki4jAppender">
+        <verbose>true</verbose>
+        <batchMaxBytes>65536</batchMaxBytes>
+        <http>
+            <url>${LOKI_URL}</url>
+            <auth>
+                <username>${LOKI_USERNAME}</username>
+                <password>${LOKI_PASSWORD}</password>
+            </auth>
+            <requestTimeoutMs>15000</requestTimeoutMs>
+        </http>
+        <format>
+            <label>
+                <pattern>job=loki4j,instance=${HOSTNAME},application=${appName},traceID=%X{traceId:-NONE},level=%level</pattern>
+            </label>
+            <message>
+                <pattern>${FILE_LOG_PATTERN}</pattern>
+            </message>
+            <sortByTime>true</sortByTime>
+        </format>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="LOKI"/>
+    </root>
+</configuration>