Add custom chat ID option in alertmanager API and telegram notifi… (#19)

Add custom chat ID option in alertmanager API and telegram notification

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## [unreleased] - YYYY-MM-DD
 
+### Added
+
+- Alertmanager API accepts a query string param with a custom chat ID.
+- Telegram notifier can send to customized chats.
+
 ## [0.1.0] - 2019-12-13
 
 ### Added

README.md

@@ -6,9 +6,25 @@ Alertgram is the easiest way to forward alerts to [Telegram] (Supports [Promethe
     <img src="https://i.imgur.com/4jdOFj9.jpg" width="40%" align="center" alt="alertgram">
 </p>
 
+## Table of contents
+
+- [Introduction](#introduction)
+- [Input alerts](#input-alerts)
+- [Options](#options)
+- [Run](#run)
+  - [Simple example](#simple-example)
+  - [Production](#production)
+- [Metrics](#metrics)
+- [Development and debugging](#development-and-debugging)
+- [FAQ](#faq)
+  - [Only alertmanager alerts are supported?](#only-alertmanager-alerts-are-supported-)
+  - [Where does alertgram listen to alertmanager alerts?](#where-does-alertgram-listen-to-alertmanager-alerts-)
+  - [Can I notify to different chats?](#can-i-notify-to-different-chats-)
+  - [Can I use custom templates?](#can-i-use-custom-templates-)
+
 ## Introduction
 
-Everything started as a way of forwarding [Prometheus alertmanager] alerts to [Telegram] because the solutions that I found where too complex, I just wanted to forward alerts to channels without trouble. And Alertgram is just that, a simple app that forwards alerts to Telegram groups and channels.
+Everything started as a way of forwarding [Prometheus alertmanager] alerts to [Telegram] because the solutions that I found were too complex, I just wanted to forward alerts to channels without trouble. And Alertgram is just that, a simple app that forwards alerts to Telegram groups and channels.
 
 ## Input alerts
 
@@ -58,7 +74,28 @@ Also remember that you can use `--debug` flag.
 
 ## FAQ
 
-### Can I use custom template?
+### Only alertmanager alerts are supported?
+
+At this moment yes, but we can add more input alert systems if you want, create an issue
+so we can discuss and implement.
+
+### Where does alertgram listen to alertmanager alerts?
+
+By default in `0.0.0.0:8080/alerts`, but you can use `--alertmanager.listen-address` and
+`--alertmanager.webhook-path` to customize.
+
+### Can I notify to different chats?
+
+There are 3 levels where you could customize the notification chat:
+
+- By default: Using the required `--telegram.chat-id` flag.
+- At URL level: using [query string] parameter, e.g. `0.0.0.0:8080/alerts?chat-id=-1009876543210`.
+  This query param can be customized with `--alertmanager.chat-id-query-string` flag.
+- At alert level: TODO
+
+The preference is in order from the lowest to the highest: Default, URL, Alert.
+
+### Can I use custom templates?
 
 Yes!, use the flag `--notify.template-path`. You can check [testdata/templates](testdata/templates) for examples.
 
@@ -93,3 +130,4 @@ curl -i http://127.0.0.1:8080/alerts -d @./testdata/alerts/base.json
 [kubernetes-deployment]: docs/kubernetes
 [html go templates]: https://golang.org/pkg/html/template/
 [sprig]: http://masterminds.github.io/sprig
+[query string]: https://en.wikipedia.org/wiki/Query_string

cmd/alertgram/config.go

@@ -15,6 +15,7 @@ var (
 const (
 	descAMListenAddr       = "The listen address where the server will be listening to alertmanager's webhook request."
 	descAMWebhookPath      = "The path where the server will be handling the alertmanager webhook alert requests."
+	descAMChatIDQS         = "The optional query string key used to customize the chat id of the notification."
 	descTelegramAPIToken   = "The token that will be used to use the telegram API to send the alerts."
 	descTelegramDefChatID  = "The default ID of the chat (group/channel) in telegram where the alerts will be sent."
 	descMetricsListenAddr  = "The listen address where the metrics will be being served."
@@ -28,23 +29,25 @@ const (
 const (
 	defAMListenAddr      = ":8080"
 	defAMWebhookPath     = "/alerts"
+	defAMChatIDQS        = "chat-id"
 	defMetricsListenAddr = ":8081"
 	defMetricsPath       = "/metrics"
 	defMetricsHCPath     = "/status"
 )
 
 // Config has the configuration of the application.
 type Config struct {
-	AlertmanagerListenAddr  string
-	AlertmanagerWebhookPath string
-	TeletramAPIToken        string
-	TelegramChatID          int64
-	MetricsListenAddr       string
-	MetricsPath             string
-	MetricsHCPath           string
-	NotifyTemplate          *os.File
-	DebugMode               bool
-	NotifyDryRun            bool
+	AlertmanagerListenAddr         string
+	AlertmanagerWebhookPath        string
+	AlertmanagerChatIDQQueryString string
+	TeletramAPIToken               string
+	TelegramChatID                 int64
+	MetricsListenAddr              string
+	MetricsPath                    string
+	MetricsHCPath                  string
+	NotifyTemplate                 *os.File
+	DebugMode                      bool
+	NotifyDryRun                   bool
 
 	app *kingpin.Application
 }
@@ -71,6 +74,7 @@ func NewConfig() (*Config, error) {
 func (c *Config) registerFlags() {
 	c.app.Flag("alertmanager.listen-address", descAMListenAddr).Default(defAMListenAddr).StringVar(&c.AlertmanagerListenAddr)
 	c.app.Flag("alertmanager.webhook-path", descAMWebhookPath).Default(defAMWebhookPath).StringVar(&c.AlertmanagerWebhookPath)
+	c.app.Flag("alertmanager.chat-id-query-string", descAMChatIDQS).Default(defAMChatIDQS).StringVar(&c.AlertmanagerChatIDQQueryString)
 	c.app.Flag("telegram.api-token", descTelegramAPIToken).Required().StringVar(&c.TeletramAPIToken)
 	c.app.Flag("telegram.chat-id", descTelegramDefChatID).Required().Int64Var(&c.TelegramChatID)
 	c.app.Flag("metrics.listen-address", descMetricsListenAddr).Default(defMetricsListenAddr).StringVar(&c.MetricsListenAddr)

docs/alertmanager/README.md

@@ -26,11 +26,20 @@ route:
     group_wait: 30s
     group_interval: 5m
     repeat_interval: 3h
-    receiver: 'telegram'
+    receiver: telegram
+      routes:
+      # Only important alerts.
+      - match_re:
+          severity: ^(oncall|critical)$
+        receiver: telegram-oncall
 
 receivers:
-- name: 'telegram'
+- name: telegram
     webhook_configs:
     - url: 'http://alertgram:8080/alerts'
       send_resolved: false
+
+- name: telegram-oncall
+    webhook_configs:
+    - url: 'http://alertgram:8080/alerts?chat-id=-1001111111111'
 ```

docs/kubernetes/alertmanager-cfg.yaml

@@ -15,9 +15,33 @@ stringData:
       group_wait: 30s
       group_interval: 5m
       repeat_interval: 12h
-      receiver: 'telegram'
+      receiver: 'telegram-default'
+      routes:
+        # Warning alerts not important.
+        - match:
+            severity: warning
+          receiver: telegram-warning
+
+        # Only important alerts.
+        - match_re:
+            severity: ^(oncall|critical)$
+          receiver: telegram-oncall
+
     receivers:
-    - name: 'telegram'
+    # Use default alert channel for the default alerts.
+    - name: telegram-default
       webhook_configs:
       - url: 'http://alertgram:8080/alerts'
         send_resolved: false
+
+    # Critical and oncall alerts to special channel.
+    - name: telegram-oncall
+      webhook_configs:
+      - url: 'http://alertgram:8080/alerts?chat-id=-1001111111111'
+        send_resolved: false
+
+    # Warning alerts to a more public informative channel.
+    - name: telegram-warning
+      webhook_configs:
+      - url: 'http://alertgram:8080/alerts?chat-id=-1002222222222'
+        send_resolved: false

internal/forward/forward.go

@@ -9,10 +9,19 @@ import (
 	"github.com/slok/alertgram/internal/model"
 )
 
+// Properties are the  properties an AlertGroup can have
+// when the forwarding process is done.
+type Properties struct {
+	// CustomChatID can be used when the forward should be done
+	// to a different target (chat, group, channel, user...)
+	// instead of using the default one.
+	CustomChatID string
+}
+
 // Service is the domain service that forwards alerts
 type Service interface {
 	// Forward knows how to forward alerts from an input to an output.
-	Forward(ctx context.Context, alertGroup *model.AlertGroup) error
+	Forward(ctx context.Context, props Properties, alertGroup *model.AlertGroup) error
 }
 
 type service struct {
@@ -33,15 +42,19 @@ var (
 	ErrInvalidAlertGroup = errors.New("invalid alert group")
 )
 
-func (s service) Forward(ctx context.Context, alertGroup *model.AlertGroup) error {
+func (s service) Forward(ctx context.Context, props Properties, alertGroup *model.AlertGroup) error {
 	// TODO(slok): Add better validation.
 	if alertGroup == nil {
 		return fmt.Errorf("alertgroup can't be empty: %w", ErrInvalidAlertGroup)
 	}
 
 	// TODO(slok): Add concurrency using workers.
+	notification := Notification{
+		AlertGroup: *alertGroup,
+		ChatID:     props.CustomChatID,
+	}
 	for _, not := range s.notifiers {
-		err := not.Notify(ctx, alertGroup)
+		err := not.Notify(ctx, notification)
 		if err != nil {
 			s.logger.WithValues(log.KV{"notifier": not.Type(), "alertGroupID": alertGroup.ID}).
 				Errorf("could not notify alert group: %s", err)

internal/forward/forward_test.go

@@ -18,6 +18,7 @@ var errTest = errors.New("whatever")
 
 func TestServiceForward(t *testing.T) {
 	tests := map[string]struct {
+		props      forward.Properties
 		alertGroup *model.AlertGroup
 		mock       func(ns []*forwardmock.Notifier)
 		expErr     error
@@ -28,17 +29,23 @@ func TestServiceForward(t *testing.T) {
 		},
 
 		"A forwarded alerts should be send to all notifiers.": {
+			props: forward.Properties{
+				CustomChatID: "-1001234567890",
+			},
 			alertGroup: &model.AlertGroup{
 				ID:     "test-group",
 				Alerts: []model.Alert{model.Alert{Name: "test"}},
 			},
 			mock: func(ns []*forwardmock.Notifier) {
-				expAlertGroup := &model.AlertGroup{
-					ID:     "test-group",
-					Alerts: []model.Alert{model.Alert{Name: "test"}},
+				expNotification := forward.Notification{
+					ChatID: "-1001234567890",
+					AlertGroup: model.AlertGroup{
+						ID:     "test-group",
+						Alerts: []model.Alert{model.Alert{Name: "test"}},
+					},
 				}
 				for _, n := range ns {
-					n.On("Notify", mock.Anything, expAlertGroup).Once().Return(nil)
+					n.On("Notify", mock.Anything, expNotification).Once().Return(nil)
 				}
 			},
 		},
@@ -49,17 +56,19 @@ func TestServiceForward(t *testing.T) {
 				Alerts: []model.Alert{model.Alert{Name: "test"}},
 			},
 			mock: func(ns []*forwardmock.Notifier) {
-				expAlertGroup := &model.AlertGroup{
-					ID:     "test-group",
-					Alerts: []model.Alert{model.Alert{Name: "test"}},
+				expNotification := forward.Notification{
+					AlertGroup: model.AlertGroup{
+						ID:     "test-group",
+						Alerts: []model.Alert{model.Alert{Name: "test"}},
+					},
 				}
 				for i, n := range ns {
 					err := errTest
 					// Set error in the first one.
 					if i != 0 {
 						err = nil
 					}
-					n.On("Notify", mock.Anything, expAlertGroup).Once().Return(err)
+					n.On("Notify", mock.Anything, expNotification).Once().Return(err)
 					n.On("Type").Maybe().Return("")
 				}
 			},
@@ -75,7 +84,7 @@ func TestServiceForward(t *testing.T) {
 			test.mock([]*forwardmock.Notifier{mn1, mn2})
 
 			svc := forward.NewService([]forward.Notifier{mn1, mn2}, log.Dummy)
-			err := svc.Forward(context.TODO(), test.alertGroup)
+			err := svc.Forward(context.TODO(), test.props, test.alertGroup)
 
 			if test.expErr != nil && assert.Error(err) {
 				assert.True(errors.Is(err, test.expErr))

internal/forward/metrics.go

@@ -25,11 +25,11 @@ func NewMeasureService(rec ServiceMetricsRecorder, next Service) Service {
 	}
 }
 
-func (m measureService) Forward(ctx context.Context, ag *model.AlertGroup) (err error) {
+func (m measureService) Forward(ctx context.Context, props Properties, ag *model.AlertGroup) (err error) {
 	defer func(t0 time.Time) {
 		m.rec.ObserveForwardServiceOpDuration(ctx, "Forward", err == nil, time.Since(t0))
 	}(time.Now())
-	return m.next.Forward(ctx, ag)
+	return m.next.Forward(ctx, props, ag)
 }
 
 // NotifierMetricsRecorder knows how to record metrics on forward.Notifier.
@@ -52,11 +52,11 @@ func NewMeasureNotifier(rec NotifierMetricsRecorder, next Notifier) Notifier {
 	}
 }
 
-func (m measureNotifier) Notify(ctx context.Context, ag *model.AlertGroup) (err error) {
+func (m measureNotifier) Notify(ctx context.Context, n Notification) (err error) {
 	defer func(t0 time.Time) {
 		m.rec.ObserveForwardNotifierOpDuration(ctx, m.notifierType, "Notify", err == nil, time.Since(t0))
 	}(time.Now())
-	return m.next.Notify(ctx, ag)
+	return m.next.Notify(ctx, n)
 }
 
 func (m measureNotifier) Type() string {

internal/forward/notify.go

@@ -6,8 +6,18 @@ import (
 	"github.com/slok/alertgram/internal/model"
 )
 
+// Notification is the notification that wants to be send
+// via a notifier.
+type Notification struct {
+	// ChatID is an ID to send the notification. In
+	// Telegram could be a channel/group ID, in Slack
+	// a room or a user.
+	ChatID     string
+	AlertGroup model.AlertGroup
+}
+
 // Notifier knows how to notify alerts to different backends.
 type Notifier interface {
-	Notify(ctx context.Context, alertGroup *model.AlertGroup) error
+	Notify(ctx context.Context, notification Notification) error
 	Type() string
 }

internal/http/alertmanager/alertmanager.go

@@ -15,11 +15,12 @@ import (
 
 // Config is the configuration of the WebhookHandler.
 type Config struct {
-	MetricsRecorder metrics.Recorder
-	WebhookPath     string
-	Forwarder       forward.Service
-	Debug           bool
-	Logger          log.Logger
+	MetricsRecorder   metrics.Recorder
+	WebhookPath       string
+	ChatIDQueryString string
+	Forwarder         forward.Service
+	Debug             bool
+	Logger            log.Logger
 }
 
 func (c *Config) defaults() error {
@@ -31,6 +32,10 @@ func (c *Config) defaults() error {
 		return fmt.Errorf("forward can't be nil")
 	}
 
+	if c.ChatIDQueryString == "" {
+		c.ChatIDQueryString = "chat-id"
+	}
+
 	if c.Logger == nil {
 		c.Logger = log.Dummy
 	}