Merge pull request #3145 from alexweav/log-warn

Emit warning-level logs when truncating messages in notifications
prometheus · Dec 13, 2022 · 217524d · 217524d
2 parents ace5662 + 0eee3ca
commit 217524d
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 28 deletions.
diff --git a/notify/opsgenie/opsgenie.go b/notify/opsgenie/opsgenie.go
@@ -34,6 +34,9 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
+const maxMessageLenRunes = 130
+
 // Notifier implements a Notifier for OpsGenie notifications.
 type Notifier struct {
 	conf    *config.OpsGenieConfig
@@ -171,10 +174,9 @@ func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*h
 		}
 		requests = append(requests, req.WithContext(ctx))
 	default:
-		// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
-		message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), 130)
+		message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
 		if truncated {
-			level.Debug(n.logger).Log("msg", "truncated message", "truncated_message", message, "alert", key)
+			level.Warn(n.logger).Log("msg", "Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
 		}
 
 		createEndpointURL := n.conf.APIURL.Copy()

diff --git a/notify/pagerduty/pagerduty.go b/notify/pagerduty/pagerduty.go
@@ -36,7 +36,13 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
-const maxEventSize int = 512000
+const (
+	maxEventSize int = 512000
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event - 1024 characters or runes.
+	maxV1DescriptionLenRunes = 1024
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1024 characters or runes.
+	maxV2SummaryLenRunes = 1024
+)
 
 // Notifier implements a Notifier for PagerDuty notifications.
 type Notifier struct {
@@ -149,10 +155,9 @@ func (n *Notifier) notifyV1(
 	var tmplErr error
 	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
 
-	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1204 characters or runes.
-	description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), 1024)
+	description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
 	if truncated {
-		level.Debug(n.logger).Log("msg", "Truncated description", "description", description, "key", key)
+		level.Warn(n.logger).Log("msg", "Truncated description", "key", key, "max_runes", maxV1DescriptionLenRunes)
 	}
 
 	serviceKey := string(n.conf.ServiceKey)
@@ -215,10 +220,9 @@ func (n *Notifier) notifyV2(
 		n.conf.Severity = "error"
 	}
 
-	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1204 characters or runes.
-	summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), 1024)
+	summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
 	if truncated {
-		level.Debug(n.logger).Log("msg", "Truncated summary", "summary", summary, "key", key)
+		level.Warn(n.logger).Log("msg", "Truncated summary", "key", key, "max_runes", maxV2SummaryLenRunes)
 	}
 
 	routingKey := string(n.conf.RoutingKey)

diff --git a/notify/pushover/pushover.go b/notify/pushover/pushover.go
@@ -31,6 +31,15 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+const (
+	// https://pushover.net/api#limits - 250 characters or runes.
+	maxTitleLenRunes = 250
+	// https://pushover.net/api#limits - 1024 characters or runes.
+	maxMessageLenRunes = 1024
+	// https://pushover.net/api#limits - 512 characters or runes.
+	maxURLLenRunes = 512
+)
+
 // Notifier implements a Notifier for Pushover notifications.
 type Notifier struct {
 	conf    *config.PushoverConfig
@@ -78,10 +87,9 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	parameters.Add("token", tmpl(string(n.conf.Token)))
 	parameters.Add("user", tmpl(string(n.conf.UserKey)))
 
-	// https://pushover.net/api#limits - 250 characters or runes.
-	title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), 250)
+	title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), maxTitleLenRunes)
 	if truncated {
-		level.Debug(n.logger).Log("msg", "Truncated title", "truncated_title", title, "incident", key)
+		level.Warn(n.logger).Log("msg", "Truncated title", "incident", key, "max_runes", maxTitleLenRunes)
 	}
 	parameters.Add("title", title)
 
@@ -92,10 +100,9 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		message = tmpl(n.conf.Message)
 	}
 
-	// https://pushover.net/api#limits - 1024 characters or runes.
-	message, truncated = notify.TruncateInRunes(message, 1024)
+	message, truncated = notify.TruncateInRunes(message, maxMessageLenRunes)
 	if truncated {
-		level.Debug(n.logger).Log("msg", "Truncated message", "truncated_message", message, "incident", key)
+		level.Warn(n.logger).Log("msg", "Truncated message", "incident", key, "max_runes", maxMessageLenRunes)
 	}
 	message = strings.TrimSpace(message)
 	if message == "" {
@@ -104,10 +111,9 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	}
 	parameters.Add("message", message)
 
-	// https://pushover.net/api#limits - 512 characters or runes.
-	supplementaryURL, truncated := notify.TruncateInRunes(tmpl(n.conf.URL), 512)
+	supplementaryURL, truncated := notify.TruncateInRunes(tmpl(n.conf.URL), maxURLLenRunes)
 	if truncated {
-		level.Debug(n.logger).Log("msg", "Truncated URL", "truncated_url", supplementaryURL, "incident", key)
+		level.Warn(n.logger).Log("msg", "Truncated URL", "incident", key, "max_runes", maxURLLenRunes)
 	}
 	parameters.Add("url", supplementaryURL)
 	parameters.Add("url_title", tmpl(n.conf.URLTitle))

diff --git a/notify/slack/slack.go b/notify/slack/slack.go
@@ -33,6 +33,9 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
+const maxTitleLenRunes = 1024
+
 // Notifier implements a Notifier for Slack notifications.
 type Notifier struct {
 	conf    *config.SlackConfig
@@ -99,14 +102,14 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	} else {
 		markdownIn = n.conf.MrkdwnIn
 	}
-	// No reference in https://api.slack.com/reference/messaging/attachments#legacy_fields - assuming runes or characters.
-	title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), 1024)
+
+	title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), maxTitleLenRunes)
 	if truncated {
 		key, err := notify.ExtractGroupKey(ctx)
 		if err != nil {
 			return false, err
 		}
-		level.Debug(n.logger).Log("msg", "Truncated title", "text", title, "key", key)
+		level.Warn(n.logger).Log("msg", "Truncated title", "key", key, "max_runes", maxTitleLenRunes)
 	}
 	att := &attachment{
 		Title:      title,

diff --git a/notify/telegram/telegram.go b/notify/telegram/telegram.go
@@ -15,6 +15,7 @@ package telegram
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 
 	"github.com/go-kit/log"
@@ -28,6 +29,9 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+// Telegram supports 4096 chars max - from https://limits.tginfo.me/en.
+const maxMessageLenRunes = 4096
+
 // Notifier implements a Notifier for telegram notifications.
 type Notifier struct {
 	conf    *config.TelegramConfig
@@ -65,10 +69,14 @@ func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, err
 		tmpl = notify.TmplText(n.tmpl, data, &err)
 	)
 
-	// Telegram supports 4096 chars max - from https://limits.tginfo.me/en.
-	messageText, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), 4096)
+	key, ok := notify.GroupKey(ctx)
+	if !ok {
+		return false, fmt.Errorf("group key missing")
+	}
+
+	messageText, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
 	if truncated {
-		level.Debug(n.logger).Log("msg", "truncated message", "truncated_message", messageText)
+		level.Warn(n.logger).Log("msg", "Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
 	}
 
 	message, err := n.client.Send(telebot.ChatID(n.conf.ChatID), messageText, &telebot.SendOptions{

diff --git a/notify/victorops/victorops.go b/notify/victorops/victorops.go
@@ -34,6 +34,9 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+// https://help.victorops.com/knowledge-base/incident-fields-glossary/ - 20480 characters.
+const maxMessageLenRunes = 20480
+
 // Notifier implements a Notifier for VictorOps notifications.
 type Notifier struct {
 	conf    *config.VictorOpsConfig
@@ -134,10 +137,9 @@ func (n *Notifier) createVictorOpsPayload(ctx context.Context, as ...*types.Aler
 		messageType = victorOpsEventResolve
 	}
 
-	// https://help.victorops.com/knowledge-base/incident-fields-glossary/ - 20480 characters.
-	stateMessage, truncated := notify.TruncateInRunes(stateMessage, 20480)
+	stateMessage, truncated := notify.TruncateInRunes(stateMessage, maxMessageLenRunes)
 	if truncated {
-		level.Debug(n.logger).Log("msg", "truncated stateMessage", "truncated_state_message", stateMessage, "incident", key)
+		level.Warn(n.logger).Log("msg", "Truncated state_message", "incident", key, "max_runes", maxMessageLenRunes)
 	}
 
 	msg := map[string]string{