Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Emit warning-level logs when truncating messages in notifications #3145

Merged
merged 7 commits into from Dec 13, 2022
Merged

Emit warning-level logs when truncating messages in notifications #3145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
bb5804e
Change truncation log level to warn
alexweav Nov 25, 2022
ed3cfbb
Avoid logging the entire message after truncation
alexweav Nov 25, 2022
7a420fe
Adjust degenerate log message in telegram notifier
alexweav Nov 25, 2022
42dcba2
Also log the length that values were truncated to
alexweav Nov 25, 2022
18181af
Fix one last log level that I missed
alexweav Nov 25, 2022
9b9237c
Fix casing in victorops log message
alexweav Nov 25, 2022
0eee3ca
Rename key to max_runes
alexweav Dec 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 5 additions & 3 deletions notify/opsgenie/opsgenie.go
View file
Open in desktop
Expand Up @@ -34,6 +34,9 @@ import (
"github.com/prometheus/alertmanager/types"
)

// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
const maxMessageLenRunes = 130

// Notifier implements a Notifier for OpsGenie notifications.
type Notifier struct {
conf *config.OpsGenieConfig
Expand Down Expand Up @@ -171,10 +174,9 @@ func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*h
}
requests = append(requests, req.WithContext(ctx))
default:
// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), 130)
message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "truncated message", "truncated_message", message, "alert", key)
level.Warn(n.logger).Log("msg", "Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
}

createEndpointURL := n.conf.APIURL.Copy()
Expand Down
18 changes: 11 additions & 7 deletions notify/pagerduty/pagerduty.go
View file
Open in desktop
Expand Up @@ -36,7 +36,13 @@ import (
"github.com/prometheus/alertmanager/types"
)

const maxEventSize int = 512000
const (
maxEventSize int = 512000
// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event - 1024 characters or runes.
maxV1DescriptionLenRunes = 1024
// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1024 characters or runes.
maxV2SummaryLenRunes = 1024
)

// Notifier implements a Notifier for PagerDuty notifications.
type Notifier struct {
Expand Down Expand Up @@ -149,10 +155,9 @@ func (n *Notifier) notifyV1(
var tmplErr error
tmpl := notify.TmplText(n.tmpl, data, &tmplErr)

// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1204 characters or runes.
description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), 1024)
description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated description", "description", description, "key", key)
level.Warn(n.logger).Log("msg", "Truncated description", "key", key, "max_runes", maxV1DescriptionLenRunes)
}

serviceKey := string(n.conf.ServiceKey)
Expand Down Expand Up @@ -215,10 +220,9 @@ func (n *Notifier) notifyV2(
n.conf.Severity = "error"
}

// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1204 characters or runes.
summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), 1024)
summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated summary", "summary", summary, "key", key)
level.Warn(n.logger).Log("msg", "Truncated summary", "key", key, "max_runes", maxV2SummaryLenRunes)
}

routingKey := string(n.conf.RoutingKey)
Expand Down
24 changes: 15 additions & 9 deletions notify/pushover/pushover.go
View file
Open in desktop
Expand Up @@ -31,6 +31,15 @@ import (
"github.com/prometheus/alertmanager/types"
)

const (
// https://pushover.net/api#limits - 250 characters or runes.
maxTitleLenRunes = 250
// https://pushover.net/api#limits - 1024 characters or runes.
maxMessageLenRunes = 1024
// https://pushover.net/api#limits - 512 characters or runes.
maxURLLenRunes = 512
)

// Notifier implements a Notifier for Pushover notifications.
type Notifier struct {
conf *config.PushoverConfig
Expand Down Expand Up @@ -78,10 +87,9 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
parameters.Add("token", tmpl(string(n.conf.Token)))
parameters.Add("user", tmpl(string(n.conf.UserKey)))

// https://pushover.net/api#limits - 250 characters or runes.
title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), 250)
title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), maxTitleLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated title", "truncated_title", title, "incident", key)
level.Warn(n.logger).Log("msg", "Truncated title", "incident", key, "max_runes", maxTitleLenRunes)
}
parameters.Add("title", title)

Expand All @@ -92,10 +100,9 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
message = tmpl(n.conf.Message)
}

// https://pushover.net/api#limits - 1024 characters or runes.
message, truncated = notify.TruncateInRunes(message, 1024)
message, truncated = notify.TruncateInRunes(message, maxMessageLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated message", "truncated_message", message, "incident", key)
level.Warn(n.logger).Log("msg", "Truncated message", "incident", key, "max_runes", maxMessageLenRunes)
}
message = strings.TrimSpace(message)
if message == "" {
Expand All @@ -104,10 +111,9 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
}
parameters.Add("message", message)

// https://pushover.net/api#limits - 512 characters or runes.
supplementaryURL, truncated := notify.TruncateInRunes(tmpl(n.conf.URL), 512)
supplementaryURL, truncated := notify.TruncateInRunes(tmpl(n.conf.URL), maxURLLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "Truncated URL", "truncated_url", supplementaryURL, "incident", key)
level.Warn(n.logger).Log("msg", "Truncated URL", "incident", key, "max_runes", maxURLLenRunes)
}
parameters.Add("url", supplementaryURL)
parameters.Add("url_title", tmpl(n.conf.URLTitle))
Expand Down
9 changes: 6 additions & 3 deletions notify/slack/slack.go
View file
Open in desktop
Expand Up @@ -33,6 +33,9 @@ import (
"github.com/prometheus/alertmanager/types"
)

// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
const maxTitleLenRunes = 1024

// Notifier implements a Notifier for Slack notifications.
type Notifier struct {
conf *config.SlackConfig
Expand Down Expand Up @@ -99,14 +102,14 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
} else {
markdownIn = n.conf.MrkdwnIn
}
// No reference in https://api.slack.com/reference/messaging/attachments#legacy_fields - assuming runes or characters.
title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), 1024)

title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), maxTitleLenRunes)
if truncated {
key, err := notify.ExtractGroupKey(ctx)
if err != nil {
return false, err
}
level.Debug(n.logger).Log("msg", "Truncated title", "text", title, "key", key)
level.Warn(n.logger).Log("msg", "Truncated title", "key", key, "max_runes", maxTitleLenRunes)
}
att := &attachment{
Title: title,
Expand Down
14 changes: 11 additions & 3 deletions notify/telegram/telegram.go
View file
Open in desktop
Expand Up @@ -15,6 +15,7 @@ package telegram

import (
"context"
"fmt"
"net/http"

"github.com/go-kit/log"
Expand All @@ -28,6 +29,9 @@ import (
"github.com/prometheus/alertmanager/types"
)

// Telegram supports 4096 chars max - from https://limits.tginfo.me/en.
const maxMessageLenRunes = 4096

// Notifier implements a Notifier for telegram notifications.
type Notifier struct {
conf *config.TelegramConfig
Expand Down Expand Up @@ -65,10 +69,14 @@ func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, err
tmpl = notify.TmplText(n.tmpl, data, &err)
)

// Telegram supports 4096 chars max - from https://limits.tginfo.me/en.
messageText, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), 4096)
key, ok := notify.GroupKey(ctx)
if !ok {
return false, fmt.Errorf("group key missing")
}

messageText, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "truncated message", "truncated_message", messageText)
level.Warn(n.logger).Log("msg", "Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
}

message, err := n.client.Send(telebot.ChatID(n.conf.ChatID), messageText, &telebot.SendOptions{
Expand Down
8 changes: 5 additions & 3 deletions notify/victorops/victorops.go
View file
Open in desktop
Expand Up @@ -34,6 +34,9 @@ import (
"github.com/prometheus/alertmanager/types"
)

// https://help.victorops.com/knowledge-base/incident-fields-glossary/ - 20480 characters.
const maxMessageLenRunes = 20480

// Notifier implements a Notifier for VictorOps notifications.
type Notifier struct {
conf *config.VictorOpsConfig
Expand Down Expand Up @@ -134,10 +137,9 @@ func (n *Notifier) createVictorOpsPayload(ctx context.Context, as ...*types.Aler
messageType = victorOpsEventResolve
}

// https://help.victorops.com/knowledge-base/incident-fields-glossary/ - 20480 characters.
stateMessage, truncated := notify.TruncateInRunes(stateMessage, 20480)
stateMessage, truncated := notify.TruncateInRunes(stateMessage, maxMessageLenRunes)
if truncated {
level.Debug(n.logger).Log("msg", "truncated stateMessage", "truncated_state_message", stateMessage, "incident", key)
level.Warn(n.logger).Log("msg", "Truncated state_message", "incident", key, "max_runes", maxMessageLenRunes)
}

msg := map[string]string{
Expand Down