feat: add new opt-in metric of reloads by namespace

Author: alexanderldavis

README.md

@@ -328,6 +328,7 @@ helm uninstall {{RELEASE_NAME}} -n {{NAMESPACE}}
 | `reloader.readOnlyRootFileSystem` | Enforce readOnlyRootFilesystem                                                                                                                      | boolean     | `false`   |
 | `reloader.legacy.rbac`            |                                                                                                                                                     | boolean     | `false`   |
 | `reloader.matchLabels`            | Pod labels to match                                                                                                                                 | map         | `{}`      |
+| `reloader.enableMetricsByNamespace`            | Expose an additional Prometheus counter of reloads by namespace (this metric may have high cardinality in clusters with many namespaces) | boolean         | `false`      |
 
 #### Deployment Reloader Parameters
 

deployments/kubernetes/chart/reloader/templates/deployment.yaml

@@ -76,7 +76,7 @@ spec:
       - image: "{{ .Values.reloader.deployment.image.name }}:{{ .Values.reloader.deployment.image.tag }}"
         imagePullPolicy: {{ .Values.reloader.deployment.image.pullPolicy }}
         name: {{ template "reloader-fullname" . }}
-      {{- if or (.Values.reloader.deployment.env.open) (.Values.reloader.deployment.env.secret) (.Values.reloader.deployment.env.field) (.Values.reloader.deployment.env.existing) (eq .Values.reloader.watchGlobally false) (.Values.reloader.enableHA)}}
+      {{- if or (.Values.reloader.deployment.env.open) (.Values.reloader.deployment.env.secret) (.Values.reloader.deployment.env.field) (.Values.reloader.deployment.env.existing) (eq .Values.reloader.watchGlobally false) (.Values.reloader.enableHA) (.Values.reloader.enableMetricsByNamespace)}}
         env:
       {{- range $name, $value := .Values.reloader.deployment.env.open }}
       {{- if not (empty $value) }}
@@ -129,6 +129,10 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
       {{- end }}
+      {{- if .Values.reloader.enableMetricsByNamespace }}
+        - name: METRICS_COUNT_BY_NAMESPACE
+          value: enabled
+      {{- end }}
       {{- end }}
 
         ports:

deployments/kubernetes/chart/reloader/values.yaml

@@ -32,6 +32,8 @@ reloader:
   legacy:
     rbac: false
   matchLabels: {}
+  # Set to true to expose a prometheus counter of reloads by namespace (this metric may have high cardinality in clusters with many namespaces)
+  enableMetricsByNamespace: false
   deployment:
     # If you wish to run multiple replicas set reloader.enableHA = true
     replicas: 1

deployments/kubernetes/templates/chart/values.yaml.tmpl

@@ -23,6 +23,8 @@ reloader:
   legacy:
     rbac: false
   matchLabels: {}
+  # Set to true to expose a prometheus counter of reloads by namespace (this metric may have high cardinality in clusters with many namespaces)
+  enableMetricsByNamespace: false
   deployment:
     replicas: 1
     nodeSelector:

internal/pkg/handler/upgrade.go

@@ -245,6 +245,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				message := fmt.Sprintf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err)
 				logrus.Errorf(message)
 				collectors.Reloaded.With(prometheus.Labels{"success": "false"}).Inc()
+				collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "false", "namespace": config.Namespace}).Inc()
 				if recorder != nil {
 					recorder.Event(i, v1.EventTypeWarning, "ReloadFail", message)
 				}
@@ -254,6 +255,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				message += fmt.Sprintf(", Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace)
 				logrus.Infof(message)
 				collectors.Reloaded.With(prometheus.Labels{"success": "true"}).Inc()
+				collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": config.Namespace}).Inc()
 				alert_on_reload, ok := os.LookupEnv("ALERT_ON_RELOAD")
 				if recorder != nil {
 					recorder.Event(i, v1.EventTypeNormal, "Reloaded", message)

internal/pkg/handler/upgrade_test.go

@@ -1,13 +1,16 @@
 package metrics
 
 import (
+	"net/http"
+	"os"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"net/http"
 )
 
 type Collectors struct {
-	Reloaded *prometheus.CounterVec
+	Reloaded            *prometheus.CounterVec
+	ReloadedByNamespace *prometheus.CounterVec
 }
 
 func NewCollectors() Collectors {
@@ -17,21 +20,40 @@ func NewCollectors() Collectors {
 			Name:      "reload_executed_total",
 			Help:      "Counter of reloads executed by Reloader.",
 		},
-		[]string{"success"},
+		[]string{
+			"success",
+		},
 	)
 
 	//set 0 as default value
 	reloaded.With(prometheus.Labels{"success": "true"}).Add(0)
 	reloaded.With(prometheus.Labels{"success": "false"}).Add(0)
 
+	reloaded_by_namespace := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "reloader",
+			Name:      "reload_executed_total_by_namespace",
+			Help:      "Counter of reloads executed by Reloader by namespace.",
+		},
+		[]string{
+			"success",
+			"namespace",
+		},
+	)
 	return Collectors{
-		Reloaded: reloaded,
+		Reloaded:            reloaded,
+		ReloadedByNamespace: reloaded_by_namespace,
 	}
 }
 
 func SetupPrometheusEndpoint() Collectors {
 	collectors := NewCollectors()
 	prometheus.MustRegister(collectors.Reloaded)
+
+	if os.Getenv("METRICS_COUNT_BY_NAMESPACE") == "enabled" {
+		prometheus.MustRegister(collectors.ReloadedByNamespace)
+	}
+
 	http.Handle("/metrics", promhttp.Handler())
 
 	return collectors