-
Notifications
You must be signed in to change notification settings - Fork 126
/
metrics.go
104 lines (94 loc) · 2.93 KB
/
metrics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
package simulator
import (
"context"
"fmt"
"github.com/armadaproject/armada/pkg/armadaevents"
"strings"
"time"
)
type MetricsCollector struct {
c <-chan *armadaevents.EventSequence
Total Metrics
MetricsByQueue map[string]Metrics
LastSeenEvent *armadaevents.EventSequence_Event
}
func NewMetricsCollector(c <-chan *armadaevents.EventSequence) *MetricsCollector {
return &MetricsCollector{
c: c,
MetricsByQueue: make(map[string]Metrics),
}
}
func (mc *MetricsCollector) String() string {
var sb strings.Builder
sb.WriteString("{")
sb.WriteString(fmt.Sprintf("Total: %s, Queues: {", mc.Total))
i := 0
for queue, metrics := range mc.MetricsByQueue {
sb.WriteString(fmt.Sprintf("%s: %s", queue, metrics))
i++
if i != len(mc.MetricsByQueue) {
sb.WriteString(", ")
}
}
sb.WriteString("}}")
return sb.String()
}
type Metrics struct {
LastJobSuccess time.Duration
NumEventsInTotal int
NumPreemptionEvents int
NumSchedulingEvents int
NumJobsSubmitted int
NumSchedules int
NumSuccesses int
}
func (m Metrics) String() string {
return fmt.Sprintf(
"{LastJobSuccess: %s, NumEventsInTotal: %d, NumPreemptionEvents: %d, NumSchedulingEvents: %d}",
m.LastJobSuccess, m.NumEventsInTotal, m.NumPreemptionEvents, m.NumSchedulingEvents,
)
}
func (mc *MetricsCollector) Run(ctx context.Context) error {
for {
select {
case <-ctx.Done():
return ctx.Err()
case eventSequence, ok := <-mc.c:
if !ok {
return nil
}
mc.addEventSequence(eventSequence)
mc.logLiveMetrics()
}
}
}
func (mc *MetricsCollector) addEventSequence(eventSequence *armadaevents.EventSequence) {
queue := eventSequence.Queue
mc.Total.NumEventsInTotal += 1
entry := mc.MetricsByQueue[queue]
entry.NumEventsInTotal += 1
for _, event := range eventSequence.Events {
mc.LastSeenEvent = event
switch event.GetEvent().(type) {
case *armadaevents.EventSequence_Event_SubmitJob:
mc.Total.NumJobsSubmitted += 1
entry.NumJobsSubmitted += 1
case *armadaevents.EventSequence_Event_JobRunLeased:
mc.Total.NumSchedulingEvents += 1
entry.NumSchedulingEvents += 1
case *armadaevents.EventSequence_Event_JobRunPreempted:
mc.Total.NumPreemptionEvents += 1
entry.NumPreemptionEvents += 1
case *armadaevents.EventSequence_Event_JobSucceeded:
mc.Total.LastJobSuccess = event.Created.Sub(time.Time{})
entry.LastJobSuccess = event.Created.Sub(time.Time{})
mc.Total.NumSuccesses += 1
}
}
mc.MetricsByQueue[queue] = entry
}
func (mc *MetricsCollector) logLiveMetrics() {
if mc.Total.NumEventsInTotal%500 == 0 {
fmt.Printf("Total Events: %d, Total Successes: %d, Total Preemptions: %d, Total Submissions: %d, Total Queued: %d, Time Simulated %s, Last Seen Event %s\n", mc.Total.NumEventsInTotal, mc.Total.NumSuccesses, mc.Total.NumPreemptionEvents, mc.Total.NumJobsSubmitted, mc.Total.NumJobsSubmitted-mc.Total.NumSuccesses, mc.Total.LastJobSuccess, EventSummary(mc.LastSeenEvent))
}
}