1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
package internal
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"strings"
"time"
)
type prometheusResponse struct {
Status string `json:"status"`
Data struct {
Alerts []prometheusAlert `json:"alerts"`
} `json:"data"`
}
type prometheusAlert struct {
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
State string `json:"state"`
}
func mergePrometheusAlerts(ctx context.Context, state state, conf config) state {
if len(conf.PrometheusHosts) == 0 {
return state
}
timeout := time.Duration(conf.PrometheusTimeoutS) * time.Second
alerts, host, err := fetchPrometheusAlerts(ctx, conf.PrometheusHosts, timeout)
if err != nil {
log.Printf("Failed to fetch Prometheus alerts from any host: %v", err)
checkName := "Prometheus alerts"
newStatus := nagiosWarning
if prevState, ok := state.checks[checkName]; ok && prevState.Status == newStatus {
if prevState.PrevStatus != newStatus {
prevState.PrevStatus = newStatus
state.checks[checkName] = prevState
}
return state
}
cs := checkResult{
name: checkName,
output: fmt.Sprintf("WARNING: %v", err),
epoch: time.Now().Unix(),
status: newStatus,
}
state.update(cs)
return state
}
log.Printf("Fetched %d firing alerts from Prometheus host %s", len(alerts), host)
// Clear the "Prometheus alerts" check if fetch succeeded (was previously failing)
checkName := "Prometheus alerts"
if prevState, ok := state.checks[checkName]; ok && prevState.Status != nagiosOk {
cs := checkResult{
name: checkName,
output: "OK: Prometheus connection restored",
epoch: time.Now().Unix(),
status: nagiosOk,
}
state.update(cs)
}
// Track currently firing alerts to clear resolved ones later
firingAlerts := make(map[string]bool)
watchdogFiring := false
for _, alert := range alerts {
alertname := alert.Labels["alertname"]
// Special handling for Prometheus Watchdog alert
if alertname == "Watchdog" {
if alert.State == "firing" {
watchdogFiring = true
firingAlerts["Prometheus: Watchdog"] = true
// Watchdog is firing as expected, treat as OK
cs := checkResult{
name: "Prometheus: Watchdog",
output: "OK [none]: Alertmanager is working properly",
epoch: time.Now().Unix(),
status: nagiosOk,
}
state.update(cs)
}
continue
}
if alert.State != "firing" {
continue
}
name := fmt.Sprintf("Prometheus: %s", alertname)
firingAlerts[name] = true
severity := alert.Labels["severity"]
description := alert.Annotations["summary"]
if description == "" {
description = alert.Annotations["description"]
}
if description == "" {
description = "no description"
}
status := nagiosWarning
if severity == "critical" {
status = nagiosCritical
}
cs := checkResult{
name: name,
output: fmt.Sprintf("%s [%s]: %s", alertname, severity, description),
epoch: time.Now().Unix(),
status: status,
}
state.update(cs)
}
// If Watchdog is not firing, alert as critical
if !watchdogFiring {
firingAlerts["Prometheus: Watchdog"] = true
cs := checkResult{
name: "Prometheus: Watchdog",
output: "CRITICAL [none]: Watchdog alert is not firing, Alertmanager may not be working",
epoch: time.Now().Unix(),
status: nagiosCritical,
}
state.update(cs)
}
// Clear any Prometheus alerts that are no longer firing
clearResolvedPrometheusAlerts(state, firingAlerts)
return state
}
// clearResolvedPrometheusAlerts removes Prometheus alerts from state that are
// no longer firing. This prevents stale alerts from accumulating.
func clearResolvedPrometheusAlerts(state state, firingAlerts map[string]bool) {
const prometheusPrefix = "Prometheus: "
for name := range state.checks {
// Skip non-Prometheus alerts and the connection status check
if !strings.HasPrefix(name, prometheusPrefix) || name == "Prometheus alerts" {
continue
}
// If this alert is not currently firing, remove it from state
if !firingAlerts[name] {
delete(state.checks, name)
log.Printf("Cleared resolved Prometheus alert: %s", name)
}
}
}
func fetchPrometheusAlerts(ctx context.Context, hosts []string, timeout time.Duration) ([]prometheusAlert, string, error) {
var lastErr error
for _, host := range hosts {
alerts, err := fetchFromHost(ctx, host, timeout)
if err != nil {
log.Printf("Failed to fetch from Prometheus host %s: %v", host, err)
lastErr = err
continue
}
return alerts, host, nil
}
return nil, "", fmt.Errorf("all Prometheus hosts failed, last error: %w", lastErr)
}
func fetchFromHost(ctx context.Context, host string, timeout time.Duration) ([]prometheusAlert, error) {
url := fmt.Sprintf("http://%s/api/v1/alerts", host)
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
var promResp prometheusResponse
if err := json.Unmarshal(body, &promResp); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}
if promResp.Status != "success" {
return nil, fmt.Errorf("prometheus returned status: %s", promResp.Status)
}
return promResp.Data.Alerts, nil
}
|
