fix prometheus handling

6 files changed, 45 insertions, 29 deletions

diff --git a/internal/html.go b/internal/html.go
index d453124..02c9c49 100644
--- a/internal/html.go
+++ b/internal/html.go
@@ -176,12 +176,12 @@ func (s state) htmlReportBy(sb *strings.Builder, showStatusChange, isStaleReport
 		sb.WriteString(": ")
 		sb.WriteString(html.EscapeString(name))
 		sb.WriteString(": ")
-		sb.WriteString(html.EscapeString(cs.output))
+		sb.WriteString(html.EscapeString(cs.Output))
 
 		// Show federated source if applicable
 		if cs.federated() {
 			sb.WriteString(" [federated from ")
-			sb.WriteString(html.EscapeString(cs.federatedFrom))
+			sb.WriteString(html.EscapeString(cs.FederatedFrom))
 			sb.WriteString("]")
 		}
 
diff --git a/internal/html_test.go b/internal/html_test.go
index b22eea4..d482eda 100644
--- a/internal/html_test.go
+++ b/internal/html_test.go
@@ -88,19 +88,19 @@ func TestHtmlReportBy(t *testing.T) {
 				Status:     nagiosCritical,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Service is down",
+				Output:     "Service is down",
 			},
 			"check2": {
 				Status:     nagiosWarning,
 				PrevStatus: nagiosWarning,
 				Epoch:      now,
-				output:     "High load",
+				Output:     "High load",
 			},
 			"check3": {
 				Status:     nagiosOk,
 				PrevStatus: nagiosOk,
 				Epoch:      staleEpoch - 100, // stale
-				output:     "All good",
+				Output:     "All good",
 			},
 		},
 	}
@@ -156,13 +156,13 @@ func TestHtmlReportChanged(t *testing.T) {
 				Status:     nagiosCritical,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Service failed",
+				Output:     "Service failed",
 			},
 			"check2": {
 				Status:     nagiosOk,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Still OK",
+				Output:     "Still OK",
 			},
 		},
 	}
@@ -195,25 +195,25 @@ func TestHtmlReport(t *testing.T) {
 				Status:     nagiosCritical,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Service is down",
+				Output:     "Service is down",
 			},
 			"warning_check": {
 				Status:     nagiosWarning,
 				PrevStatus: nagiosWarning,
 				Epoch:      now,
-				output:     "High CPU usage",
+				Output:     "High CPU usage",
 			},
 			"ok_check": {
 				Status:     nagiosOk,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Everything fine",
+				Output:     "Everything fine",
 			},
 			"stale_check": {
 				Status:     nagiosOk,
 				PrevStatus: nagiosOk,
 				Epoch:      staleEpoch - 100,
-				output:     "Not updated recently",
+				Output:     "Not updated recently",
 			},
 		},
 	}
@@ -263,7 +263,7 @@ func TestHtmlEscaping(t *testing.T) {
 				Status:     nagiosCritical,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Output with <tags> & \"quotes\"",
+				Output:     "Output with <tags> & \"quotes\"",
 			},
 		},
 	}
@@ -305,7 +305,7 @@ func TestPersistHTMLReport(t *testing.T) {
 				Status:     nagiosCritical,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Test output",
+				Output:     "Test output",
 			},
 		},
 	}
@@ -361,8 +361,8 @@ func TestFederatedChecks(t *testing.T) {
 				Status:        nagiosCritical,
 				PrevStatus:    nagiosOk,
 				Epoch:         now,
-				output:        "Remote service down",
-				federatedFrom: "remote.example.com",
+				Output:        "Remote service down",
+				FederatedFrom: "remote.example.com",
 			},
 		},
 	}
@@ -393,7 +393,7 @@ func TestW3CCompliance(t *testing.T) {
 				Status:     nagiosCritical,
 				PrevStatus: nagiosOk,
 				Epoch:      now,
-				output:     "Test output",
+				Output:     "Test output",
 			},
 		},
 	}
diff --git a/internal/prometheus.go b/internal/prometheus.go
index ab92e1a..5bf2638 100644
--- a/internal/prometheus.go
+++ b/internal/prometheus.go
@@ -35,6 +35,10 @@ func mergePrometheusAlerts(ctx context.Context, state state, conf config) state
 		checkName := "Prometheus alerts"
 		newStatus := nagiosWarning
 		if prevState, ok := state.checks[checkName]; ok && prevState.Status == newStatus {
+			if prevState.PrevStatus != newStatus {
+				prevState.PrevStatus = newStatus
+				state.checks[checkName] = prevState
+			}
 			return state
 		}
 		cs := checkResult{
@@ -49,6 +53,18 @@ func mergePrometheusAlerts(ctx context.Context, state state, conf config) state
 
 	log.Printf("Fetched %d firing alerts from Prometheus host %s", len(alerts), host)
 
+	// Clear the "Prometheus alerts" check if fetch succeeded (was previously failing)
+	checkName := "Prometheus alerts"
+	if prevState, ok := state.checks[checkName]; ok && prevState.Status != nagiosOk {
+		cs := checkResult{
+			name:   checkName,
+			output: "OK: Prometheus connection restored",
+			epoch:  time.Now().Unix(),
+			status: nagiosOk,
+		}
+		state.update(cs)
+	}
+
 	// Check if Watchdog alert is firing
 	watchdogFiring := false
 
diff --git a/internal/prometheus_test.go b/internal/prometheus_test.go
index 150ba52..25976d7 100644
--- a/internal/prometheus_test.go
+++ b/internal/prometheus_test.go
@@ -202,8 +202,8 @@ func TestMergePrometheusAlertsWatchdogFiring(t *testing.T) {
 		t.Errorf("expected Watchdog status OK, got %v", watchdog.Status)
 	}
 
-	if !strings.Contains(watchdog.output, "working properly") {
-		t.Errorf("expected working properly message, got: %s", watchdog.output)
+	if !strings.Contains(watchdog.Output, "working properly") {
+		t.Errorf("expected working properly message, got: %s", watchdog.Output)
 	}
 
 	// Verify other alerts are still processed
@@ -255,7 +255,7 @@ func TestMergePrometheusAlertsWatchdogNotFiring(t *testing.T) {
 		t.Errorf("expected Watchdog status CRITICAL, got %v", watchdog.Status)
 	}
 
-	if !strings.Contains(watchdog.output, "not firing") {
-		t.Errorf("expected not firing message, got: %s", watchdog.output)
+	if !strings.Contains(watchdog.Output, "not firing") {
+		t.Errorf("expected not firing message, got: %s", watchdog.Output)
 	}
 }
diff --git a/internal/runchecks.go b/internal/runchecks.go
index e3e802f..6d88f54 100644
--- a/internal/runchecks.go
+++ b/internal/runchecks.go
@@ -44,10 +44,10 @@ func runChecks(ctx context.Context, state state, conf config) state {
 					int(age.Seconds()), age, check.RunInterval)
 				outputCh <- checkResult{
 					name:          check.name,
-					output:        lastCheckState.output,
+					output:        lastCheckState.Output,
 					epoch:         lastCheckState.Epoch,
 					status:        lastCheckState.Status,
-					federatedFrom: lastCheckState.federatedFrom,
+					federatedFrom: lastCheckState.FederatedFrom,
 				}
 				inputWg.Done()
 				continue
diff --git a/internal/state.go b/internal/state.go
index d1225f0..d71c71c 100644
--- a/internal/state.go
+++ b/internal/state.go
@@ -14,13 +14,13 @@ import (
 type checkState struct {
 	Status        nagiosCode
 	PrevStatus    nagiosCode
-	Epoch         int64 `json:"Epoch,omitempty"`
-	output        string
-	federatedFrom string
+	Epoch         int64  `json:"Epoch,omitempty"`
+	Output        string `json:"Output,omitempty"`
+	FederatedFrom string `json:"FederatedFrom,omitempty"`
 }
 
 func (cs checkState) federated() bool {
-	return cs.federatedFrom != ""
+	return cs.FederatedFrom != ""
 }
 
 func (cs checkState) changed() bool {
@@ -62,7 +62,7 @@ func newState(conf config) (state, error) {
 
 	var obsolete []string
 	for name := range s.checks {
-		if _, ok := conf.Checks[name]; !ok {
+		if _, ok := conf.Checks[name]; !ok && !strings.HasPrefix(name, "Prometheus") {
 			obsolete = append(obsolete, name)
 		}
 	}
@@ -240,10 +240,10 @@ func (s state) reportBy(sb *strings.Builder, showStatusChange, isStaleReport boo
 		sb.WriteString(": ")
 		sb.WriteString(name)
 		sb.WriteString(": ")
-		sb.WriteString(cs.output)
+		sb.WriteString(cs.Output)
 		if cs.federated() {
 			sb.WriteString(" [federated from ")
-			sb.WriteString(cs.federatedFrom)
+			sb.WriteString(cs.FederatedFrom)
 			sb.WriteString("]")
 		}