daemon: add /livez and /readyz HTTP probes for Kubernetes

Liveness: GET /livez matches GET /health (200, process serving).
Readiness: GET /readyz verifies stats-dir exists, is a directory, and is
readable and writable; if -auth-db resolves outside stats-dir, that
directory is checked too. Failures return 503 Service Unavailable.

Refs: ask task 53
Made-with: Cursor

2 files changed, 203 insertions, 7 deletions

diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go
index 8713797..893e062 100644
--- a/internal/daemon/daemon.go
+++ b/internal/daemon/daemon.go
@@ -8,6 +8,7 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	"codeberg.org/snonux/goprecords/internal/authkeys"
@@ -21,9 +22,11 @@ type Config struct {
 	LogOutput io.Writer
 }
 
-func routes(statsDir string, store *authkeys.Store) http.Handler {
+func routes(statsDir, authDB string, store *authkeys.Store) http.Handler {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/health", health)
+	mux.HandleFunc("/livez", health)
+	mux.HandleFunc("/readyz", readiness(statsDir, authDB))
 	mux.HandleFunc("/report", report(statsDir))
 	mux.Handle("/upload/", uploadHandler(statsDir, store))
 	return mux
@@ -34,7 +37,7 @@ func Handler(statsDir string) http.Handler {
 	if err != nil {
 		panic(err)
 	}
-	return routes(statsDir, store)
+	return routes(statsDir, "", store)
 }
 
 func logWriter(cfg Config) io.Writer {
@@ -97,7 +100,7 @@ func Run(ctx context.Context, cfg Config) error {
 	defer store.Close()
 	srv := &http.Server{
 		Addr:     cfg.Addr,
-		Handler:  withAccessLog(log, routes(cfg.StatsDir, store)),
+		Handler:  withAccessLog(log, routes(cfg.StatsDir, cfg.AuthDB, store)),
 		ErrorLog: slog.NewLogLogger(textHandler, slog.LevelError),
 	}
 	log.Info("daemon_listen", "addr", cfg.Addr)
@@ -134,6 +137,82 @@ func health(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte("ok\n"))
 }
 
+func readiness(statsDir, authDB string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+		if err := checkReadinessDirs(statsDir, authDB); err != nil {
+			http.Error(w, err.Error(), http.StatusServiceUnavailable)
+			return
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok\n"))
+	}
+}
+
+func checkReadinessDirs(statsDir, authDB string) error {
+	absStats, err := filepath.Abs(statsDir)
+	if err != nil {
+		return fmt.Errorf("stats-dir: %w", err)
+	}
+	if err := checkDirReadableWritable(absStats); err != nil {
+		return fmt.Errorf("stats-dir: %w", err)
+	}
+	authPath := authDB
+	if authPath == "" {
+		authPath = authkeys.DefaultPath(statsDir)
+	}
+	absAuth, err := filepath.Abs(authPath)
+	if err != nil {
+		return fmt.Errorf("auth-db: %w", err)
+	}
+	dbDir := filepath.Clean(filepath.Dir(absAuth))
+	if dbDir != filepath.Clean(absStats) {
+		if err := checkDirReadableWritable(dbDir); err != nil {
+			return fmt.Errorf("auth-db dir: %w", err)
+		}
+	}
+	return nil
+}
+
+func checkDirReadableWritable(dir string) error {
+	fi, err := os.Stat(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("missing")
+		}
+		return err
+	}
+	if !fi.IsDir() {
+		return fmt.Errorf("not a directory")
+	}
+	f, err := os.Open(dir)
+	if err != nil {
+		return fmt.Errorf("not readable: %w", err)
+	}
+	_, err = f.Readdirnames(1)
+	_ = f.Close()
+	if err != nil && err != io.EOF {
+		return fmt.Errorf("not readable: %w", err)
+	}
+	tmp, err := os.CreateTemp(dir, ".goprecords-ready-*")
+	if err != nil {
+		return fmt.Errorf("not writable: %w", err)
+	}
+	name := tmp.Name()
+	if err := tmp.Close(); err != nil {
+		_ = os.Remove(name)
+		return fmt.Errorf("not writable: %w", err)
+	}
+	if err := os.Remove(name); err != nil {
+		return fmt.Errorf("not writable: %w", err)
+	}
+	return nil
+}
+
 func report(statsDir string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodGet {
diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go
index 404ff45..e92359b 100644
--- a/internal/daemon/daemon_test.go
+++ b/internal/daemon/daemon_test.go
@@ -45,6 +45,123 @@ func TestHealthMethodNotAllowed(t *testing.T) {
 	}
 }
 
+func TestLivez(t *testing.T) {
+	srv := httptest.NewServer(Handler(t.TempDir()))
+	defer srv.Close()
+	res, err := http.Get(srv.URL + "/livez")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		t.Fatalf("status %d", res.StatusCode)
+	}
+	b, _ := io.ReadAll(res.Body)
+	if string(b) != "ok\n" {
+		t.Fatalf("body %q", b)
+	}
+}
+
+func TestReadyzOK(t *testing.T) {
+	srv := httptest.NewServer(Handler(t.TempDir()))
+	defer srv.Close()
+	res, err := http.Get(srv.URL + "/readyz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		t.Fatalf("status %d", res.StatusCode)
+	}
+	b, _ := io.ReadAll(res.Body)
+	if string(b) != "ok\n" {
+		t.Fatalf("body %q", b)
+	}
+}
+
+func TestReadyzMethodNotAllowed(t *testing.T) {
+	srv := httptest.NewServer(Handler(t.TempDir()))
+	defer srv.Close()
+	req, _ := http.NewRequest(http.MethodPost, srv.URL+"/readyz", nil)
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res.Body.Close()
+	if res.StatusCode != http.StatusMethodNotAllowed {
+		t.Fatalf("status %d", res.StatusCode)
+	}
+}
+
+func TestReadyzMissingStatsDir(t *testing.T) {
+	statsDir := filepath.Join(t.TempDir(), "absent")
+	srv := httptest.NewServer(readiness(statsDir, ""))
+	defer srv.Close()
+	res, err := http.Get(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res.Body.Close()
+	if res.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("status %d want 503", res.StatusCode)
+	}
+}
+
+func TestReadyzStatsDirNotDirectory(t *testing.T) {
+	f := filepath.Join(t.TempDir(), "file")
+	if err := os.WriteFile(f, []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	srv := httptest.NewServer(readiness(f, ""))
+	defer srv.Close()
+	res, err := http.Get(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res.Body.Close()
+	if res.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("status %d want 503", res.StatusCode)
+	}
+}
+
+func TestReadyzStatsDirNotWritable(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.Chmod(dir, 0o555); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Chmod(dir, 0o755) }()
+	srv := httptest.NewServer(readiness(dir, ""))
+	defer srv.Close()
+	res, err := http.Get(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res.Body.Close()
+	if res.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("status %d want 503", res.StatusCode)
+	}
+}
+
+func TestReadyzAuthDBDirNotWritable(t *testing.T) {
+	statsDir := t.TempDir()
+	authDir := t.TempDir()
+	authDB := filepath.Join(authDir, "auth.db")
+	if err := os.Chmod(authDir, 0o555); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Chmod(authDir, 0o755) }()
+	srv := httptest.NewServer(readiness(statsDir, authDB))
+	defer srv.Close()
+	res, err := http.Get(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	res.Body.Close()
+	if res.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("status %d want 503", res.StatusCode)
+	}
+}
+
 func TestReport(t *testing.T) {
 	fixtures := filepath.Join("..", "..", "fixtures")
 	h := Handler(fixtures)
@@ -189,7 +306,7 @@ func TestAccessLogLineToWriter(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer store.Close()
-	srv := httptest.NewServer(withAccessLog(log, routes(statsDir, store)))
+	srv := httptest.NewServer(withAccessLog(log, routes(statsDir, "", store)))
 	defer srv.Close()
 	res, err := http.Get(srv.URL + "/health")
 	if err != nil {
@@ -238,7 +355,7 @@ func TestUploadRequiresBearerWhenKeysExist(t *testing.T) {
 	if _, err := store.CreateKey(ctx, "myhost"); err != nil {
 		t.Fatal(err)
 	}
-	srv := httptest.NewServer(routes(statsDir, store))
+	srv := httptest.NewServer(routes(statsDir, "", store))
 	defer srv.Close()
 	req, _ := http.NewRequest(http.MethodPut, srv.URL+"/upload/myhost/txt", strings.NewReader("x"))
 	res, err := http.DefaultClient.Do(req)
@@ -263,7 +380,7 @@ func TestUploadWithValidBearer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	srv := httptest.NewServer(routes(statsDir, store))
+	srv := httptest.NewServer(routes(statsDir, "", store))
 	defer srv.Close()
 	req, _ := http.NewRequest(http.MethodPut, srv.URL+"/upload/myhost/os.txt", strings.NewReader("os"))
 	req.Header.Set("Authorization", "Bearer "+tok)
@@ -289,7 +406,7 @@ func TestUploadWrongHostForbidden(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	srv := httptest.NewServer(routes(statsDir, store))
+	srv := httptest.NewServer(routes(statsDir, "", store))
 	defer srv.Close()
 	req, _ := http.NewRequest(http.MethodPut, srv.URL+"/upload/other/txt", strings.NewReader("x"))
 	req.Header.Set("Authorization", "Bearer "+tok)