From 07d654f76e1002b6ac18a43aab3c64797dcd2a32 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 13 Mar 2026 12:46:20 +0200 Subject: Harden integration server startup checks --- integrationtests/authkey_test.go | 5 +- integrationtests/commandutils.go | 45 ++++++- integrationtests/dcat_test.go | 147 +++++++++----------- integrationtests/dgrep_literal_info_test.go | 62 ++++----- integrationtests/dgrep_literal_regex_test.go | 73 +++++----- integrationtests/dgrep_test.go | 148 +++++++++------------ integrationtests/dtail_test.go | 6 +- integrationtests/dtailhealth_test.go | 13 +- integrationtests/interactive_runtime_query_test.go | 14 ++ integrationtests/testhelpers.go | 40 +++++- 10 files changed, 301 insertions(+), 252 deletions(-) diff --git a/integrationtests/authkey_test.go b/integrationtests/authkey_test.go index 13f0069..15a10b4 100644 --- a/integrationtests/authkey_test.go +++ b/integrationtests/authkey_test.go @@ -284,7 +284,10 @@ func startAuthKeyServer(t *testing.T, cfgFile string) *authKeyServer { } }() - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, "localhost", port); err != nil { + cancel() + t.Fatalf("Unable to start dserver: %v", err) + } return &authKeyServer{ ctx: ctx, cancel: cancel, diff --git a/integrationtests/commandutils.go b/integrationtests/commandutils.go index fd63b5d..6dfe069 100644 --- a/integrationtests/commandutils.go +++ b/integrationtests/commandutils.go @@ -28,7 +28,7 @@ func runCommand(ctx context.Context, t *testing.T, stdoutFile, cmdStr string, t.Log("Creating stdout file", stdoutFile) fd, err := os.Create(stdoutFile) if err != nil { - return 0, nil + return 0, err } defer fd.Close() @@ -53,6 +53,47 @@ func runCommandRetry(ctx context.Context, t *testing.T, retries int, stdoutFile, return } +func runCommandUntilValid(ctx context.Context, t *testing.T, attempts int, delay time.Duration, + stdoutFile, cmd string, validate func() error, args ...string) error { + + t.Helper() + + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for i := 0; i < attempts; i++ { + exitCode, err := runCommand(ctx, t, stdoutFile, cmd, args...) + if err == nil { + if validateErr := validate(); validateErr == nil { + return nil + } else { + lastErr = validateErr + } + } else { + lastErr = fmt.Errorf("command %s failed with exit code %d: %w", cmd, exitCode, err) + } + + if i == attempts-1 { + break + } + + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + if lastErr != nil { + return lastErr + } + return ctx.Err() + case <-timer.C: + } + } + + return lastErr +} + func startCommand(ctx context.Context, t *testing.T, inPipeFile, cmdStr string, args ...string) (<-chan string, <-chan string, <-chan error, error) { return startCommandWithEnv(ctx, t, inPipeFile, cmdStr, nil, args...) @@ -76,7 +117,7 @@ func startCommandWithEnv(ctx context.Context, t *testing.T, inPipeFile, t.Log(cmdStr, strings.Join(args, " ")) cmd := exec.CommandContext(ctx, cmdStr, args...) - + // Always inherit environment variables cmd.Env = os.Environ() // Add any additional environment variables if provided diff --git a/integrationtests/dcat_test.go b/integrationtests/dcat_test.go index 68cbf9c..bfd6c50 100644 --- a/integrationtests/dcat_test.go +++ b/integrationtests/dcat_test.go @@ -82,27 +82,23 @@ func testDCat1WithServer(t *testing.T, logger *TestLogger, inFile string) error return err } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return err + } - // Run dcat against the server - _, err = runCommand(ctx, t, outFile, - "../dcat", "--plain", "--cfg", "none", + // Run dcat against the server and wait for the full file to be available. + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error { + return compareFilesWithContext(ctx, t, outFile, inFile) + }, + "--plain", "--cfg", "none", "--servers", fmt.Sprintf("%s:%d", bindAddress, port), "--files", inFile, "--trustAllHosts", "--noColor") - if err != nil { - return err - } cancel() - - if err := compareFilesWithContext(ctx, t, outFile, inFile); err != nil { - return err - } - - return nil + return err } func TestDCat1Colors(t *testing.T) { @@ -189,50 +185,42 @@ func testDCat1ColorsWithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) - - // Run without --plain and without --noColor to get colored output - _, err = runCommand(ctx, t, outFile, - "../dcat", "--cfg", "none", - "--servers", fmt.Sprintf("%s:%d", bindAddress, port), - "--files", inFile, - "--trustAllHosts") - if err != nil { + if err := waitForServerReady(ctx, bindAddress, port); err != nil { t.Error(err) return } - cancel() - - // Just verify it ran successfully and produced output - info, err := os.Stat(outFile) - if err != nil { - t.Error("Output file not created:", err) - return - } - if info.Size() == 0 { - t.Error("Output file is empty") - return - } + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error { + info, statErr := os.Stat(outFile) + if statErr != nil { + return fmt.Errorf("output file not created: %w", statErr) + } + if info.Size() == 0 { + return fmt.Errorf("output file is empty") + } - // In server mode, output should contain server metadata unless --plain is used - content, err := os.ReadFile(outFile) - if err != nil { - t.Error("Failed to read output file:", err) - return - } - // In server mode with colors, look for REMOTE or SERVER (without pipe as it may be colored) - if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") { - preview := string(content) - if len(preview) > 500 { - preview = preview[:500] + content, readErr := os.ReadFile(outFile) + if readErr != nil { + return fmt.Errorf("failed to read output file: %w", readErr) } - t.Errorf("Server mode output does not contain server metadata. First 500 chars:\n%s", preview) + if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") { + preview := string(content) + if len(preview) > 500 { + preview = preview[:500] + } + return fmt.Errorf("server mode output does not contain server metadata. First 500 chars:\n%s", preview) + } + return nil + }, + "--cfg", "none", + "--servers", fmt.Sprintf("%s:%d", bindAddress, port), + "--files", inFile, + "--trustAllHosts") + cancel() + if err != nil { + t.Error(err) return } - - // Log verification logger.LogFileComparison(outFile, "server metadata (REMOTE/SERVER)", "contains check") } @@ -309,28 +297,26 @@ func testDCat2WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } // Cat file 100 times in one session. var files []string for i := 0; i < 100; i++ { files = append(files, inFile) } - + args := []string{"--plain", "--logLevel", "error", "--cfg", "none", "--servers", fmt.Sprintf("%s:%d", bindAddress, port), "--trustAllHosts", "--noColor", "--files", strings.Join(files, ",")} - _, err = runCommand(ctx, t, outFile, "../dcat", args...) - if err != nil { - t.Error(err) - return - } - + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error { + return compareFilesContentsWithContext(ctx, t, outFile, expectedFile) + }, args...) cancel() - - if err := compareFilesContentsWithContext(ctx, t, outFile, expectedFile); err != nil { + if err != nil { t.Error(err) return } @@ -403,8 +389,10 @@ func testDCat3WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } args := []string{"--plain", "--logLevel", "error", "--cfg", "none", "--servers", fmt.Sprintf("%s:%d", bindAddress, port), @@ -414,15 +402,11 @@ func testDCat3WithServer(t *testing.T, logger *TestLogger) { // Notice, with DTAIL_INTEGRATION_TEST_RUN_MODE the DTail max line length is set // to 1024! - _, err = runCommand(ctx, t, outFile, "../dcat", args...) - if err != nil { - t.Error(err) - return - } - + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error { + return compareFilesContentsWithContext(ctx, t, outFile, expectedFile) + }, args...) cancel() - - if err := compareFilesContentsWithContext(ctx, t, outFile, expectedFile); err != nil { + if err != nil { t.Error(err) return } @@ -493,24 +477,21 @@ func testDCatColorsWithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } - _, err = runCommand(ctx, t, outFile, - "../dcat", "--logLevel", "error", "--cfg", "none", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error { + return compareFilesWithContext(ctx, t, outFile, expectedFile) + }, + "--logLevel", "error", "--cfg", "none", "--servers", fmt.Sprintf("%s:%d", bindAddress, port), "--files", inFile, "--trustAllHosts", "--noColor") - - if err != nil { - t.Error(err) - return - } - cancel() - - if err := compareFilesWithContext(ctx, t, outFile, expectedFile); err != nil { + if err != nil { t.Error(err) return } diff --git a/integrationtests/dgrep_literal_info_test.go b/integrationtests/dgrep_literal_info_test.go index c007263..cc66c9d 100644 --- a/integrationtests/dgrep_literal_info_test.go +++ b/integrationtests/dgrep_literal_info_test.go @@ -38,10 +38,10 @@ ERROR test line 5 // Test patterns - both literal and regex tests := []struct { - name string - pattern string - expectLiteral bool - expectedCount int + name string + pattern string + expectLiteral bool + expectedCount int }{ { name: "SimpleLiteral", @@ -83,7 +83,7 @@ ERROR test line 5 "", "../dserver", "--cfg", "none", "--logger", "stdout", - "--logLevel", "info", // Changed from error to info + "--logLevel", "info", // Changed from error to info "--bindAddress", bindAddress, "--port", fmt.Sprintf("%d", port), ) @@ -96,7 +96,7 @@ ERROR test line 5 var serverOutput strings.Builder var outputMutex sync.Mutex outputDone := make(chan struct{}) - + go func() { defer close(outputDone) for { @@ -117,15 +117,34 @@ ERROR test line 5 } }() - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } // Run dgrep outFile := fmt.Sprintf("dgrep_info_%s.stdout.tmp", test.name) defer os.Remove(outFile) - _, err = runCommand(ctx, t, outFile, - "../dgrep", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error { + content, readErr := os.ReadFile(outFile) + if readErr != nil { + return fmt.Errorf("failed to read output file: %w", readErr) + } + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + actualCount := 0 + for _, line := range lines { + if line != "" { + actualCount++ + } + } + + if actualCount != test.expectedCount { + return fmt.Errorf("pattern %q: expected %d matches, got %d", test.pattern, test.expectedCount, actualCount) + } + return nil + }, "--plain", "--cfg", "none", "--grep", test.pattern, @@ -144,7 +163,7 @@ ERROR test line 5 // Stop server cancel() - + // Wait for output capture goroutine to finish select { case <-outputDone: @@ -152,25 +171,6 @@ ERROR test line 5 t.Log("Warning: output capture goroutine did not finish in time") } - // Check grep output for correctness - content, err := os.ReadFile(outFile) - if err != nil { - t.Errorf("Failed to read output file: %v", err) - return - } - - lines := strings.Split(strings.TrimSpace(string(content)), "\n") - actualCount := 0 - for _, line := range lines { - if line != "" { - actualCount++ - } - } - - if actualCount != test.expectedCount { - t.Errorf("Pattern '%s': expected %d matches, got %d", test.pattern, test.expectedCount, actualCount) - } - // Check server output for literal mode message outputMutex.Lock() serverLog := serverOutput.String() @@ -188,4 +188,4 @@ ERROR test line 5 } }) } -} \ No newline at end of file +} diff --git a/integrationtests/dgrep_literal_regex_test.go b/integrationtests/dgrep_literal_regex_test.go index d078899..1f06267 100644 --- a/integrationtests/dgrep_literal_regex_test.go +++ b/integrationtests/dgrep_literal_regex_test.go @@ -259,19 +259,19 @@ ERROR: Network down isRegex bool expectedCount int }{ - {"ERROR", false, 3}, // Literal - {"ERROR.*full", true, 1}, // Regex - {"WARNING", false, 1}, // Literal - {"(ERROR|WARNING)", true, 4}, // Regex - {"Process started", false, 1}, // Literal with space - {"^INFO:", true, 1}, // Regex with anchor + {"ERROR", false, 3}, // Literal + {"ERROR.*full", true, 1}, // Regex + {"WARNING", false, 1}, // Literal + {"(ERROR|WARNING)", true, 4}, // Regex + {"Process started", false, 1}, // Literal with space + {"^INFO:", true, 1}, // Regex with anchor } t.Run("ServerlessMode", func(t *testing.T) { for i, p := range patterns { outFile := fmt.Sprintf("mixed_%d.stdout.tmp", i) defer os.Remove(outFile) - + testLiteralPatternServerless(t, testLogger, testFile, outFile, p.pattern, p.expectedCount) } }) @@ -339,11 +339,33 @@ func testLiteralPatternWithServer(t *testing.T, logger *TestLogger, inFile, outF return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } - _, err = runCommand(ctx, t, outFile, - "../dgrep", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error { + content, readErr := os.ReadFile(outFile) + if readErr != nil { + return fmt.Errorf("failed to read output file: %w", readErr) + } + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + actualCount := 0 + for _, line := range lines { + if line != "" { + actualCount++ + } + } + + if actualCount != expectedCount { + if actualCount > 0 && actualCount <= 10 { + return fmt.Errorf("pattern %q: expected %d matches, got %d\noutput:\n%s", pattern, expectedCount, actualCount, string(content)) + } + return fmt.Errorf("pattern %q: expected %d matches, got %d", pattern, expectedCount, actualCount) + } + return nil + }, "--plain", "--cfg", "none", "--grep", pattern, @@ -351,34 +373,9 @@ func testLiteralPatternWithServer(t *testing.T, logger *TestLogger, inFile, outF "--trustAllHosts", "--noColor", "--files", inFile) - - if err != nil { - t.Errorf("Failed to run dgrep with pattern '%s': %v", pattern, err) - return - } - cancel() - - // Count matching lines - content, err := os.ReadFile(outFile) if err != nil { - t.Errorf("Failed to read output file: %v", err) - return - } - - lines := strings.Split(strings.TrimSpace(string(content)), "\n") - actualCount := 0 - for _, line := range lines { - if line != "" { - actualCount++ - } - } - - if actualCount != expectedCount { - t.Errorf("Pattern '%s': expected %d matches, got %d", pattern, expectedCount, actualCount) - if actualCount > 0 && actualCount <= 10 { - t.Errorf("Output:\n%s", string(content)) - } + t.Error(err) } } @@ -392,4 +389,4 @@ func testRegexPatternWithServer(t *testing.T, logger *TestLogger, inFile, outFil // Same implementation as testLiteralPatternWithServer // The regex vs literal detection happens internally testLiteralPatternWithServer(t, logger, inFile, outFile, pattern, expectedCount) -} \ No newline at end of file +} diff --git a/integrationtests/dgrep_test.go b/integrationtests/dgrep_test.go index b8728e1..e8a7cb3 100644 --- a/integrationtests/dgrep_test.go +++ b/integrationtests/dgrep_test.go @@ -81,11 +81,14 @@ func testDGrep1WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } - _, err = runCommand(ctx, t, outFile, - "../dgrep", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error { + return compareFilesWithContext(ctx, t, outFile, expectedOutFile) + }, "--plain", "--cfg", "none", "--grep", "1002-071947", @@ -93,15 +96,8 @@ func testDGrep1WithServer(t *testing.T, logger *TestLogger) { "--trustAllHosts", "--noColor", "--files", inFile) - - if err != nil { - t.Error(err) - return - } - cancel() - - if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil { + if err != nil { t.Error(err) return } @@ -195,52 +191,42 @@ func testDGrep1ColorsWithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } - // Run without --plain and without --noColor to get colored output - _, err = runCommand(ctx, t, outFile, - "../dgrep", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error { + info, statErr := os.Stat(outFile) + if statErr != nil { + return fmt.Errorf("output file not created: %w", statErr) + } + if info.Size() == 0 { + return fmt.Errorf("output file is empty") + } + content, readErr := os.ReadFile(outFile) + if readErr != nil { + return fmt.Errorf("failed to read output file: %w", readErr) + } + if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") { + preview := string(content) + if len(preview) > 500 { + preview = preview[:500] + } + return fmt.Errorf("server mode output does not contain server metadata. First 500 chars:\n%s", preview) + } + return nil + }, "--cfg", "none", "--grep", "1002-071947", "--servers", fmt.Sprintf("%s:%d", bindAddress, port), "--trustAllHosts", "--files", inFile) - - if err != nil { - t.Error(err) - return - } - cancel() - - // Verify it ran successfully and produced output - info, err := os.Stat(outFile) - if err != nil { - t.Error("Output file not created:", err) - return - } - if info.Size() == 0 { - t.Error("Output file is empty") - return - } - - // In server mode, output should contain server metadata - content, err := os.ReadFile(outFile) if err != nil { - t.Error("Failed to read output file:", err) - return - } - if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") { - preview := string(content) - if len(preview) > 500 { - preview = preview[:500] - } - t.Errorf("Server mode output does not contain server metadata. First 500 chars:\n%s", preview) + t.Error(err) return } - - // Log verification logger.LogFileComparison(outFile, "server metadata (REMOTE/SERVER)", "contains check") } @@ -314,11 +300,14 @@ func testDGrep2WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } - _, err = runCommand(ctx, t, outFile, - "../dgrep", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error { + return compareFilesWithContext(ctx, t, outFile, expectedOutFile) + }, "--plain", "--cfg", "none", "--grep", "1002-07194[789]", @@ -326,15 +315,8 @@ func testDGrep2WithServer(t *testing.T, logger *TestLogger) { "--trustAllHosts", "--noColor", "--files", inFile) - - if err != nil { - t.Error(err) - return - } - cancel() - - if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil { + if err != nil { t.Error(err) return } @@ -412,11 +394,14 @@ func testDGrepContext1WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } - _, err = runCommand(ctx, t, outFile, - "../dgrep", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error { + return compareFilesWithContext(ctx, t, outFile, expectedOutFile) + }, "--plain", "--cfg", "none", "--grep", "1002-071947", @@ -426,15 +411,8 @@ func testDGrepContext1WithServer(t *testing.T, logger *TestLogger) { "--trustAllHosts", "--noColor", "--files", inFile) - - if err != nil { - t.Error(err) - return - } - cancel() - - if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil { + if err != nil { t.Error(err) return } @@ -513,11 +491,14 @@ func testDGrepContext2WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } - _, err = runCommand(ctx, t, outFile, - "../dgrep", + err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error { + return compareFilesWithContext(ctx, t, outFile, expectedOutFile) + }, "--plain", "--cfg", "none", "--grep", "1002-071947", @@ -528,15 +509,8 @@ func testDGrepContext2WithServer(t *testing.T, logger *TestLogger) { "--trustAllHosts", "--noColor", "--files", inFile) - - if err != nil { - t.Error(err) - return - } - cancel() - - if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil { + if err != nil { t.Error(err) return } @@ -560,7 +534,7 @@ func TestDGrepPipeToStdin(t *testing.T) { func testDGrepStdinServerless(t *testing.T, logger *TestLogger) { inFile := "mapr_testdata.log" - outFile := "dgrepstdin.stdout.tmp" + outFile := "dgrepstdin.stdout.tmp" expectedOutFile := "dgrep1.txt.expected" ctx := WithTestLogger(context.Background(), logger) @@ -571,7 +545,7 @@ func testDGrepStdinServerless(t *testing.T, logger *TestLogger) { "--plain", "--cfg", "none", "--grep", "1002-071947") - + if err != nil { t.Error(err) return @@ -621,4 +595,4 @@ func testDGrepStdinServerless(t *testing.T, logger *TestLogger) { t.Error(err) return } -} \ No newline at end of file +} diff --git a/integrationtests/dtail_test.go b/integrationtests/dtail_test.go index cfa96c8..1db96ee 100644 --- a/integrationtests/dtail_test.go +++ b/integrationtests/dtail_test.go @@ -197,8 +197,10 @@ func testDTailColorTableWithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } _, err = runCommand(ctx, t, outFile, "../dtail", "--colorTable", diff --git a/integrationtests/dtailhealth_test.go b/integrationtests/dtailhealth_test.go index 74773f2..89e2996 100644 --- a/integrationtests/dtailhealth_test.go +++ b/integrationtests/dtailhealth_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "testing" - "time" "github.com/mimecast/dtail/internal/config" ) @@ -73,8 +72,10 @@ func testDTailHealth1WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } t.Log("Server mode check without --server flag, is supposed to exit with warning state.") // Run dtailhealth without specifying --server flag @@ -157,8 +158,10 @@ func testDTailHealth2WithServer(t *testing.T, logger *TestLogger) { return } - // Give server time to start - time.Sleep(500 * time.Millisecond) + if err := waitForServerReady(ctx, bindAddress, port); err != nil { + t.Error(err) + return + } t.Log("Server mode negative test, checking unreachable server, is supposed to exit with a critical state.") // Check an unreachable server (not the one we started) diff --git a/integrationtests/interactive_runtime_query_test.go b/integrationtests/interactive_runtime_query_test.go index 213c877..48e2301 100644 --- a/integrationtests/interactive_runtime_query_test.go +++ b/integrationtests/interactive_runtime_query_test.go @@ -51,6 +51,10 @@ func TestDTailInteractiveReloadReusesSessionAndDropsLateOldMatches(t *testing.T) t.Fatalf("start dserver: %v", err) } serverLogs := startProcessOutputCollector(ctx, serverStdout, serverStderr) + if err := waitForServerReady(ctx, "localhost", port); err != nil { + t.Fatalf("wait for dserver: %v", err) + } + serverLogs.reset() writerDone := make(chan error, 1) go func() { @@ -130,6 +134,10 @@ func TestDGrepInteractiveReloadReusesSessionAfterCompletedRead(t *testing.T) { t.Fatalf("start dserver: %v", err) } serverLogs := startProcessOutputCollector(ctx, serverStdout, serverStderr) + if err := waitForServerReady(ctx, "localhost", port); err != nil { + t.Fatalf("wait for dserver: %v", err) + } + serverLogs.reset() clientOutput, err := runInteractivePTYCommand(ctx, []string{ "../dgrep", @@ -199,6 +207,12 @@ func (c *processOutputCollector) snapshot() []string { return out } +func (c *processOutputCollector) reset() { + c.mu.Lock() + defer c.mu.Unlock() + c.lines = c.lines[:0] +} + func appendLinesOnSchedule(ctx context.Context, path string, steps []interactiveStep) error { fd, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0600) if err != nil { diff --git a/integrationtests/testhelpers.go b/integrationtests/testhelpers.go index da0ecd8..ca9f7a7 100644 --- a/integrationtests/testhelpers.go +++ b/integrationtests/testhelpers.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" "strings" "sync" @@ -157,9 +158,42 @@ func startTestServer(ctx context.Context, t *testing.T, cfg *ServerConfig) error return err } - // Give server time to start - time.Sleep(500 * time.Millisecond) - return nil + return waitForServerReady(ctx, cfg.BindAddress, cfg.Port) +} + +func waitForServerReady(ctx context.Context, bindAddress string, port int) error { + address := fmt.Sprintf("%s:%d", bindAddress, port) + deadline := time.Now().Add(10 * time.Second) + var lastErr error + var lastOutput string + + for { + cmd := exec.CommandContext(ctx, "../dtailhealth", "--server", address, "--no-auth-key") + out, err := cmd.CombinedOutput() + if err == nil { + return nil + } + lastErr = err + lastOutput = strings.TrimSpace(string(out)) + + if ctx.Err() != nil { + return fmt.Errorf("wait for dserver %s: %w", address, ctx.Err()) + } + if time.Now().After(deadline) { + if lastOutput != "" { + return fmt.Errorf("timed out waiting for dserver %s: %w (%s)", address, lastErr, lastOutput) + } + return fmt.Errorf("timed out waiting for dserver %s: %w", address, lastErr) + } + + timer := time.NewTimer(50 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("wait for dserver %s: %w", address, ctx.Err()) + case <-timer.C: + } + } } // createTestContext creates a context with cancel that will be cleaned up automatically -- cgit v1.2.3