Harden integration server startup checks

10 files changed, 301 insertions, 252 deletions

diff --git a/integrationtests/authkey_test.go b/integrationtests/authkey_test.go
index 13f0069..15a10b4 100644
--- a/integrationtests/authkey_test.go
+++ b/integrationtests/authkey_test.go
@@ -284,7 +284,10 @@ func startAuthKeyServer(t *testing.T, cfgFile string) *authKeyServer {
 		}
 	}()
 
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, "localhost", port); err != nil {
+		cancel()
+		t.Fatalf("Unable to start dserver: %v", err)
+	}
 	return &authKeyServer{
 		ctx:    ctx,
 		cancel: cancel,
diff --git a/integrationtests/commandutils.go b/integrationtests/commandutils.go
index fd63b5d..6dfe069 100644
--- a/integrationtests/commandutils.go
+++ b/integrationtests/commandutils.go
@@ -28,7 +28,7 @@ func runCommand(ctx context.Context, t *testing.T, stdoutFile, cmdStr string,
 	t.Log("Creating stdout file", stdoutFile)
 	fd, err := os.Create(stdoutFile)
 	if err != nil {
-		return 0, nil
+		return 0, err
 	}
 	defer fd.Close()
 
@@ -53,6 +53,47 @@ func runCommandRetry(ctx context.Context, t *testing.T, retries int, stdoutFile,
 	return
 }
 
+func runCommandUntilValid(ctx context.Context, t *testing.T, attempts int, delay time.Duration,
+	stdoutFile, cmd string, validate func() error, args ...string) error {
+
+	t.Helper()
+
+	if attempts < 1 {
+		attempts = 1
+	}
+
+	var lastErr error
+	for i := 0; i < attempts; i++ {
+		exitCode, err := runCommand(ctx, t, stdoutFile, cmd, args...)
+		if err == nil {
+			if validateErr := validate(); validateErr == nil {
+				return nil
+			} else {
+				lastErr = validateErr
+			}
+		} else {
+			lastErr = fmt.Errorf("command %s failed with exit code %d: %w", cmd, exitCode, err)
+		}
+
+		if i == attempts-1 {
+			break
+		}
+
+		timer := time.NewTimer(delay)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			if lastErr != nil {
+				return lastErr
+			}
+			return ctx.Err()
+		case <-timer.C:
+		}
+	}
+
+	return lastErr
+}
+
 func startCommand(ctx context.Context, t *testing.T, inPipeFile,
 	cmdStr string, args ...string) (<-chan string, <-chan string, <-chan error, error) {
 	return startCommandWithEnv(ctx, t, inPipeFile, cmdStr, nil, args...)
@@ -76,7 +117,7 @@ func startCommandWithEnv(ctx context.Context, t *testing.T, inPipeFile,
 
 	t.Log(cmdStr, strings.Join(args, " "))
 	cmd := exec.CommandContext(ctx, cmdStr, args...)
-	
+
 	// Always inherit environment variables
 	cmd.Env = os.Environ()
 	// Add any additional environment variables if provided
diff --git a/integrationtests/dcat_test.go b/integrationtests/dcat_test.go
index 68cbf9c..bfd6c50 100644
--- a/integrationtests/dcat_test.go
+++ b/integrationtests/dcat_test.go
@@ -82,27 +82,23 @@ func testDCat1WithServer(t *testing.T, logger *TestLogger, inFile string) error
 		return err
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return err
+	}
 
-	// Run dcat against the server
-	_, err = runCommand(ctx, t, outFile,
-		"../dcat", "--plain", "--cfg", "none",
+	// Run dcat against the server and wait for the full file to be available.
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error {
+		return compareFilesWithContext(ctx, t, outFile, inFile)
+	},
+		"--plain", "--cfg", "none",
 		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
 		"--files", inFile,
 		"--trustAllHosts",
 		"--noColor")
-	if err != nil {
-		return err
-	}
 
 	cancel()
-
-	if err := compareFilesWithContext(ctx, t, outFile, inFile); err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 func TestDCat1Colors(t *testing.T) {
@@ -189,50 +185,42 @@ func testDCat1ColorsWithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
-
-	// Run without --plain and without --noColor to get colored output
-	_, err = runCommand(ctx, t, outFile,
-		"../dcat", "--cfg", "none",
-		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
-		"--files", inFile,
-		"--trustAllHosts")
-	if err != nil {
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
 		t.Error(err)
 		return
 	}
 
-	cancel()
-
-	// Just verify it ran successfully and produced output
-	info, err := os.Stat(outFile)
-	if err != nil {
-		t.Error("Output file not created:", err)
-		return
-	}
-	if info.Size() == 0 {
-		t.Error("Output file is empty")
-		return
-	}
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error {
+		info, statErr := os.Stat(outFile)
+		if statErr != nil {
+			return fmt.Errorf("output file not created: %w", statErr)
+		}
+		if info.Size() == 0 {
+			return fmt.Errorf("output file is empty")
+		}
 
-	// In server mode, output should contain server metadata unless --plain is used
-	content, err := os.ReadFile(outFile)
-	if err != nil {
-		t.Error("Failed to read output file:", err)
-		return
-	}
-	// In server mode with colors, look for REMOTE or SERVER (without pipe as it may be colored)
-	if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") {
-		preview := string(content)
-		if len(preview) > 500 {
-			preview = preview[:500]
+		content, readErr := os.ReadFile(outFile)
+		if readErr != nil {
+			return fmt.Errorf("failed to read output file: %w", readErr)
 		}
-		t.Errorf("Server mode output does not contain server metadata. First 500 chars:\n%s", preview)
+		if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") {
+			preview := string(content)
+			if len(preview) > 500 {
+				preview = preview[:500]
+			}
+			return fmt.Errorf("server mode output does not contain server metadata. First 500 chars:\n%s", preview)
+		}
+		return nil
+	},
+		"--cfg", "none",
+		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
+		"--files", inFile,
+		"--trustAllHosts")
+	cancel()
+	if err != nil {
+		t.Error(err)
 		return
 	}
-
-	// Log verification
 	logger.LogFileComparison(outFile, "server metadata (REMOTE/SERVER)", "contains check")
 }
 
@@ -309,28 +297,26 @@ func testDCat2WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
 	// Cat file 100 times in one session.
 	var files []string
 	for i := 0; i < 100; i++ {
 		files = append(files, inFile)
 	}
-	
+
 	args := []string{"--plain", "--logLevel", "error", "--cfg", "none",
 		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
 		"--trustAllHosts", "--noColor", "--files", strings.Join(files, ",")}
 
-	_, err = runCommand(ctx, t, outFile, "../dcat", args...)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error {
+		return compareFilesContentsWithContext(ctx, t, outFile, expectedFile)
+	}, args...)
 	cancel()
-
-	if err := compareFilesContentsWithContext(ctx, t, outFile, expectedFile); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
@@ -403,8 +389,10 @@ func testDCat3WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
 	args := []string{"--plain", "--logLevel", "error", "--cfg", "none",
 		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
@@ -414,15 +402,11 @@ func testDCat3WithServer(t *testing.T, logger *TestLogger) {
 
 	// Notice, with DTAIL_INTEGRATION_TEST_RUN_MODE the DTail max line length is set
 	// to 1024!
-	_, err = runCommand(ctx, t, outFile, "../dcat", args...)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error {
+		return compareFilesContentsWithContext(ctx, t, outFile, expectedFile)
+	}, args...)
 	cancel()
-
-	if err := compareFilesContentsWithContext(ctx, t, outFile, expectedFile); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
@@ -493,24 +477,21 @@ func testDCatColorsWithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
-	_, err = runCommand(ctx, t, outFile,
-		"../dcat", "--logLevel", "error", "--cfg", "none",
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dcat", func() error {
+		return compareFilesWithContext(ctx, t, outFile, expectedFile)
+	},
+		"--logLevel", "error", "--cfg", "none",
 		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
 		"--files", inFile,
 		"--trustAllHosts",
 		"--noColor")
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
 	cancel()
-
-	if err := compareFilesWithContext(ctx, t, outFile, expectedFile); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
diff --git a/integrationtests/dgrep_literal_info_test.go b/integrationtests/dgrep_literal_info_test.go
index c007263..cc66c9d 100644
--- a/integrationtests/dgrep_literal_info_test.go
+++ b/integrationtests/dgrep_literal_info_test.go
@@ -38,10 +38,10 @@ ERROR test line 5
 
 	// Test patterns - both literal and regex
 	tests := []struct {
-		name           string
-		pattern        string
-		expectLiteral  bool
-		expectedCount  int
+		name          string
+		pattern       string
+		expectLiteral bool
+		expectedCount int
 	}{
 		{
 			name:          "SimpleLiteral",
@@ -83,7 +83,7 @@ ERROR test line 5
 				"", "../dserver",
 				"--cfg", "none",
 				"--logger", "stdout",
-				"--logLevel", "info",  // Changed from error to info
+				"--logLevel", "info", // Changed from error to info
 				"--bindAddress", bindAddress,
 				"--port", fmt.Sprintf("%d", port),
 			)
@@ -96,7 +96,7 @@ ERROR test line 5
 			var serverOutput strings.Builder
 			var outputMutex sync.Mutex
 			outputDone := make(chan struct{})
-			
+
 			go func() {
 				defer close(outputDone)
 				for {
@@ -117,15 +117,34 @@ ERROR test line 5
 				}
 			}()
 
-			// Give server time to start
-			time.Sleep(500 * time.Millisecond)
+			if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+				t.Error(err)
+				return
+			}
 
 			// Run dgrep
 			outFile := fmt.Sprintf("dgrep_info_%s.stdout.tmp", test.name)
 			defer os.Remove(outFile)
 
-			_, err = runCommand(ctx, t, outFile,
-				"../dgrep",
+			err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error {
+				content, readErr := os.ReadFile(outFile)
+				if readErr != nil {
+					return fmt.Errorf("failed to read output file: %w", readErr)
+				}
+
+				lines := strings.Split(strings.TrimSpace(string(content)), "\n")
+				actualCount := 0
+				for _, line := range lines {
+					if line != "" {
+						actualCount++
+					}
+				}
+
+				if actualCount != test.expectedCount {
+					return fmt.Errorf("pattern %q: expected %d matches, got %d", test.pattern, test.expectedCount, actualCount)
+				}
+				return nil
+			},
 				"--plain",
 				"--cfg", "none",
 				"--grep", test.pattern,
@@ -144,7 +163,7 @@ ERROR test line 5
 
 			// Stop server
 			cancel()
-			
+
 			// Wait for output capture goroutine to finish
 			select {
 			case <-outputDone:
@@ -152,25 +171,6 @@ ERROR test line 5
 				t.Log("Warning: output capture goroutine did not finish in time")
 			}
 
-			// Check grep output for correctness
-			content, err := os.ReadFile(outFile)
-			if err != nil {
-				t.Errorf("Failed to read output file: %v", err)
-				return
-			}
-
-			lines := strings.Split(strings.TrimSpace(string(content)), "\n")
-			actualCount := 0
-			for _, line := range lines {
-				if line != "" {
-					actualCount++
-				}
-			}
-
-			if actualCount != test.expectedCount {
-				t.Errorf("Pattern '%s': expected %d matches, got %d", test.pattern, test.expectedCount, actualCount)
-			}
-
 			// Check server output for literal mode message
 			outputMutex.Lock()
 			serverLog := serverOutput.String()
@@ -188,4 +188,4 @@ ERROR test line 5
 			}
 		})
 	}
-}
\ No newline at end of file
+}
diff --git a/integrationtests/dgrep_literal_regex_test.go b/integrationtests/dgrep_literal_regex_test.go
index d078899..1f06267 100644
--- a/integrationtests/dgrep_literal_regex_test.go
+++ b/integrationtests/dgrep_literal_regex_test.go
@@ -259,19 +259,19 @@ ERROR: Network down
 		isRegex       bool
 		expectedCount int
 	}{
-		{"ERROR", false, 3},            // Literal
-		{"ERROR.*full", true, 1},       // Regex
-		{"WARNING", false, 1},          // Literal
-		{"(ERROR|WARNING)", true, 4},   // Regex
-		{"Process started", false, 1},  // Literal with space
-		{"^INFO:", true, 1},            // Regex with anchor
+		{"ERROR", false, 3},           // Literal
+		{"ERROR.*full", true, 1},      // Regex
+		{"WARNING", false, 1},         // Literal
+		{"(ERROR|WARNING)", true, 4},  // Regex
+		{"Process started", false, 1}, // Literal with space
+		{"^INFO:", true, 1},           // Regex with anchor
 	}
 
 	t.Run("ServerlessMode", func(t *testing.T) {
 		for i, p := range patterns {
 			outFile := fmt.Sprintf("mixed_%d.stdout.tmp", i)
 			defer os.Remove(outFile)
-			
+
 			testLiteralPatternServerless(t, testLogger, testFile, outFile, p.pattern, p.expectedCount)
 		}
 	})
@@ -339,11 +339,33 @@ func testLiteralPatternWithServer(t *testing.T, logger *TestLogger, inFile, outF
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
-	_, err = runCommand(ctx, t, outFile,
-		"../dgrep",
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error {
+		content, readErr := os.ReadFile(outFile)
+		if readErr != nil {
+			return fmt.Errorf("failed to read output file: %w", readErr)
+		}
+
+		lines := strings.Split(strings.TrimSpace(string(content)), "\n")
+		actualCount := 0
+		for _, line := range lines {
+			if line != "" {
+				actualCount++
+			}
+		}
+
+		if actualCount != expectedCount {
+			if actualCount > 0 && actualCount <= 10 {
+				return fmt.Errorf("pattern %q: expected %d matches, got %d\noutput:\n%s", pattern, expectedCount, actualCount, string(content))
+			}
+			return fmt.Errorf("pattern %q: expected %d matches, got %d", pattern, expectedCount, actualCount)
+		}
+		return nil
+	},
 		"--plain",
 		"--cfg", "none",
 		"--grep", pattern,
@@ -351,34 +373,9 @@ func testLiteralPatternWithServer(t *testing.T, logger *TestLogger, inFile, outF
 		"--trustAllHosts",
 		"--noColor",
 		"--files", inFile)
-
-	if err != nil {
-		t.Errorf("Failed to run dgrep with pattern '%s': %v", pattern, err)
-		return
-	}
-
 	cancel()
-
-	// Count matching lines
-	content, err := os.ReadFile(outFile)
 	if err != nil {
-		t.Errorf("Failed to read output file: %v", err)
-		return
-	}
-
-	lines := strings.Split(strings.TrimSpace(string(content)), "\n")
-	actualCount := 0
-	for _, line := range lines {
-		if line != "" {
-			actualCount++
-		}
-	}
-
-	if actualCount != expectedCount {
-		t.Errorf("Pattern '%s': expected %d matches, got %d", pattern, expectedCount, actualCount)
-		if actualCount > 0 && actualCount <= 10 {
-			t.Errorf("Output:\n%s", string(content))
-		}
+		t.Error(err)
 	}
 }
 
@@ -392,4 +389,4 @@ func testRegexPatternWithServer(t *testing.T, logger *TestLogger, inFile, outFil
 	// Same implementation as testLiteralPatternWithServer
 	// The regex vs literal detection happens internally
 	testLiteralPatternWithServer(t, logger, inFile, outFile, pattern, expectedCount)
-}
\ No newline at end of file
+}
diff --git a/integrationtests/dgrep_test.go b/integrationtests/dgrep_test.go
index b8728e1..e8a7cb3 100644
--- a/integrationtests/dgrep_test.go
+++ b/integrationtests/dgrep_test.go
@@ -81,11 +81,14 @@ func testDGrep1WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
-	_, err = runCommand(ctx, t, outFile,
-		"../dgrep",
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error {
+		return compareFilesWithContext(ctx, t, outFile, expectedOutFile)
+	},
 		"--plain",
 		"--cfg", "none",
 		"--grep", "1002-071947",
@@ -93,15 +96,8 @@ func testDGrep1WithServer(t *testing.T, logger *TestLogger) {
 		"--trustAllHosts",
 		"--noColor",
 		"--files", inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
 	cancel()
-
-	if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
@@ -195,52 +191,42 @@ func testDGrep1ColorsWithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
-	// Run without --plain and without --noColor to get colored output
-	_, err = runCommand(ctx, t, outFile,
-		"../dgrep",
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error {
+		info, statErr := os.Stat(outFile)
+		if statErr != nil {
+			return fmt.Errorf("output file not created: %w", statErr)
+		}
+		if info.Size() == 0 {
+			return fmt.Errorf("output file is empty")
+		}
+		content, readErr := os.ReadFile(outFile)
+		if readErr != nil {
+			return fmt.Errorf("failed to read output file: %w", readErr)
+		}
+		if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") {
+			preview := string(content)
+			if len(preview) > 500 {
+				preview = preview[:500]
+			}
+			return fmt.Errorf("server mode output does not contain server metadata. First 500 chars:\n%s", preview)
+		}
+		return nil
+	},
 		"--cfg", "none",
 		"--grep", "1002-071947",
 		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
 		"--trustAllHosts",
 		"--files", inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
 	cancel()
-
-	// Verify it ran successfully and produced output
-	info, err := os.Stat(outFile)
-	if err != nil {
-		t.Error("Output file not created:", err)
-		return
-	}
-	if info.Size() == 0 {
-		t.Error("Output file is empty")
-		return
-	}
-
-	// In server mode, output should contain server metadata
-	content, err := os.ReadFile(outFile)
 	if err != nil {
-		t.Error("Failed to read output file:", err)
-		return
-	}
-	if !strings.Contains(string(content), "REMOTE") && !strings.Contains(string(content), "SERVER") {
-		preview := string(content)
-		if len(preview) > 500 {
-			preview = preview[:500]
-		}
-		t.Errorf("Server mode output does not contain server metadata. First 500 chars:\n%s", preview)
+		t.Error(err)
 		return
 	}
-
-	// Log verification
 	logger.LogFileComparison(outFile, "server metadata (REMOTE/SERVER)", "contains check")
 }
 
@@ -314,11 +300,14 @@ func testDGrep2WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
-	_, err = runCommand(ctx, t, outFile,
-		"../dgrep",
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error {
+		return compareFilesWithContext(ctx, t, outFile, expectedOutFile)
+	},
 		"--plain",
 		"--cfg", "none",
 		"--grep", "1002-07194[789]",
@@ -326,15 +315,8 @@ func testDGrep2WithServer(t *testing.T, logger *TestLogger) {
 		"--trustAllHosts",
 		"--noColor",
 		"--files", inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
 	cancel()
-
-	if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
@@ -412,11 +394,14 @@ func testDGrepContext1WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
-	_, err = runCommand(ctx, t, outFile,
-		"../dgrep",
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error {
+		return compareFilesWithContext(ctx, t, outFile, expectedOutFile)
+	},
 		"--plain",
 		"--cfg", "none",
 		"--grep", "1002-071947",
@@ -426,15 +411,8 @@ func testDGrepContext1WithServer(t *testing.T, logger *TestLogger) {
 		"--trustAllHosts",
 		"--noColor",
 		"--files", inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
 	cancel()
-
-	if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
@@ -513,11 +491,14 @@ func testDGrepContext2WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
-	_, err = runCommand(ctx, t, outFile,
-		"../dgrep",
+	err = runCommandUntilValid(ctx, t, 5, 200*time.Millisecond, outFile, "../dgrep", func() error {
+		return compareFilesWithContext(ctx, t, outFile, expectedOutFile)
+	},
 		"--plain",
 		"--cfg", "none",
 		"--grep", "1002-071947",
@@ -528,15 +509,8 @@ func testDGrepContext2WithServer(t *testing.T, logger *TestLogger) {
 		"--trustAllHosts",
 		"--noColor",
 		"--files", inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
 	cancel()
-
-	if err := compareFilesWithContext(ctx, t, outFile, expectedOutFile); err != nil {
+	if err != nil {
 		t.Error(err)
 		return
 	}
@@ -560,7 +534,7 @@ func TestDGrepPipeToStdin(t *testing.T) {
 
 func testDGrepStdinServerless(t *testing.T, logger *TestLogger) {
 	inFile := "mapr_testdata.log"
-	outFile := "dgrepstdin.stdout.tmp" 
+	outFile := "dgrepstdin.stdout.tmp"
 	expectedOutFile := "dgrep1.txt.expected"
 	ctx := WithTestLogger(context.Background(), logger)
 
@@ -571,7 +545,7 @@ func testDGrepStdinServerless(t *testing.T, logger *TestLogger) {
 		"--plain",
 		"--cfg", "none",
 		"--grep", "1002-071947")
-	
+
 	if err != nil {
 		t.Error(err)
 		return
@@ -621,4 +595,4 @@ func testDGrepStdinServerless(t *testing.T, logger *TestLogger) {
 		t.Error(err)
 		return
 	}
-}
\ No newline at end of file
+}
diff --git a/integrationtests/dtail_test.go b/integrationtests/dtail_test.go
index cfa96c8..1db96ee 100644
--- a/integrationtests/dtail_test.go
+++ b/integrationtests/dtail_test.go
@@ -197,8 +197,10 @@ func testDTailColorTableWithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
 	_, err = runCommand(ctx, t, outFile, "../dtail",
 		"--colorTable",
diff --git a/integrationtests/dtailhealth_test.go b/integrationtests/dtailhealth_test.go
index 74773f2..89e2996 100644
--- a/integrationtests/dtailhealth_test.go
+++ b/integrationtests/dtailhealth_test.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/mimecast/dtail/internal/config"
 )
@@ -73,8 +72,10 @@ func testDTailHealth1WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
 	t.Log("Server mode check without --server flag, is supposed to exit with warning state.")
 	// Run dtailhealth without specifying --server flag
@@ -157,8 +158,10 @@ func testDTailHealth2WithServer(t *testing.T, logger *TestLogger) {
 		return
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
+	if err := waitForServerReady(ctx, bindAddress, port); err != nil {
+		t.Error(err)
+		return
+	}
 
 	t.Log("Server mode negative test, checking unreachable server, is supposed to exit with a critical state.")
 	// Check an unreachable server (not the one we started)
diff --git a/integrationtests/interactive_runtime_query_test.go b/integrationtests/interactive_runtime_query_test.go
index 213c877..48e2301 100644
--- a/integrationtests/interactive_runtime_query_test.go
+++ b/integrationtests/interactive_runtime_query_test.go
@@ -51,6 +51,10 @@ func TestDTailInteractiveReloadReusesSessionAndDropsLateOldMatches(t *testing.T)
 		t.Fatalf("start dserver: %v", err)
 	}
 	serverLogs := startProcessOutputCollector(ctx, serverStdout, serverStderr)
+	if err := waitForServerReady(ctx, "localhost", port); err != nil {
+		t.Fatalf("wait for dserver: %v", err)
+	}
+	serverLogs.reset()
 
 	writerDone := make(chan error, 1)
 	go func() {
@@ -130,6 +134,10 @@ func TestDGrepInteractiveReloadReusesSessionAfterCompletedRead(t *testing.T) {
 		t.Fatalf("start dserver: %v", err)
 	}
 	serverLogs := startProcessOutputCollector(ctx, serverStdout, serverStderr)
+	if err := waitForServerReady(ctx, "localhost", port); err != nil {
+		t.Fatalf("wait for dserver: %v", err)
+	}
+	serverLogs.reset()
 
 	clientOutput, err := runInteractivePTYCommand(ctx, []string{
 		"../dgrep",
@@ -199,6 +207,12 @@ func (c *processOutputCollector) snapshot() []string {
 	return out
 }
 
+func (c *processOutputCollector) reset() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.lines = c.lines[:0]
+}
+
 func appendLinesOnSchedule(ctx context.Context, path string, steps []interactiveStep) error {
 	fd, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0600)
 	if err != nil {
diff --git a/integrationtests/testhelpers.go b/integrationtests/testhelpers.go
index da0ecd8..ca9f7a7 100644
--- a/integrationtests/testhelpers.go
+++ b/integrationtests/testhelpers.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -157,9 +158,42 @@ func startTestServer(ctx context.Context, t *testing.T, cfg *ServerConfig) error
 		return err
 	}
 
-	// Give server time to start
-	time.Sleep(500 * time.Millisecond)
-	return nil
+	return waitForServerReady(ctx, cfg.BindAddress, cfg.Port)
+}
+
+func waitForServerReady(ctx context.Context, bindAddress string, port int) error {
+	address := fmt.Sprintf("%s:%d", bindAddress, port)
+	deadline := time.Now().Add(10 * time.Second)
+	var lastErr error
+	var lastOutput string
+
+	for {
+		cmd := exec.CommandContext(ctx, "../dtailhealth", "--server", address, "--no-auth-key")
+		out, err := cmd.CombinedOutput()
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		lastOutput = strings.TrimSpace(string(out))
+
+		if ctx.Err() != nil {
+			return fmt.Errorf("wait for dserver %s: %w", address, ctx.Err())
+		}
+		if time.Now().After(deadline) {
+			if lastOutput != "" {
+				return fmt.Errorf("timed out waiting for dserver %s: %w (%s)", address, lastErr, lastOutput)
+			}
+			return fmt.Errorf("timed out waiting for dserver %s: %w", address, lastErr)
+		}
+
+		timer := time.NewTimer(50 * time.Millisecond)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return fmt.Errorf("wait for dserver %s: %w", address, ctx.Err())
+		case <-timer.C:
+		}
+	}
 }
 
 // createTestContext creates a context with cancel that will be cleaned up automatically