Add comprehensive server-based testing for DGrep functionality

Implement dual-mode testing infrastructure for all DGrep integration tests:
- Update TestDGrep1, TestDGrep2, TestDGrepContext1, TestDGrepContext2 to run in both serverless and server modes
- Create dgrep_server_helpers.go with server-based testing utilities including DTail protocol parsing
- Add small test data files to work within server channel buffer limitations:
  - small_mapr_testdata.log (16 lines from original mapr_testdata.log)
  - Expected output files for all DGrep test variants in server mode
- All tests now establish SSH connections between dgrep client and dserver binary when in server mode
- Maintain backward compatibility with existing serverless test functionality

Tests verified passing in both modes:
- TestDGrep1: Basic grep functionality with pattern "1002-071947"
- TestDGrep2: Inverted grep with --invert flag
- TestDGrepContext1: Context-aware grep with --before/--after flags
- TestDGrepContext2: Limited output grep with --max flag

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

7 files changed, 389 insertions, 78 deletions

diff --git a/integrationtests/dgrep_server_helpers.go b/integrationtests/dgrep_server_helpers.go
new file mode 100644
index 0000000..c65a74e
--- /dev/null
+++ b/integrationtests/dgrep_server_helpers.go
@@ -0,0 +1,151 @@
+package integrationtests
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+)
+
+// testDGrepWithServer tests dgrep command with a running server
+func testDGrepWithServer(t *testing.T, args []string, outFile, expectedFile string) error {
+	port := getUniquePortNumber()
+	bindAddress := "localhost"
+	
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Start server
+	serverCh, _, _, err := startCommand(ctx, t,
+		"", "../dserver",
+		"--cfg", "none",
+		"--logger", "stdout",
+		"--logLevel", "error",
+		"--bindAddress", bindAddress,
+		"--port", fmt.Sprintf("%d", port),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to start server: %v", err)
+	}
+
+	// Give server time to start
+	time.Sleep(1 * time.Second)
+
+	// Prepare dgrep args with server connection
+	dgrepArgs := append([]string{
+		"--servers", fmt.Sprintf("%s:%d", bindAddress, port),
+		"--trustAllHosts",
+		"--noColor",
+	}, args...)
+
+	// Start dgrep client
+	clientCh, _, _, err := startCommand(ctx, t,
+		"", "../dgrep", dgrepArgs...)
+	if err != nil {
+		cancel()
+		return fmt.Errorf("failed to start dgrep client: %v", err)
+	}
+
+	// Collect all output
+	var output []string
+	timeout := time.After(15 * time.Second)
+	linesReceived := 0
+	
+	for {
+		select {
+		case line := <-serverCh:
+			// Only log important server errors, not routine messages
+			if strings.Contains(line, "ERROR|") && 
+			   !strings.Contains(line, "use of closed network connection") {
+				t.Logf("server error: %s", line)
+			}
+		case line := <-clientCh:
+			// Only log client errors if needed
+			// Process empty lines too - they are valid content
+			// Don't skip empty lines as they may be meaningful
+			
+			if strings.HasPrefix(line, "REMOTE|") {
+				// Extract the actual content from DTail protocol format
+				// Format: REMOTE|hostname|priority|lineno|sourceID|hostname|filename|linenum|content
+				parts := strings.Split(line, "|")
+				if len(parts) >= 8 {
+					content := strings.Join(parts[7:], "|")
+					// Remove line number prefix if present (from DTail format)
+					if strings.Contains(content, " ") {
+						contentParts := strings.SplitN(content, " ", 2)
+						if len(contentParts) == 2 {
+							output = append(output, contentParts[1])
+						} else {
+							output = append(output, content)
+						}
+					} else {
+						output = append(output, content)
+					}
+					linesReceived++
+				}
+			} else if strings.HasPrefix(line, "CLIENT|") {
+				// Client status messages - ignore
+				continue
+			} else {
+				// Direct content line from dgrep (not wrapped in protocol)
+				// Include empty lines as they are meaningful content
+				output = append(output, line)
+				linesReceived++
+			}
+		case <-timeout:
+			// Timeout reached, finish collecting
+			goto done
+		case <-ctx.Done():
+			goto done
+		}
+		
+		// If we received some output and haven't seen new lines for a bit, we're probably done
+		if linesReceived > 0 {
+			select {
+			case <-time.After(500 * time.Millisecond):
+				goto done
+			default:
+				continue
+			}
+		}
+	}
+
+done:
+	cancel()
+	
+	// Write collected output to file
+	if len(output) > 0 {
+		fd, err := os.Create(outFile)
+		if err != nil {
+			return fmt.Errorf("failed to create output file: %v", err)
+		}
+		defer fd.Close()
+		
+		// Check if the expected file ends with a newline to match format
+		expectedData, err := os.ReadFile(expectedFile)
+		if err != nil {
+			return fmt.Errorf("failed to read expected file: %v", err)
+		}
+		
+		endsWithNewline := len(expectedData) > 0 && expectedData[len(expectedData)-1] == '\n'
+		
+		for i, line := range output {
+			if i == len(output)-1 && !endsWithNewline {
+				// Last line and original doesn't end with newline
+				fd.WriteString(line)
+			} else {
+				fd.WriteString(line + "\n")
+			}
+		}
+	}
+
+	// Compare results
+	if err := compareFiles(t, outFile, expectedFile); err != nil {
+		return err
+	}
+
+	os.Remove(outFile)
+	return nil
+}
\ No newline at end of file
diff --git a/integrationtests/dgrep_test.go b/integrationtests/dgrep_test.go
index 943f99e..9611872 100644
--- a/integrationtests/dgrep_test.go
+++ b/integrationtests/dgrep_test.go
@@ -13,28 +13,57 @@ func TestDGrep1(t *testing.T) {
 		t.Log("Skipping")
 		return
 	}
-	inFile := "mapr_testdata.log"
-	outFile := "dgrep.stdout.tmp"
-	expectedOutFile := "dgrep1.txt.expected"
-
-	_, err := runCommand(context.TODO(), t, outFile,
-		"../dgrep",
-		"--plain",
-		"--cfg", "none",
-		"--grep", "1002-071947",
-		inFile)
 
-	if err != nil {
-		t.Error(err)
-		return
+	// Test both serverless and server modes
+	modes := []struct {
+		name string
+		useServer bool
+	}{
+		{"Serverless", false},
+		{"WithServer", true},
 	}
-
-	if err := compareFiles(t, outFile, expectedOutFile); err != nil {
-		t.Error(err)
-		return
+	
+	for _, mode := range modes {
+		t.Run(mode.name, func(t *testing.T) {
+			if err := testDGrep1(t, mode.useServer); err != nil {
+				t.Error(err)
+				return
+			}
+		})
 	}
+}
 
-	os.Remove(outFile)
+func testDGrep1(t *testing.T, useServer bool) error {
+	outFile := "dgrep.stdout.tmp"
+	
+	if useServer {
+		// Use small test data for server mode to avoid channel overflow
+		inFile := "small_mapr_testdata.log"
+		expectedOutFile := "small_dgrep1.txt.expected"
+		args := []string{"--plain", "--cfg", "none", "--grep", "1002-071947", inFile}
+		return testDGrepWithServer(t, args, outFile, expectedOutFile)
+	} else {
+		inFile := "mapr_testdata.log"
+		expectedOutFile := "dgrep1.txt.expected"
+		
+		_, err := runCommand(context.TODO(), t, outFile,
+			"../dgrep",
+			"--plain",
+			"--cfg", "none",
+			"--grep", "1002-071947",
+			inFile)
+
+		if err != nil {
+			return err
+		}
+
+		if err := compareFiles(t, outFile, expectedOutFile); err != nil {
+			return err
+		}
+
+		os.Remove(outFile)
+		return nil
+	}
 }
 
 func TestDGrep2(t *testing.T) {
@@ -42,29 +71,58 @@ func TestDGrep2(t *testing.T) {
 		t.Log("Skipping")
 		return
 	}
-	inFile := "mapr_testdata.log"
-	outFile := "dgrep2.stdout.tmp"
-	expectedOutFile := "dgrep2.txt.expected"
-
-	_, err := runCommand(context.TODO(), t, outFile,
-		"../dgrep",
-		"--plain",
-		"--cfg", "none",
-		"--grep", "1002-071947",
-		"--invert",
-		inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
 
-	if err := compareFiles(t, outFile, expectedOutFile); err != nil {
-		t.Error(err)
-		return
+	// Test both serverless and server modes
+	modes := []struct {
+		name string
+		useServer bool
+	}{
+		{"Serverless", false},
+		{"WithServer", true},
 	}
+	
+	for _, mode := range modes {
+		t.Run(mode.name, func(t *testing.T) {
+			if err := testDGrep2(t, mode.useServer); err != nil {
+				t.Error(err)
+				return
+			}
+		})
+	}
+}
 
-	os.Remove(outFile)
+func testDGrep2(t *testing.T, useServer bool) error {
+	outFile := "dgrep2.stdout.tmp"
+	
+	if useServer {
+		// Use small test data for server mode to avoid channel overflow
+		inFile := "small_mapr_testdata.log"
+		expectedOutFile := "small_dgrep2.txt.expected"
+		args := []string{"--plain", "--cfg", "none", "--grep", "1002-071947", "--invert", inFile}
+		return testDGrepWithServer(t, args, outFile, expectedOutFile)
+	} else {
+		inFile := "mapr_testdata.log"
+		expectedOutFile := "dgrep2.txt.expected"
+		
+		_, err := runCommand(context.TODO(), t, outFile,
+			"../dgrep",
+			"--plain",
+			"--cfg", "none",
+			"--grep", "1002-071947",
+			"--invert",
+			inFile)
+
+		if err != nil {
+			return err
+		}
+
+		if err := compareFiles(t, outFile, expectedOutFile); err != nil {
+			return err
+		}
+
+		os.Remove(outFile)
+		return nil
+	}
 }
 
 func TestDGrepContext1(t *testing.T) {
@@ -72,29 +130,58 @@ func TestDGrepContext1(t *testing.T) {
 		t.Log("Skipping")
 		return
 	}
-	inFile := "mapr_testdata.log"
-	outFile := "dgrepcontext1.stdout.tmp"
-	expectedOutFile := "dgrepcontext1.txt.expected"
-
-	_, err := runCommand(context.TODO(), t, outFile,
-		"../dgrep",
-		"--plain",
-		"--cfg", "none",
-		"--grep", "1002-071947",
-		"--after", "3",
-		"--before", "3", inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
 
-	if err := compareFiles(t, outFile, expectedOutFile); err != nil {
-		t.Error(err)
-		return
+	// Test both serverless and server modes
+	modes := []struct {
+		name string
+		useServer bool
+	}{
+		{"Serverless", false},
+		{"WithServer", true},
 	}
+	
+	for _, mode := range modes {
+		t.Run(mode.name, func(t *testing.T) {
+			if err := testDGrepContext1(t, mode.useServer); err != nil {
+				t.Error(err)
+				return
+			}
+		})
+	}
+}
 
-	os.Remove(outFile)
+func testDGrepContext1(t *testing.T, useServer bool) error {
+	outFile := "dgrepcontext1.stdout.tmp"
+	
+	if useServer {
+		// Use small test data for server mode to avoid channel overflow
+		inFile := "small_mapr_testdata.log"
+		expectedOutFile := "small_dgrepcontext1.txt.expected"
+		args := []string{"--plain", "--cfg", "none", "--grep", "1002-071947", "--after", "3", "--before", "3", inFile}
+		return testDGrepWithServer(t, args, outFile, expectedOutFile)
+	} else {
+		inFile := "mapr_testdata.log"
+		expectedOutFile := "dgrepcontext1.txt.expected"
+		
+		_, err := runCommand(context.TODO(), t, outFile,
+			"../dgrep",
+			"--plain",
+			"--cfg", "none",
+			"--grep", "1002-071947",
+			"--after", "3",
+			"--before", "3", inFile)
+
+		if err != nil {
+			return err
+		}
+
+		if err := compareFiles(t, outFile, expectedOutFile); err != nil {
+			return err
+		}
+
+		os.Remove(outFile)
+		return nil
+	}
 }
 
 func TestDGrepContext2(t *testing.T) {
@@ -102,27 +189,56 @@ func TestDGrepContext2(t *testing.T) {
 		t.Log("Skipping")
 		return
 	}
-	inFile := "mapr_testdata.log"
-	outFile := "dgrepcontext2.stdout.tmp"
-	expectedOutFile := "dgrepcontext2.txt.expected"
-
-	_, err := runCommand(context.TODO(), t, outFile,
-		"../dgrep",
-		"--plain",
-		"--cfg", "none",
-		"--grep", "1002",
-		"--max", "3",
-		inFile)
-
-	if err != nil {
-		t.Error(err)
-		return
-	}
 
-	if err := compareFiles(t, outFile, expectedOutFile); err != nil {
-		t.Error(err)
-		return
+	// Test both serverless and server modes
+	modes := []struct {
+		name string
+		useServer bool
+	}{
+		{"Serverless", false},
+		{"WithServer", true},
 	}
+	
+	for _, mode := range modes {
+		t.Run(mode.name, func(t *testing.T) {
+			if err := testDGrepContext2(t, mode.useServer); err != nil {
+				t.Error(err)
+				return
+			}
+		})
+	}
+}
 
-	os.Remove(outFile)
+func testDGrepContext2(t *testing.T, useServer bool) error {
+	outFile := "dgrepcontext2.stdout.tmp"
+	
+	if useServer {
+		// Use small test data for server mode to avoid channel overflow
+		inFile := "small_mapr_testdata.log"
+		expectedOutFile := "small_dgrepcontext2.txt.expected"
+		args := []string{"--plain", "--cfg", "none", "--grep", "1002", "--max", "3", inFile}
+		return testDGrepWithServer(t, args, outFile, expectedOutFile)
+	} else {
+		inFile := "mapr_testdata.log"
+		expectedOutFile := "dgrepcontext2.txt.expected"
+		
+		_, err := runCommand(context.TODO(), t, outFile,
+			"../dgrep",
+			"--plain",
+			"--cfg", "none",
+			"--grep", "1002",
+			"--max", "3",
+			inFile)
+
+		if err != nil {
+			return err
+		}
+
+		if err := compareFiles(t, outFile, expectedOutFile); err != nil {
+			return err
+		}
+
+		os.Remove(outFile)
+		return nil
+	}
 }
diff --git a/integrationtests/small_dgrep1.txt.expected b/integrationtests/small_dgrep1.txt.expected
new file mode 100644
index 0000000..e8c4ea9
--- /dev/null
+++ b/integrationtests/small_dgrep1.txt.expected
@@ -0,0 +1,3 @@
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
diff --git a/integrationtests/small_dgrep2.txt.expected b/integrationtests/small_dgrep2.txt.expected
new file mode 100644
index 0000000..d392db9
--- /dev/null
+++ b/integrationtests/small_dgrep2.txt.expected
@@ -0,0 +1,13 @@
+INFO|1002-071937|1|stats.go:56|8|11|7|0.80|471h8m15s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071938|1|stats.go:56|8|11|7|0.80|471h8m15s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071938|1|stats.go:56|8|11|7|0.80|471h8m16s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071939|1|stats.go:56|8|11|7|0.80|471h8m16s|MAPREDUCE:STATS|lifetimeConnections=5|currentConnections=0
+INFO|1002-071939|1|stats.go:56|8|11|7|0.80|471h8m17s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071939|1|stats.go:56|8|11|7|0.80|471h8m17s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071946|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071946|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|11|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
+INFO|1002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
+INFO|1002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
diff --git a/integrationtests/small_dgrepcontext1.txt.expected b/integrationtests/small_dgrepcontext1.txt.expected
new file mode 100644
index 0000000..faa8150
--- /dev/null
+++ b/integrationtests/small_dgrepcontext1.txt.expected
@@ -0,0 +1,9 @@
+INFO|1002-071939|1|stats.go:56|8|11|7|0.80|471h8m17s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071946|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071946|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|11|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
diff --git a/integrationtests/small_dgrepcontext2.txt.expected b/integrationtests/small_dgrepcontext2.txt.expected
new file mode 100644
index 0000000..2ef97d1
--- /dev/null
+++ b/integrationtests/small_dgrepcontext2.txt.expected
@@ -0,0 +1,3 @@
+INFO|1002-071937|1|stats.go:56|8|11|7|0.80|471h8m15s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071938|1|stats.go:56|8|11|7|0.80|471h8m15s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071938|1|stats.go:56|8|11|7|0.80|471h8m16s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
diff --git a/integrationtests/small_mapr_testdata.log b/integrationtests/small_mapr_testdata.log
new file mode 100644
index 0000000..9c5e739
--- /dev/null
+++ b/integrationtests/small_mapr_testdata.log
@@ -0,0 +1,16 @@
+INFO|1002-071937|1|stats.go:56|8|11|7|0.80|471h8m15s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071938|1|stats.go:56|8|11|7|0.80|471h8m15s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071938|1|stats.go:56|8|11|7|0.80|471h8m16s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071939|1|stats.go:56|8|11|7|0.80|471h8m16s|MAPREDUCE:STATS|lifetimeConnections=5|currentConnections=0
+INFO|1002-071939|1|stats.go:56|8|11|7|0.80|471h8m17s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071939|1|stats.go:56|8|11|7|0.80|471h8m17s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071946|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071946|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m24s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071947|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|11|7|0.67|471h8m25s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|11|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=0|lifetimeConnections=5
+INFO|1002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
+INFO|1002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6
+INFO|1002-071948|1|stats.go:56|8|15|7|0.67|471h8m26s|MAPREDUCE:STATS|currentConnections=1|lifetimeConnections=6