From 92e1c683ef38f1eab71737cc1877ba5f9e7c8777 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 11:11:06 +0100 Subject: update dependencies --- go.mod | 10 +++++----- go.sum | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 4b9d798..73711b3 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/mimecast/dtail -go 1.17 +go 1.18 require ( - github.com/DataDog/zstd v1.5.0 - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 + github.com/DataDog/zstd v1.5.2 + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d ) require ( - golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect + golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect ) diff --git a/go.sum b/go.sum index 0201a8f..6a15bee 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ -github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= -github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew= +golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -- cgit v1.2.3 From 5a7c00f6fa0d47dccac59de53af9603ac03275eb Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 11:16:50 +0100 Subject: Update dependencies --- go.mod | 2 +- go.sum | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 2329f96..73711b3 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/DataDog/zstd v1.5.2 - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d ) require ( diff --git a/go.sum b/go.sum index 6a15bee..e4ff218 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,7 @@ github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew= golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -- cgit v1.2.3 From 3c9e5c71d30209b415df9c0d7149da2c3a923c7a Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 11:32:14 +0100 Subject: Fix typos. --- internal/clients/stats.go | 2 +- internal/discovery/discovery.go | 2 +- internal/server/server.go | 2 +- internal/user/server/user.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/clients/stats.go b/internal/clients/stats.go index 1315aea..2da3cf7 100644 --- a/internal/clients/stats.go +++ b/internal/clients/stats.go @@ -34,7 +34,7 @@ func newTailStats(servers int) *stats { } } -// Start starts printing client connection stats every time a signal is recieved or +// Start starts printing client connection stats every time a signal is received or // connection count has changed. func (s *stats) Start(ctx context.Context, throttleCh <-chan struct{}, statsCh <-chan string, quiet bool) { diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index 8bb1e85..f709501 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -24,7 +24,7 @@ const ( type Discovery struct { // To plug in a custom server discovery module. module string - // To specifiy optional server discovery module options. + // To specify optional server discovery module options. options string // To either filter a server list or to secify an exact list. server string diff --git a/internal/server/server.go b/internal/server/server.go index c7d7aaa..30602ff 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -200,7 +200,7 @@ func (s *Server) handleRequests(ctx context.Context, sshConn gossh.Conn, req.Reply(true, nil) default: req.Reply(false, nil) - return fmt.Errorf("Closing SSH connection as unknown request recieved|%s|%v", + return fmt.Errorf("Closing SSH connection as unknown request received|%s|%v", req.Type, payload.Value) } } diff --git a/internal/user/server/user.go b/internal/user/server/user.go index 004bda4..e704c8b 100644 --- a/internal/user/server/user.go +++ b/internal/user/server/user.go @@ -42,7 +42,7 @@ func (u *User) String() string { return fmt.Sprintf("%s@%s", u.Name, u.remoteAddress) } -// HasFilePermission is used to determine whether user is alowed to read a file. +// HasFilePermission is used to determine whether user is allowed to read a file. func (u *User) HasFilePermission(filePath, permissionType string) (hasPermission bool) { dlog.Server.Debug(u, filePath, permissionType, "Checking config permissions") if u.Name == config.ScheduleUser || u.Name == config.ContinuousUser { @@ -121,7 +121,7 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) { continue } - regexStr = permission + regexStr := permission if strings.HasPrefix(permission, "!") { regexStr = permission[1:] negate = true -- cgit v1.2.3 From a3bb8a44a6ba6f7184e8ce2f5bd16a69162794f1 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 5 Jun 2023 16:21:31 +0300 Subject: can configure SSH algorithms --- internal/config/server.go | 6 ++++++ internal/server/server.go | 16 +++++++++++----- internal/user/server/user.go | 2 +- internal/version/version.go | 2 +- samples/dtail.schema.json | 18 ++++++++++++++++++ 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/internal/config/server.go b/internal/config/server.go index 4c96567..cb9ca2b 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -61,6 +61,12 @@ type ServerConfig struct { Schedule []Scheduled `json:",omitempty"` // Continuous mapreduce jobs Continuous []Continuous `json:",omitempty"` + // The allowed key exchanges algorithms. + KeyExchanges []string `json:",omitempty"` + // The allowed cipher algorithms. + Ciphers []string `json:",omitempty"` + // The allowed MAC algorithms. + MACs []string `json:",omitempty"` } // Create a new default server configuration. diff --git a/internal/server/server.go b/internal/server/server.go index 30602ff..761880d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -39,11 +39,17 @@ func New() *Server { dlog.Server.Info("Starting server", version.String()) s := Server{ - sshServerConfig: &gossh.ServerConfig{}, - catLimiter: make(chan struct{}, config.Server.MaxConcurrentCats), - tailLimiter: make(chan struct{}, config.Server.MaxConcurrentTails), - sched: newScheduler(), - cont: newContinuous(), + sshServerConfig: &gossh.ServerConfig{ + Config: gossh.Config{ + KeyExchanges: config.Server.KeyExchanges, + Ciphers: config.Server.Ciphers, + MACs: config.Server.MACs, + }, + }, + catLimiter: make(chan struct{}, config.Server.MaxConcurrentCats), + tailLimiter: make(chan struct{}, config.Server.MaxConcurrentTails), + sched: newScheduler(), + cont: newContinuous(), } s.sshServerConfig.PasswordCallback = s.Callback diff --git a/internal/user/server/user.go b/internal/user/server/user.go index e704c8b..abf74f3 100644 --- a/internal/user/server/user.go +++ b/internal/user/server/user.go @@ -121,7 +121,7 @@ func (u *User) iteratePaths(cleanPath, permissionType string) (bool, error) { continue } - regexStr := permission + regexStr = permission if strings.HasPrefix(permission, "!") { regexStr = permission[1:] negate = true diff --git a/internal/version/version.go b/internal/version/version.go index 3eef9ae..c60c04c 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ const ( // Name of DTail. Name string = "DTail" // Version of DTail. - Version string = "4.1.0" + Version string = "4.2.0" // Additional information for DTail Additional string = "SNAPSHOT release" ) diff --git a/samples/dtail.schema.json b/samples/dtail.schema.json index 1ee9980..44fb4c2 100755 --- a/samples/dtail.schema.json +++ b/samples/dtail.schema.json @@ -246,6 +246,24 @@ }, "TextFg": { "#ref": "#/definitions/color" + }, + "KeyExchanges": { + "type": "array", + "items": { + "type": "string" + } + }, + "Ciphers": { + "type": "array", + "items": { + "type": "string" + } + }, + "MACs": { + "type": "array", + "items": { + "type": "string" + } } } }, -- cgit v1.2.3 From 3e3641d85716fae2890163e7afd8d79478dae4ef Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 5 Jun 2023 17:35:33 +0300 Subject: update dependencies --- go.mod | 10 +++++----- go.sum | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 73711b3..5c9cf64 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/mimecast/dtail -go 1.18 +go 1.20 require ( - github.com/DataDog/zstd v1.5.2 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d + github.com/DataDog/zstd v1.5.5 + golang.org/x/crypto v0.9.0 ) require ( - golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect - golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index e4ff218..5e3fe8c 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= -github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew= -golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -- cgit v1.2.3 From d3e98fb73a5966736f9c5a9479bad27eff2cccd3 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 21 Jun 2023 11:26:31 +0300 Subject: rename samples to examples --- Makefile | 2 +- doc/installation.md | 18 +- examples/check_dserver.sh.example | 3 + examples/dserver-update-keycache.service.example | 6 + examples/dserver-update-keycache.timer.example | 5 + examples/dserver.service.example | 19 + examples/dtail.json.example | 127 ++++++ examples/dtail.schema.json | 532 +++++++++++++++++++++++ examples/update_key_cache.sh.example | 33 ++ samples/check_dserver.sh.sample | 3 - samples/dserver-update-keycache.service.sample | 6 - samples/dserver-update-keycache.timer.sample | 5 - samples/dserver.service.sample | 19 - samples/dtail.json.sample | 127 ------ samples/dtail.schema.json | 532 ----------------------- samples/update_key_cache.sh.sample | 33 -- 16 files changed, 735 insertions(+), 735 deletions(-) create mode 100755 examples/check_dserver.sh.example create mode 100644 examples/dserver-update-keycache.service.example create mode 100644 examples/dserver-update-keycache.timer.example create mode 100644 examples/dserver.service.example create mode 100644 examples/dtail.json.example create mode 100755 examples/dtail.schema.json create mode 100644 examples/update_key_cache.sh.example delete mode 100755 samples/check_dserver.sh.sample delete mode 100644 samples/dserver-update-keycache.service.sample delete mode 100644 samples/dserver-update-keycache.timer.sample delete mode 100644 samples/dserver.service.sample delete mode 100644 samples/dtail.json.sample delete mode 100755 samples/dtail.schema.json delete mode 100644 samples/update_key_cache.sh.sample diff --git a/Makefile b/Makefile index 680cef6..d764051 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ clean: test -f $$cmd && rm $$cmd; \ done vet: - find . -type d | egrep -v '(./samples|./log|./doc)' | while read dir; do \ + find . -type d | egrep -v '(./examples|./log|./doc)' | while read dir; do \ echo ${GO} vet $$dir; \ ${GO} vet $$dir; \ done diff --git a/doc/installation.md b/doc/installation.md index 0f6143b..a2c271a 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -53,18 +53,18 @@ uid=1001(dserver) 1001=670(dserver) groups=1001(dserver) % sudo chown -R dserver:dserver /var/run/dserver ``` -4. Install the ``dtail.json`` config to ``/etc/dserver/dtail.json``. An example can be found [here](../samples/dtail.json.sample). +4. Install the ``dtail.json`` config to ``/etc/dserver/dtail.json``. An example can be found [here](../examples/dtail.json.example). ```console % sudo mkdir /etc/dserver -% curl https://raw.githubusercontent.com/mimecast/dtail/master/samples/dtail.json.sample | +% curl https://raw.githubusercontent.com/mimecast/dtail/master/examples/dtail.json.example | sudo tee /etc/dserver/dtail.json ``` -5. It is recommended to configure DTail server as a service to ``systemd``. An example unit file for ``systemd`` can be found [here](../samples/dserver.service.sample). +5. It is recommended to configure DTail server as a service to ``systemd``. An example unit file for ``systemd`` can be found [here](../examples/dserver.service.example). ```console -% curl https://raw.githubusercontent.com/mimecast/dtail/master/samples/dserver.service.sample | +% curl https://raw.githubusercontent.com/mimecast/dtail/master/examples/dserver.service.example | sudo tee /etc/systemd/system/dserver.service % sudo systemctl daemon-reload % sudo systemctl enable dserver @@ -97,15 +97,15 @@ To start the DTail server via ``systemd`` run: The DTail server now runs as a ``systemd`` service under system user ``dserver``. However, the system user ``dserver`` has no permissions to read the SSH public keys from ``/home/USER/.ssh/authorized_keys``. Therefore, no user would be able to establish an SSH session to DTail server. As an alternative path DTail server also checks for public SSH key files in ``/var/run/dserver/cache/USER.authorized_keys``. -It is recommended to execute [update_key_cache.sh](../samples/update_key_cache.sh.sample) periodically to update the key cache. In case you manage your public SSH keys via Puppet you could subscribe the script to corresponding module. Or alternatively just configure a cron job or a systemd timer to run every once in a while, e.g. every 30 minutes: +It is recommended to execute [update_key_cache.sh](../examples/update_key_cache.sh.example) periodically to update the key cache. In case you manage your public SSH keys via Puppet you could subscribe the script to corresponding module. Or alternatively just configure a cron job or a systemd timer to run every once in a while, e.g. every 30 minutes: ```console -% curl https://raw.githubusercontent.com/mimecast/dtail/master/samples/update_key_cache.sh.sample | +% curl https://raw.githubusercontent.com/mimecast/dtail/master/examples/update_key_cache.sh.example | sudo tee /var/run/dserver/update_key_cache.sh % sudo chmod 755 /var/run/dserver/update_key_cache.sh -% curl https://raw.githubusercontent.com/mimecast/dtail/master/samples/dserver-update-keycache.service.sample | +% curl https://raw.githubusercontent.com/mimecast/dtail/master/examples/dserver-update-keycache.service.example | sudo tee /etc/systemd/system/dserver-update-keycache.service -% curl https://raw.githubusercontent.com/mimecast/dtail/master/samples/dserver-update-keycache.timer.sample | +% curl https://raw.githubusercontent.com/mimecast/dtail/master/examples/dserver-update-keycache.timer.example | sudo tee /etc/systemd/system/dserver-update-keycache.timer % sudo systemctl daemon-reload % sudo systemctl start dserver-update-keycache.service @@ -119,7 +119,7 @@ Now you should be able to use DTail client like outlined in the [Quick Starting # Monitor it -To verify that DTail server is up and running and functioning as expected, you should configure the Nagios check [check_dserver.sh](../samples/check_dserver.sh.sample) in your monitoring system. The check has to be executed locally on the server (e.g. via NRPE). How to configure the monitoring system in detail is out of scope of this guide. +To verify that DTail server is up and running and functioning as expected, you should configure the Nagios check [check_dserver.sh](../examples/check_dserver.sh.example) in your monitoring system. The check has to be executed locally on the server (e.g. via NRPE). How to configure the monitoring system in detail is out of scope of this guide. ```console % ./check_dserver.sh diff --git a/examples/check_dserver.sh.example b/examples/check_dserver.sh.example new file mode 100755 index 0000000..77f01f0 --- /dev/null +++ b/examples/check_dserver.sh.example @@ -0,0 +1,3 @@ +#!/bin/sh + +exec /usr/local/bin/dtailhealth --server localhost:2222 diff --git a/examples/dserver-update-keycache.service.example b/examples/dserver-update-keycache.service.example new file mode 100644 index 0000000..7e6144c --- /dev/null +++ b/examples/dserver-update-keycache.service.example @@ -0,0 +1,6 @@ +[Unit] +Description=Refresh DServer SSH Key Cache + +[Service] +Type=oneshot +ExecStart=/var/run/dserver/update_key_cache.sh diff --git a/examples/dserver-update-keycache.timer.example b/examples/dserver-update-keycache.timer.example new file mode 100644 index 0000000..e7158ca --- /dev/null +++ b/examples/dserver-update-keycache.timer.example @@ -0,0 +1,5 @@ +[Unit] +Description=Refresh DServer SSH Key Cache every 30 minutes. + +[Timer] +OnCalendar=*:0/30 diff --git a/examples/dserver.service.example b/examples/dserver.service.example new file mode 100644 index 0000000..c5e5e59 --- /dev/null +++ b/examples/dserver.service.example @@ -0,0 +1,19 @@ +[Unit] +Description=DTail server +After=network.target + +[Service] +Slice=dserver.slice +User=dserver +Group=dserver +ExecStart=/usr/local/bin/dserver -cfg /etc/dserver/dtail.json +WorkingDirectory=/var/run/dserver +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +CPUAccounting=true +MemoryAccounting=true +BlockIOAccounting=true + +[Install] +WantedBy=multi-user.target diff --git a/examples/dtail.json.example b/examples/dtail.json.example new file mode 100644 index 0000000..26eb8a1 --- /dev/null +++ b/examples/dtail.json.example @@ -0,0 +1,127 @@ +{ + "Client": { + "TermColorsEnable": true, + "TermColors": { + "Remote": { + "DelimiterAttr": "Dim", + "DelimiterBg": "Blue", + "DelimiterFg": "Cyan", + "RemoteAttr": "Dim", + "RemoteBg": "Blue", + "RemoteFg": "White", + "CountAttr": "Dim", + "CountBg": "Blue", + "CountFg": "White", + "HostnameAttr": "Bold", + "HostnameBg": "Blue", + "HostnameFg": "White", + "IDAttr": "Dim", + "IDBg": "Blue", + "IDFg": "White", + "StatsOkAttr": "None", + "StatsOkBg": "Green", + "StatsOkFg": "Black", + "StatsWarnAttr": "None", + "StatsWarnBg": "Red", + "StatsWarnFg": "White", + "TextAttr": "None", + "TextBg": "Black", + "TextFg": "White" + }, + "Client": { + "DelimiterAttr": "Dim", + "DelimiterBg": "Yellow", + "DelimiterFg": "Black", + "ClientAttr": "Dim", + "ClientBg": "Yellow", + "ClientFg": "Black", + "HostnameAttr": "Dim", + "HostnameBg": "Yellow", + "HostnameFg": "Black", + "TextAttr": "None", + "TextBg": "Black", + "TextFg": "White" + }, + "Server": { + "DelimiterAttr": "AttrDim", + "DelimiterBg": "BgCyan", + "DelimiterFg": "FgBlack", + "ServerAttr": "AttrDim", + "ServerBg": "BgCyan", + "ServerFg": "FgBlack", + "HostnameAttr": "AttrBold", + "HostnameBg": "BgCyan", + "HostnameFg": "FgBlack", + "TextAttr": "AttrNone", + "TextBg": "BgBlack", + "TextFg": "FgWhite" + }, + "Common": { + "SeverityErrorAttr": "AttrBold", + "SeverityErrorBg": "BgRed", + "SeverityErrorFg": "FgWhite", + "SeverityFatalAttr": "AttrBold", + "SeverityFatalBg": "BgMagenta", + "SeverityFatalFg": "FgWhite", + "SeverityWarnAttr": "AttrBold", + "SeverityWarnBg": "BgBlack", + "SeverityWarnFg": "FgWhite" + }, + "MaprTable": { + "DataAttr": "AttrNone", + "DataBg": "BgBlue", + "DataFg": "FgWhite", + "DelimiterAttr": "AttrDim", + "DelimiterBg": "BgBlue", + "DelimiterFg": "FgWhite", + "HeaderAttr": "AttrBold", + "HeaderBg": "BgBlue", + "HeaderFg": "FgWhite", + "HeaderDelimiterAttr": "AttrDim", + "HeaderDelimiterBg": "BgBlue", + "HeaderDelimiterFg": "FgWhite", + "HeaderSortKeyAttr": "AttrUnderline", + "HeaderGroupKeyAttr": "AttrReverse", + "RawQueryAttr": "AttrDim", + "RawQueryBg": "BgBlack", + "RawQueryFg": "FgCyan" + } + } + }, + "Server": { + "SSHBindAddress": "0.0.0.0", + "HostKeyFile": "cache/ssh_host_key", + "HostKeyBits": 2048, + "MapreduceLogFormat": "default", + "MaxConcurrentCats": 2, + "MaxConcurrentTails": 50, + "MaxConnections": 50, + "MaxLineLength": 1048576, + "Permissions": { + "Default": [ + "readfiles:^/.*$" + ], + "Users": { + "paul": [ + "readfiles:^/.*$" + ], + "pbuetow": [ + "readfiles:^/.*$" + ], + "jamesblake": [ + "readfiles:^/tmp/foo.log$", + "readfiles:^/.*$", + "readfiles:!^/tmp/bar.log$" + ] + } + } + }, + "Common": { + "LogDir": "log", + "Logger": "Fout", + "LogRotation": "Daily", + "CacheDir": "cache", + "SSHPort": 2222, + "LogLevel": "Info" + } +} diff --git a/examples/dtail.schema.json b/examples/dtail.schema.json new file mode 100755 index 0000000..44fb4c2 --- /dev/null +++ b/examples/dtail.schema.json @@ -0,0 +1,532 @@ +{ + "$schema": "https://json-schema.org/2019-09/schema", + "description": "Schema for dtail.json", + "definitions": { + "userPermission": { + "type": "array", + "items": { + "type": "string" + } + }, + "userPermissions": { + "type": "object", + "patternProperties": { + "^.*$": { + "$ref": "#/definitions/userPermission" + } + } + }, + "loglevel": { + "type": "string", + "enum": [ + "None", + "Fatal", + "Error", + "Warn", + "Info", + "Default", + "Verbose", + "Debug", + "Devel", + "Trace", + "All" + ] + }, + "logger": { + "type": "string", + "enum": [ + "None", + "Stdout", + "File", + "Fout" + ] + }, + "logrotation": { + "type": "string", + "enum": [ + "Daily", + "Signal" + ] + }, + "color": { + "type": "string", + "enum": [ + "Black", + "Red", + "Green", + "Yellow", + "Blue", + "Magenta", + "Cyan", + "White" + ] + }, + "attribute": { + "type": "string", + "enum": [ + "None", + "Bold", + "Dim", + "Italic", + "Underline", + "Blink", + "SlowBlink", + "RapidBlink", + "Reverse", + "Hidden" + ] + } + }, + "type": "object", + "additionalProperties": false, + "properties": { + "Client": { + "additionalProperties": false, + "properties": { + "TermColorsEnable": { + "type": "boolean" + }, + "TermColors": { + "type": "object", + "additionalProperties": false, + "properties": { + "Remote": { + "additionalProperties": false, + "properties": { + "DelimiterAttr": { + "$ref": "#/definitions/attribute" + }, + "DelimiterBg": { + "$ref": "#/definitions/color" + }, + "DelimiterFg": { + "$ref": "#/definitions/color" + }, + "RemoteAttr": { + "$ref": "#/definitions/attribute" + }, + "RemoteBg": { + "$ref": "#/definitions/color" + }, + "RemoteFg": { + "$ref": "#/definitions/color" + }, + "CountAttr": { + "$ref": "#/definitions/attribute" + }, + "CountBg": { + "$ref": "#/definitions/color" + }, + "CountFg": { + "$ref": "#/definitions/color" + }, + "HostnameAttr": { + "$ref": "#/definitions/attribute" + }, + "HostnameBg": { + "$ref": "#/definitions/color" + }, + "HostnameFg": { + "$ref": "#/definitions/color" + }, + "IDAttr": { + "$ref": "#/definitions/attribute" + }, + "IDBg": { + "$ref": "#/definitions/color" + }, + "IDFg": { + "$ref": "#/definitions/color" + }, + "StatsOkAttr": { + "$ref": "#/definitions/attribute" + }, + "StatsOkBg": { + "$ref": "#/definitions/color" + }, + "StatsOkFg": { + "$ref": "#/definitions/color" + }, + "StatsWarnAttr": { + "$ref": "#/definitions/attribute" + }, + "StatsWarnBg": { + "$ref": "#/definitions/color" + }, + "StatsWarnFg": { + "$ref": "#/definitions/color" + }, + "TextAttr": { + "$ref": "#/definitions/attribute" + }, + "TextBg": { + "$ref": "#/definitions/color" + }, + "TextFg": { + "$ref": "#/definitions/color" + } + } + }, + "Client": { + "additionalProperties": false, + "properties": { + "DelimiterAttr": { + "$ref": "#/definitions/attribute" + }, + "DelimiterBg": { + "$ref": "#/definitions/color" + }, + "DelimiterFg": { + "$ref": "#/definitions/color" + }, + "ClientAttr": { + "$ref": "#/definitions/attribute" + }, + "ClientBg": { + "$ref": "#/definitions/color" + }, + "ClientFg": { + "$ref": "#/definitions/color" + }, + "HostnameAttr": { + "$ref": "#/definitions/attribute" + }, + "HostnameBg": { + "$ref": "#/definitions/color" + }, + "HostnameFg": { + "$ref": "#/definitions/color" + }, + "TextAttr": { + "$ref": "#/definitions/attribute" + }, + "TextBg": { + "$ref": "#/definitions/color" + }, + "TextFg": { + "$ref": "#/definitions/color" + } + } + }, + "Server": { + "additionalProperties": false, + "properties": { + "DelimiterAttr": { + "#ref": "#/definitions/attribute" + }, + "DelimiterBg": { + "#ref": "#/definitions/color" + }, + "DelimiterFg": { + "#ref": "#/definitions/color" + }, + "ServerAttr": { + "#ref": "#/definitions/attribute" + }, + "ServerBg": { + "#ref": "#/definitions/color" + }, + "ServerFg": { + "#ref": "#/definitions/color" + }, + "HostnameAttr": { + "#ref": "#/definitions/attribute" + }, + "HostnameBg": { + "#ref": "#/definitions/color" + }, + "HostnameFg": { + "#ref": "#/definitions/color" + }, + "TextAttr": { + "#ref": "#/definitions/attribute" + }, + "TextBg": { + "#ref": "#/definitions/color" + }, + "TextFg": { + "#ref": "#/definitions/color" + }, + "KeyExchanges": { + "type": "array", + "items": { + "type": "string" + } + }, + "Ciphers": { + "type": "array", + "items": { + "type": "string" + } + }, + "MACs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Common": { + "additionalProperties": false, + "properties": { + "SeverityErrorAttr": { + "#ref": "#/definitions/attribute" + }, + "SeverityErrorBg": { + "#ref": "#/definitions/color" + }, + "SeverityErrorFg": { + "#ref": "#/definitions/color" + }, + "SeverityFatalAttr": { + "#ref": "#/definitions/attribute" + }, + "SeverityFatalBg": { + "#ref": "#/definitions/color" + }, + "SeverityFatalFg": { + "#ref": "#/definitions/color" + }, + "SeverityWarnAttr": { + "#ref": "#/definitions/attribute" + }, + "SeverityWarnBg": { + "#ref": "#/definitions/color" + }, + "SeverityWarnFg": { + "#ref": "#/definitions/color" + } + } + }, + "MaprTable": { + "additionalProperties": false, + "properties": { + "DataAttr": { + "#ref": "#/definitions/attribute" + }, + "DataBg": { + "#ref": "#/definitions/color" + }, + "DataFg": { + "#ref": "#/definitions/color" + }, + "DelimiterAttr": { + "#ref": "#/definitions/attribute" + }, + "DelimiterBg": { + "#ref": "#/definitions/color" + }, + "DelimiterFg": { + "#ref": "#/definitions/color" + }, + "HeaderAttr": { + "#ref": "#/definitions/attribute" + }, + "HeaderBg": { + "#ref": "#/definitions/color" + }, + "HeaderFg": { + "#ref": "#/definitions/color" + }, + "HeaderDelimiterAttr": { + "#ref": "#/definitions/attribute" + }, + "HeaderDelimiterBg": { + "#ref": "#/definitions/color" + }, + "HeaderDelimiterFg": { + "#ref": "#/definitions/color" + }, + "HeaderSortKeyAttr": { + "#ref": "#/definitions/attribute" + }, + "HeaderGroupKeyAttr": { + "#ref": "#/definitions/attribute" + }, + "RawQueryAttr": { + "#ref": "#/definitions/attribute" + }, + "RawQueryBg": { + "#ref": "#/definitions/color" + }, + "RawQueryFg": { + "#ref": "#/definitions/color" + } + } + } + } + } + } + }, + "Server": { + "additionalProperties": false, + "properties": { + "SSHBindAddress": { + "type": "string" + }, + "HostKeyFile": { + "type": "string" + }, + "HostKeyBits": { + "type": "integer", + "minimum": 2048 + }, + "MapreduceLogFormat": { + "type": "string" + }, + "MaxConcurrentCats": { + "type": "integer", + "minimum": 1, + "maximum": 20 + }, + "MaxConcurrentTails": { + "type": "integer", + "minimum": 1, + "maximum": 200 + }, + "MaxConnections": { + "type": "integer", + "minimum": 1, + "maximum": 200 + }, + "MaxLineLength": { + "type": "integer", + "minimum": 1024, + "maximum": 10240000 + }, + "Permissions": { + "type": "object", + "additionalProperties": true, + "patternProperties": { + "^Default$": { + "$ref": "#/definitions/userPermission" + }, + "^Users$": { + "$ref": "#/definitions/userPermissions" + } + } + }, + "Schedule": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "Name": { + "type": "string" + }, + "Enable": { + "type": "boolean" + }, + "AllowFrom": { + "type": "array", + "items": { + "type": "string" + } + }, + "Servers": { + "type": "array", + "items": { + "type": "string" + } + }, + "TimeRange": { + "type": "array", + "items": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ] + }, + "Files": { + "type": "string" + }, + "Outfile": { + "type": "string" + }, + "Query": { + "type": "string" + } + } + } + }, + "Continuous": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "Name": { + "type": "string" + }, + "Enable": { + "type": "boolean" + }, + "AllowFrom": { + "type": "array", + "items": { + "type": "string" + } + }, + "Servers": { + "type": "array", + "items": { + "type": "string" + } + }, + "RestartOnDayChange": { + "type": "boolean" + }, + "Files": { + "type": "string" + }, + "Outfile": { + "type": "string" + }, + "Query": { + "type": "string" + } + } + } + } + } + }, + "Common": { + "additionalProperties": false, + "properties": { + "LogDir": { + "type": "string" + }, + "Logger": { + "#ref": "#/definitions/logger" + }, + "LogLevel": { + "#ref": "#/definitions/loglevel" + }, + "LogRotation": { + "#ref": "#/definitions/logrotation" + }, + "CacheDir": { + "type": "string" + }, + "SSHPort": { + "type": "integer", + "minimum": 2, + "maximum": 16000 + }, + "ExperimentalFeaturesEnable": { + "type": "boolean" + } + } + } + }, + "required": [ + "Client", + "Server", + "Common" + ] +} diff --git a/examples/update_key_cache.sh.example b/examples/update_key_cache.sh.example new file mode 100644 index 0000000..9817f04 --- /dev/null +++ b/examples/update_key_cache.sh.example @@ -0,0 +1,33 @@ +#!/bin/bash + +declare -r CACHEDIR=/var/run/dserver/cache +declare -r DSERVER_USER=dserver + +echo "Updating SSH key cache" + +ls /home/ | while read remoteuser; do + keysfile=/home/$remoteuser/.ssh/authorized_keys + + if [ -f $keysfile ]; then + cachefile=$CACHEDIR/$remoteuser.authorized_keys + echo "Caching $keysfile -> $cachefile" + + cp $keysfile $cachefile + chown $DSERVER_USER $cachefile + chmod 600 $cachefile + fi +done + +# Cleanup obsolete public SSH keys +find $CACHEDIR -name \*.authorized_keys -type f | +while read cachefile; do + remoteuser=$(basename $cachefile | cut -d. -f1) + keysfile=/home/$remoteuser/.ssh/authorized_keys + + if [ ! -f $keysfile ]; then + echo "Deleting obsolete cache file $cachefile" + rm $cachefile + fi +done + +echo "All set..." diff --git a/samples/check_dserver.sh.sample b/samples/check_dserver.sh.sample deleted file mode 100755 index 77f01f0..0000000 --- a/samples/check_dserver.sh.sample +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -exec /usr/local/bin/dtailhealth --server localhost:2222 diff --git a/samples/dserver-update-keycache.service.sample b/samples/dserver-update-keycache.service.sample deleted file mode 100644 index 7e6144c..0000000 --- a/samples/dserver-update-keycache.service.sample +++ /dev/null @@ -1,6 +0,0 @@ -[Unit] -Description=Refresh DServer SSH Key Cache - -[Service] -Type=oneshot -ExecStart=/var/run/dserver/update_key_cache.sh diff --git a/samples/dserver-update-keycache.timer.sample b/samples/dserver-update-keycache.timer.sample deleted file mode 100644 index e7158ca..0000000 --- a/samples/dserver-update-keycache.timer.sample +++ /dev/null @@ -1,5 +0,0 @@ -[Unit] -Description=Refresh DServer SSH Key Cache every 30 minutes. - -[Timer] -OnCalendar=*:0/30 diff --git a/samples/dserver.service.sample b/samples/dserver.service.sample deleted file mode 100644 index c5e5e59..0000000 --- a/samples/dserver.service.sample +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=DTail server -After=network.target - -[Service] -Slice=dserver.slice -User=dserver -Group=dserver -ExecStart=/usr/local/bin/dserver -cfg /etc/dserver/dtail.json -WorkingDirectory=/var/run/dserver -NoNewPrivileges=true -PrivateDevices=true -PrivateTmp=true -CPUAccounting=true -MemoryAccounting=true -BlockIOAccounting=true - -[Install] -WantedBy=multi-user.target diff --git a/samples/dtail.json.sample b/samples/dtail.json.sample deleted file mode 100644 index 26eb8a1..0000000 --- a/samples/dtail.json.sample +++ /dev/null @@ -1,127 +0,0 @@ -{ - "Client": { - "TermColorsEnable": true, - "TermColors": { - "Remote": { - "DelimiterAttr": "Dim", - "DelimiterBg": "Blue", - "DelimiterFg": "Cyan", - "RemoteAttr": "Dim", - "RemoteBg": "Blue", - "RemoteFg": "White", - "CountAttr": "Dim", - "CountBg": "Blue", - "CountFg": "White", - "HostnameAttr": "Bold", - "HostnameBg": "Blue", - "HostnameFg": "White", - "IDAttr": "Dim", - "IDBg": "Blue", - "IDFg": "White", - "StatsOkAttr": "None", - "StatsOkBg": "Green", - "StatsOkFg": "Black", - "StatsWarnAttr": "None", - "StatsWarnBg": "Red", - "StatsWarnFg": "White", - "TextAttr": "None", - "TextBg": "Black", - "TextFg": "White" - }, - "Client": { - "DelimiterAttr": "Dim", - "DelimiterBg": "Yellow", - "DelimiterFg": "Black", - "ClientAttr": "Dim", - "ClientBg": "Yellow", - "ClientFg": "Black", - "HostnameAttr": "Dim", - "HostnameBg": "Yellow", - "HostnameFg": "Black", - "TextAttr": "None", - "TextBg": "Black", - "TextFg": "White" - }, - "Server": { - "DelimiterAttr": "AttrDim", - "DelimiterBg": "BgCyan", - "DelimiterFg": "FgBlack", - "ServerAttr": "AttrDim", - "ServerBg": "BgCyan", - "ServerFg": "FgBlack", - "HostnameAttr": "AttrBold", - "HostnameBg": "BgCyan", - "HostnameFg": "FgBlack", - "TextAttr": "AttrNone", - "TextBg": "BgBlack", - "TextFg": "FgWhite" - }, - "Common": { - "SeverityErrorAttr": "AttrBold", - "SeverityErrorBg": "BgRed", - "SeverityErrorFg": "FgWhite", - "SeverityFatalAttr": "AttrBold", - "SeverityFatalBg": "BgMagenta", - "SeverityFatalFg": "FgWhite", - "SeverityWarnAttr": "AttrBold", - "SeverityWarnBg": "BgBlack", - "SeverityWarnFg": "FgWhite" - }, - "MaprTable": { - "DataAttr": "AttrNone", - "DataBg": "BgBlue", - "DataFg": "FgWhite", - "DelimiterAttr": "AttrDim", - "DelimiterBg": "BgBlue", - "DelimiterFg": "FgWhite", - "HeaderAttr": "AttrBold", - "HeaderBg": "BgBlue", - "HeaderFg": "FgWhite", - "HeaderDelimiterAttr": "AttrDim", - "HeaderDelimiterBg": "BgBlue", - "HeaderDelimiterFg": "FgWhite", - "HeaderSortKeyAttr": "AttrUnderline", - "HeaderGroupKeyAttr": "AttrReverse", - "RawQueryAttr": "AttrDim", - "RawQueryBg": "BgBlack", - "RawQueryFg": "FgCyan" - } - } - }, - "Server": { - "SSHBindAddress": "0.0.0.0", - "HostKeyFile": "cache/ssh_host_key", - "HostKeyBits": 2048, - "MapreduceLogFormat": "default", - "MaxConcurrentCats": 2, - "MaxConcurrentTails": 50, - "MaxConnections": 50, - "MaxLineLength": 1048576, - "Permissions": { - "Default": [ - "readfiles:^/.*$" - ], - "Users": { - "paul": [ - "readfiles:^/.*$" - ], - "pbuetow": [ - "readfiles:^/.*$" - ], - "jamesblake": [ - "readfiles:^/tmp/foo.log$", - "readfiles:^/.*$", - "readfiles:!^/tmp/bar.log$" - ] - } - } - }, - "Common": { - "LogDir": "log", - "Logger": "Fout", - "LogRotation": "Daily", - "CacheDir": "cache", - "SSHPort": 2222, - "LogLevel": "Info" - } -} diff --git a/samples/dtail.schema.json b/samples/dtail.schema.json deleted file mode 100755 index 44fb4c2..0000000 --- a/samples/dtail.schema.json +++ /dev/null @@ -1,532 +0,0 @@ -{ - "$schema": "https://json-schema.org/2019-09/schema", - "description": "Schema for dtail.json", - "definitions": { - "userPermission": { - "type": "array", - "items": { - "type": "string" - } - }, - "userPermissions": { - "type": "object", - "patternProperties": { - "^.*$": { - "$ref": "#/definitions/userPermission" - } - } - }, - "loglevel": { - "type": "string", - "enum": [ - "None", - "Fatal", - "Error", - "Warn", - "Info", - "Default", - "Verbose", - "Debug", - "Devel", - "Trace", - "All" - ] - }, - "logger": { - "type": "string", - "enum": [ - "None", - "Stdout", - "File", - "Fout" - ] - }, - "logrotation": { - "type": "string", - "enum": [ - "Daily", - "Signal" - ] - }, - "color": { - "type": "string", - "enum": [ - "Black", - "Red", - "Green", - "Yellow", - "Blue", - "Magenta", - "Cyan", - "White" - ] - }, - "attribute": { - "type": "string", - "enum": [ - "None", - "Bold", - "Dim", - "Italic", - "Underline", - "Blink", - "SlowBlink", - "RapidBlink", - "Reverse", - "Hidden" - ] - } - }, - "type": "object", - "additionalProperties": false, - "properties": { - "Client": { - "additionalProperties": false, - "properties": { - "TermColorsEnable": { - "type": "boolean" - }, - "TermColors": { - "type": "object", - "additionalProperties": false, - "properties": { - "Remote": { - "additionalProperties": false, - "properties": { - "DelimiterAttr": { - "$ref": "#/definitions/attribute" - }, - "DelimiterBg": { - "$ref": "#/definitions/color" - }, - "DelimiterFg": { - "$ref": "#/definitions/color" - }, - "RemoteAttr": { - "$ref": "#/definitions/attribute" - }, - "RemoteBg": { - "$ref": "#/definitions/color" - }, - "RemoteFg": { - "$ref": "#/definitions/color" - }, - "CountAttr": { - "$ref": "#/definitions/attribute" - }, - "CountBg": { - "$ref": "#/definitions/color" - }, - "CountFg": { - "$ref": "#/definitions/color" - }, - "HostnameAttr": { - "$ref": "#/definitions/attribute" - }, - "HostnameBg": { - "$ref": "#/definitions/color" - }, - "HostnameFg": { - "$ref": "#/definitions/color" - }, - "IDAttr": { - "$ref": "#/definitions/attribute" - }, - "IDBg": { - "$ref": "#/definitions/color" - }, - "IDFg": { - "$ref": "#/definitions/color" - }, - "StatsOkAttr": { - "$ref": "#/definitions/attribute" - }, - "StatsOkBg": { - "$ref": "#/definitions/color" - }, - "StatsOkFg": { - "$ref": "#/definitions/color" - }, - "StatsWarnAttr": { - "$ref": "#/definitions/attribute" - }, - "StatsWarnBg": { - "$ref": "#/definitions/color" - }, - "StatsWarnFg": { - "$ref": "#/definitions/color" - }, - "TextAttr": { - "$ref": "#/definitions/attribute" - }, - "TextBg": { - "$ref": "#/definitions/color" - }, - "TextFg": { - "$ref": "#/definitions/color" - } - } - }, - "Client": { - "additionalProperties": false, - "properties": { - "DelimiterAttr": { - "$ref": "#/definitions/attribute" - }, - "DelimiterBg": { - "$ref": "#/definitions/color" - }, - "DelimiterFg": { - "$ref": "#/definitions/color" - }, - "ClientAttr": { - "$ref": "#/definitions/attribute" - }, - "ClientBg": { - "$ref": "#/definitions/color" - }, - "ClientFg": { - "$ref": "#/definitions/color" - }, - "HostnameAttr": { - "$ref": "#/definitions/attribute" - }, - "HostnameBg": { - "$ref": "#/definitions/color" - }, - "HostnameFg": { - "$ref": "#/definitions/color" - }, - "TextAttr": { - "$ref": "#/definitions/attribute" - }, - "TextBg": { - "$ref": "#/definitions/color" - }, - "TextFg": { - "$ref": "#/definitions/color" - } - } - }, - "Server": { - "additionalProperties": false, - "properties": { - "DelimiterAttr": { - "#ref": "#/definitions/attribute" - }, - "DelimiterBg": { - "#ref": "#/definitions/color" - }, - "DelimiterFg": { - "#ref": "#/definitions/color" - }, - "ServerAttr": { - "#ref": "#/definitions/attribute" - }, - "ServerBg": { - "#ref": "#/definitions/color" - }, - "ServerFg": { - "#ref": "#/definitions/color" - }, - "HostnameAttr": { - "#ref": "#/definitions/attribute" - }, - "HostnameBg": { - "#ref": "#/definitions/color" - }, - "HostnameFg": { - "#ref": "#/definitions/color" - }, - "TextAttr": { - "#ref": "#/definitions/attribute" - }, - "TextBg": { - "#ref": "#/definitions/color" - }, - "TextFg": { - "#ref": "#/definitions/color" - }, - "KeyExchanges": { - "type": "array", - "items": { - "type": "string" - } - }, - "Ciphers": { - "type": "array", - "items": { - "type": "string" - } - }, - "MACs": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "Common": { - "additionalProperties": false, - "properties": { - "SeverityErrorAttr": { - "#ref": "#/definitions/attribute" - }, - "SeverityErrorBg": { - "#ref": "#/definitions/color" - }, - "SeverityErrorFg": { - "#ref": "#/definitions/color" - }, - "SeverityFatalAttr": { - "#ref": "#/definitions/attribute" - }, - "SeverityFatalBg": { - "#ref": "#/definitions/color" - }, - "SeverityFatalFg": { - "#ref": "#/definitions/color" - }, - "SeverityWarnAttr": { - "#ref": "#/definitions/attribute" - }, - "SeverityWarnBg": { - "#ref": "#/definitions/color" - }, - "SeverityWarnFg": { - "#ref": "#/definitions/color" - } - } - }, - "MaprTable": { - "additionalProperties": false, - "properties": { - "DataAttr": { - "#ref": "#/definitions/attribute" - }, - "DataBg": { - "#ref": "#/definitions/color" - }, - "DataFg": { - "#ref": "#/definitions/color" - }, - "DelimiterAttr": { - "#ref": "#/definitions/attribute" - }, - "DelimiterBg": { - "#ref": "#/definitions/color" - }, - "DelimiterFg": { - "#ref": "#/definitions/color" - }, - "HeaderAttr": { - "#ref": "#/definitions/attribute" - }, - "HeaderBg": { - "#ref": "#/definitions/color" - }, - "HeaderFg": { - "#ref": "#/definitions/color" - }, - "HeaderDelimiterAttr": { - "#ref": "#/definitions/attribute" - }, - "HeaderDelimiterBg": { - "#ref": "#/definitions/color" - }, - "HeaderDelimiterFg": { - "#ref": "#/definitions/color" - }, - "HeaderSortKeyAttr": { - "#ref": "#/definitions/attribute" - }, - "HeaderGroupKeyAttr": { - "#ref": "#/definitions/attribute" - }, - "RawQueryAttr": { - "#ref": "#/definitions/attribute" - }, - "RawQueryBg": { - "#ref": "#/definitions/color" - }, - "RawQueryFg": { - "#ref": "#/definitions/color" - } - } - } - } - } - } - }, - "Server": { - "additionalProperties": false, - "properties": { - "SSHBindAddress": { - "type": "string" - }, - "HostKeyFile": { - "type": "string" - }, - "HostKeyBits": { - "type": "integer", - "minimum": 2048 - }, - "MapreduceLogFormat": { - "type": "string" - }, - "MaxConcurrentCats": { - "type": "integer", - "minimum": 1, - "maximum": 20 - }, - "MaxConcurrentTails": { - "type": "integer", - "minimum": 1, - "maximum": 200 - }, - "MaxConnections": { - "type": "integer", - "minimum": 1, - "maximum": 200 - }, - "MaxLineLength": { - "type": "integer", - "minimum": 1024, - "maximum": 10240000 - }, - "Permissions": { - "type": "object", - "additionalProperties": true, - "patternProperties": { - "^Default$": { - "$ref": "#/definitions/userPermission" - }, - "^Users$": { - "$ref": "#/definitions/userPermissions" - } - } - }, - "Schedule": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "Name": { - "type": "string" - }, - "Enable": { - "type": "boolean" - }, - "AllowFrom": { - "type": "array", - "items": { - "type": "string" - } - }, - "Servers": { - "type": "array", - "items": { - "type": "string" - } - }, - "TimeRange": { - "type": "array", - "items": [ - { - "type": "integer" - }, - { - "type": "integer" - } - ] - }, - "Files": { - "type": "string" - }, - "Outfile": { - "type": "string" - }, - "Query": { - "type": "string" - } - } - } - }, - "Continuous": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "Name": { - "type": "string" - }, - "Enable": { - "type": "boolean" - }, - "AllowFrom": { - "type": "array", - "items": { - "type": "string" - } - }, - "Servers": { - "type": "array", - "items": { - "type": "string" - } - }, - "RestartOnDayChange": { - "type": "boolean" - }, - "Files": { - "type": "string" - }, - "Outfile": { - "type": "string" - }, - "Query": { - "type": "string" - } - } - } - } - } - }, - "Common": { - "additionalProperties": false, - "properties": { - "LogDir": { - "type": "string" - }, - "Logger": { - "#ref": "#/definitions/logger" - }, - "LogLevel": { - "#ref": "#/definitions/loglevel" - }, - "LogRotation": { - "#ref": "#/definitions/logrotation" - }, - "CacheDir": { - "type": "string" - }, - "SSHPort": { - "type": "integer", - "minimum": 2, - "maximum": 16000 - }, - "ExperimentalFeaturesEnable": { - "type": "boolean" - } - } - } - }, - "required": [ - "Client", - "Server", - "Common" - ] -} diff --git a/samples/update_key_cache.sh.sample b/samples/update_key_cache.sh.sample deleted file mode 100644 index 9817f04..0000000 --- a/samples/update_key_cache.sh.sample +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -declare -r CACHEDIR=/var/run/dserver/cache -declare -r DSERVER_USER=dserver - -echo "Updating SSH key cache" - -ls /home/ | while read remoteuser; do - keysfile=/home/$remoteuser/.ssh/authorized_keys - - if [ -f $keysfile ]; then - cachefile=$CACHEDIR/$remoteuser.authorized_keys - echo "Caching $keysfile -> $cachefile" - - cp $keysfile $cachefile - chown $DSERVER_USER $cachefile - chmod 600 $cachefile - fi -done - -# Cleanup obsolete public SSH keys -find $CACHEDIR -name \*.authorized_keys -type f | -while read cachefile; do - remoteuser=$(basename $cachefile | cut -d. -f1) - keysfile=/home/$remoteuser/.ssh/authorized_keys - - if [ ! -f $keysfile ]; then - echo "Deleting obsolete cache file $cachefile" - rm $cachefile - fi -done - -echo "All set..." -- cgit v1.2.3 From 70a36830b83e50989101bed7840e648ce25f9134 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 7 Sep 2023 15:28:37 +0300 Subject: fix conflict --- go.mod | 9 +++------ go.sum | 14 +++++++------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 5c9cf64..fce2a35 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,7 @@ go 1.20 require ( github.com/DataDog/zstd v1.5.5 - golang.org/x/crypto v0.9.0 -) - -require ( - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.8.0 // indirect + golang.org/x/crypto v0.10.0 + golang.org/x/sys v0.9.0 // indirect + golang.org/x/term v0.9.0 // indirect ) diff --git a/go.sum b/go.sum index 5e3fe8c..90a6d4b 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ -github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -- cgit v1.2.3 From c1aad31d274f646bf827f84d79c874ac04f60e91 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 11:33:18 +0100 Subject: gofmt permission file headers --- internal/io/fs/permissions/permission.go | 1 + internal/io/fs/permissions/permission_linuxacl.go | 1 + internal/io/fs/permissions/permission_test.go | 1 + 3 files changed, 3 insertions(+) diff --git a/internal/io/fs/permissions/permission.go b/internal/io/fs/permissions/permission.go index d621c09..aaab9e7 100644 --- a/internal/io/fs/permissions/permission.go +++ b/internal/io/fs/permissions/permission.go @@ -1,3 +1,4 @@ +//go:build !linuxacl // +build !linuxacl package permissions diff --git a/internal/io/fs/permissions/permission_linuxacl.go b/internal/io/fs/permissions/permission_linuxacl.go index 904b90f..bfac7e2 100644 --- a/internal/io/fs/permissions/permission_linuxacl.go +++ b/internal/io/fs/permissions/permission_linuxacl.go @@ -1,3 +1,4 @@ +//go:build linuxacl // +build linuxacl package permissions diff --git a/internal/io/fs/permissions/permission_test.go b/internal/io/fs/permissions/permission_test.go index c0ef038..e28c67b 100644 --- a/internal/io/fs/permissions/permission_test.go +++ b/internal/io/fs/permissions/permission_test.go @@ -1,3 +1,4 @@ +//go:build linuxacl // +build linuxacl package permissions -- cgit v1.2.3 From 3ba8778d1d25be0a1cf38f69adae9e09222d6e61 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 11:36:43 +0100 Subject: Fix typos. --- internal/clients/baseclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/clients/baseclient.go b/internal/clients/baseclient.go index 764d53e..3025f72 100644 --- a/internal/clients/baseclient.go +++ b/internal/clients/baseclient.go @@ -78,7 +78,7 @@ func (c *baseClient) Start(ctx context.Context, statsCh <-chan string) (status i // Periodically check for unknown hosts, and ask the user whether to trust them or not. go c.hostKeyCallback.PromptAddHosts(ctx) } - // Print client stats every time something on statsCh is recieved. + // Print client stats every time something on statsCh is received. go c.stats.Start(ctx, c.throttleCh, statsCh, c.Args.Quiet) var wg sync.WaitGroup -- cgit v1.2.3 From e50be6690b1a0efa8ea7880237c3b97165b56800 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 11:42:37 +0100 Subject: document Outfile - tidy mods --- internal/mapr/query.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/mapr/query.go b/internal/mapr/query.go index 4eeb7b6..247cdaf 100644 --- a/internal/mapr/query.go +++ b/internal/mapr/query.go @@ -13,6 +13,7 @@ const ( unexpectedEnd string = "Unexpected end of query" ) +// Outfile represents the output file of a mapreduce query. type Outfile struct { FilePath string AppendMode bool -- cgit v1.2.3 From 2600d7a02bae5cd246c7520df1d7da3190c99d6d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 11:45:40 +0100 Subject: This is not a snapshot release anymore --- internal/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version/version.go b/internal/version/version.go index c60c04c..90073b5 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -15,7 +15,7 @@ const ( // Version of DTail. Version string = "4.2.0" // Additional information for DTail - Additional string = "SNAPSHOT release" + Additional string = "Have a lot of fun!" ) // String representation of the DTail version. -- cgit v1.2.3 From dcced5746b735dbe171c88f7e9106b40cf31c995 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Fri, 15 Jul 2022 12:25:31 +0100 Subject: Refactor - reduce code complexity --- internal/mapr/groupsetresult.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/mapr/groupsetresult.go b/internal/mapr/groupsetresult.go index 9c3c134..d01a3c9 100644 --- a/internal/mapr/groupsetresult.go +++ b/internal/mapr/groupsetresult.go @@ -221,14 +221,7 @@ func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, fd *os.Fi lastColumn := len(query.Select) - 1 if writeHeader { - for i, sc := range query.Select { - fd.WriteString(sc.FieldStorage) - if i == lastColumn { - continue - } - fd.WriteString(protocol.CSVDelimiter) - } - fd.WriteString("\n") + g.resultWriteUnformattedHeader(query, fd, lastColumn) } // And now write the data @@ -256,3 +249,14 @@ func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, fd *os.Fi return nil } + +func (g *GroupSet) resultWriteUnformattedHeader(query *Query, fd *os.File, lastColumn int) { + for i, sc := range query.Select { + fd.WriteString(sc.FieldStorage) + if i == lastColumn { + continue + } + fd.WriteString(protocol.CSVDelimiter) + } + fd.WriteString("\n") +} -- cgit v1.2.3 From cc17406ae125f40c4100afbedcd8c40ba319cfa8 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 18 Jul 2022 10:20:59 +0100 Subject: Include example for "outfile append". --- doc/examples.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/examples.md b/doc/examples.md index 56744f5..26ce002 100644 --- a/doc/examples.md +++ b/doc/examples.md @@ -65,6 +65,14 @@ Here is another example: ![dtail-map](dtail-map2.gif "Tail mapreduce example 2") +You can also continuously append the results to a CSV file by adding `outfile append filename.csv` to the query: + +```shell +% dtail --servers serverlist.txt \ + --files '/var/log/dserver/*.log' \ + --query 'from STATS select ... outfile append result.csv' +``` + ## How to use `dcat` The following example demonstrates how to cat files (display the full content of the files) of multiple servers at once. -- cgit v1.2.3 From 6d836951fc1d6f7009dbefda487cc9d88dd56594 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Feb 2023 18:02:47 +0000 Subject: Update dependencies --- internal/version/version.go | 2 +- inventory.yaml | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/version/version.go b/internal/version/version.go index 90073b5..15ea50f 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ const ( // Name of DTail. Name string = "DTail" // Version of DTail. - Version string = "4.2.0" + Version string = "4.3.0" // Additional information for DTail Additional string = "Have a lot of fun!" ) diff --git a/inventory.yaml b/inventory.yaml index 3750fc0..c75016a 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -1,5 +1,7 @@ -component_name: dtail -owning team: Paul Buetow -component_usage: OPENSOURCE -application_purpose: tool -development_strategy: opensource +component: dtail +project_name: dtail +full_path: 'https://github.com/mimecast/dtail.git' +component_usage: Open Source +dev_language: + - Go +application_type: Tooling -- cgit v1.2.3 From 02c6848270ee5ee025ea80b459456e96f31d6eb8 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 21 Jun 2023 13:04:43 +0300 Subject: move new SSH Server crypto configs to the correct Server section in the schema --- examples/dtail.schema.json | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/dtail.schema.json b/examples/dtail.schema.json index 44fb4c2..d13b133 100755 --- a/examples/dtail.schema.json +++ b/examples/dtail.schema.json @@ -246,24 +246,6 @@ }, "TextFg": { "#ref": "#/definitions/color" - }, - "KeyExchanges": { - "type": "array", - "items": { - "type": "string" - } - }, - "Ciphers": { - "type": "array", - "items": { - "type": "string" - } - }, - "MACs": { - "type": "array", - "items": { - "type": "string" - } } } }, @@ -365,6 +347,24 @@ "SSHBindAddress": { "type": "string" }, + "KeyExchanges": { + "type": "array", + "items": { + "type": "string" + } + }, + "Ciphers": { + "type": "array", + "items": { + "type": "string" + } + }, + "MACs": { + "type": "array", + "items": { + "type": "string" + } + }, "HostKeyFile": { "type": "string" }, -- cgit v1.2.3 From 9d1ce0304e5213e66a0d916a2a44eb60b73c7197 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Wed, 21 Jun 2023 10:56:41 +0000 Subject: DTail: Restrict SSH MAC algorithms allowed - Update of few dependencies --- internal/mapr/logformat/parser.go | 1 - internal/version/version.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go index d6aac8c..b980e3d 100644 --- a/internal/mapr/logformat/parser.go +++ b/internal/mapr/logformat/parser.go @@ -17,7 +17,6 @@ var ErrIgnoreFields error = errors.New("Ignore this field set") // Parser is used to parse the mapreduce information from the server log files. type Parser struct { hostname string - logFormatName string makeFieldsFunc reflect.Value makeFieldsReceiver reflect.Value timeZoneName string diff --git a/internal/version/version.go b/internal/version/version.go index 15ea50f..90073b5 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ const ( // Name of DTail. Name string = "DTail" // Version of DTail. - Version string = "4.3.0" + Version string = "4.2.0" // Additional information for DTail Additional string = "Have a lot of fun!" ) -- cgit v1.2.3 From 63fa5de929f7c0cbd68db9bc0761e912c97c253b Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 13:57:37 +0300 Subject: Refactor logformats so that they don't use reflection anymore. --- internal/mapr/logformat/default.go | 17 +++++++-- internal/mapr/logformat/default_test.go | 4 +++ internal/mapr/logformat/generic.go | 15 ++++++-- internal/mapr/logformat/generickv.go | 15 ++++++-- internal/mapr/logformat/parser.go | 61 +++++++-------------------------- internal/mapr/server/aggregate.go | 2 +- 6 files changed, 58 insertions(+), 56 deletions(-) diff --git a/internal/mapr/logformat/default.go b/internal/mapr/logformat/default.go index a44b49a..a499bc5 100644 --- a/internal/mapr/logformat/default.go +++ b/internal/mapr/logformat/default.go @@ -7,8 +7,21 @@ import ( "github.com/mimecast/dtail/internal/protocol" ) -// MakeFieldsDEFAULT is the default DTail log file key-value parser. -func (p *Parser) MakeFieldsDEFAULT(maprLine string) (map[string]string, error) { +type defaultParser struct { + hostname string + timeZoneName string + timeZoneOffset string +} + +func newDefaultParser(hostname, timeZoneName string, timeZoneOffset int) (*defaultParser, error) { + return &defaultParser{ + hostname: hostname, + timeZoneName: timeZoneName, + timeZoneOffset: fmt.Sprintf("%d", timeZoneOffset), + }, nil +} + +func (p *defaultParser) MakeFields(maprLine string) (map[string]string, error) { splitted := strings.Split(maprLine, protocol.FieldDelimiter) if len(splitted) < 11 || !strings.HasPrefix(splitted[9], "MAPREDUCE:") || diff --git a/internal/mapr/logformat/default_test.go b/internal/mapr/logformat/default_test.go index 28e1acc..4eae81b 100644 --- a/internal/mapr/logformat/default_test.go +++ b/internal/mapr/logformat/default_test.go @@ -87,6 +87,10 @@ func TestDefaultLogFormat(t *testing.T) { } fields, err := parser.MakeFields("foozoo=bar|bazbay") + if err != nil && err != ErrIgnoreFields { + t.Errorf(err.Error()) + } + if _, ok := fields["foo"]; ok { t.Errorf("Expected fiending field 'foo', but found it\n") } diff --git a/internal/mapr/logformat/generic.go b/internal/mapr/logformat/generic.go index 14ac2a9..32d9b4a 100644 --- a/internal/mapr/logformat/generic.go +++ b/internal/mapr/logformat/generic.go @@ -1,7 +1,18 @@ package logformat -// MakeFieldsGENERIC is the generic log line parser. -func (p *Parser) MakeFieldsGENERIC(maprLine string) (map[string]string, error) { +type genericParser struct { + defaultParser +} + +func newGenericParser(hostname, timeZoneName string, timeZoneOffset int) (*genericParser, error) { + defaultParser, err := newDefaultParser(hostname, timeZoneName, timeZoneOffset) + if err != nil { + return &genericParser{}, err + } + return &genericParser{defaultParser: *defaultParser}, nil +} + +func (p *genericParser) MakeFields(maprLine string) (map[string]string, error) { fields := make(map[string]string, 3) fields["*"] = "*" diff --git a/internal/mapr/logformat/generickv.go b/internal/mapr/logformat/generickv.go index 3452e97..9c3de92 100644 --- a/internal/mapr/logformat/generickv.go +++ b/internal/mapr/logformat/generickv.go @@ -6,8 +6,19 @@ import ( "github.com/mimecast/dtail/internal/protocol" ) -// MakeFieldsGENERIGKV is the generic key-value logfile parser. -func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error) { +type genericKVParser struct { + defaultParser +} + +func newGenericKVParser(hostname, timeZoneName string, timeZoneOffset int) (*genericKVParser, error) { + defaultParser, err := newDefaultParser(hostname, timeZoneName, timeZoneOffset) + if err != nil { + return &genericKVParser{}, err + } + return &genericKVParser{defaultParser: *defaultParser}, nil +} + +func (p *genericKVParser) MakeFields(maprLine string) (map[string]string, error) { splitted := strings.Split(maprLine, protocol.FieldDelimiter) fields := make(map[string]string, len(splitted)) diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go index b980e3d..2d9634d 100644 --- a/internal/mapr/logformat/parser.go +++ b/internal/mapr/logformat/parser.go @@ -2,9 +2,6 @@ package logformat import ( "errors" - "fmt" - "reflect" - "strings" "time" "github.com/mimecast/dtail/internal/config" @@ -15,61 +12,27 @@ import ( var ErrIgnoreFields error = errors.New("Ignore this field set") // Parser is used to parse the mapreduce information from the server log files. -type Parser struct { - hostname string - makeFieldsFunc reflect.Value - makeFieldsReceiver reflect.Value - timeZoneName string - timeZoneOffset string +type Parser interface { + // MakeFields creates a field map from an input log line. + MakeFields(string) (map[string]string, error) } // NewParser returns a new log parser. -func NewParser(logFormatName string, query *mapr.Query) (*Parser, error) { +func NewParser(logFormatName string, query *mapr.Query) (Parser, error) { hostname, err := config.Hostname() if err != nil { return nil, err } now := time.Now() - zone, offset := now.Zone() + timeZoneName, timeZoneOffset := now.Zone() - p := Parser{ - hostname: hostname, - timeZoneName: zone, - timeZoneOffset: fmt.Sprintf("%d", offset), - } - - err = p.reflectLogFormat(logFormatName) - if err != nil { - return nil, err - } - return &p, nil -} - -// The aim of this is that everyone can plug in their own mapr log format -// parsing method to DTail. Just add a method MakeFieldsMODULENAME to type -// Parser. Whereas MODULENAME must be a upeprcase string. -func (p *Parser) reflectLogFormat(logFormatName string) error { - methodName := fmt.Sprintf("MakeFields%s", strings.ToUpper(logFormatName)) - rt := reflect.TypeOf(p) - method, ok := rt.MethodByName(methodName) - if !ok { - return errors.New("No such mapr log format module: " + methodName) - } - - p.makeFieldsFunc = method.Func - p.makeFieldsReceiver = reflect.ValueOf(p) - return nil -} + switch logFormatName { + case "generic": + return newGenericParser(hostname, timeZoneName, timeZoneOffset) + case "generickv": + return newGenericKVParser(hostname, timeZoneName, timeZoneOffset) + default: + return newDefaultParser(hostname, timeZoneName, timeZoneOffset) -// MakeFields is for returning the fields from a given log line. -func (p *Parser) MakeFields(maprLine string) (fields map[string]string, err error) { - inputValues := []reflect.Value{p.makeFieldsReceiver, reflect.ValueOf(maprLine)} - returnValues := p.makeFieldsFunc.Call(inputValues) - errInterface := returnValues[1].Interface() - if errInterface == nil { - fields, err = returnValues[0].Interface().(map[string]string), nil - return } - fields, err = returnValues[0].Interface().(map[string]string), errInterface.(error) - return } diff --git a/internal/mapr/server/aggregate.go b/internal/mapr/server/aggregate.go index ed32f8f..4f14751 100644 --- a/internal/mapr/server/aggregate.go +++ b/internal/mapr/server/aggregate.go @@ -27,7 +27,7 @@ type Aggregate struct { // The mapr query query *mapr.Query // The mapr log format parser - parser *logformat.Parser + parser logformat.Parser } // NewAggregate return a new server side aggregator. -- cgit v1.2.3 From 54914d67e98116ec93ee324d28db4a772be02bc2 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 14:04:20 +0300 Subject: Add mimecast parser stub. Open source version of DTail won't be with it. --- internal/mapr/logformat/mimecast.go | 16 ++++++++++++++++ internal/mapr/logformat/parser.go | 2 ++ 2 files changed, 18 insertions(+) create mode 100644 internal/mapr/logformat/mimecast.go diff --git a/internal/mapr/logformat/mimecast.go b/internal/mapr/logformat/mimecast.go new file mode 100644 index 0000000..bc6e30a --- /dev/null +++ b/internal/mapr/logformat/mimecast.go @@ -0,0 +1,16 @@ +package logformat + +import "errors" + +// ErrMimecastNotAvailable is thrown in the open source version of DTail +var ErrMimecastNotAvailable error = errors.New("The mimecast logformat is not available in this build of DTail") + +type mimecastParser struct{} + +func newMimecastParser(hostname, timeZoneName string, timeZoneOffset int) (*mimecastParser, error) { + return &mimecastParser{}, ErrMimecastNotAvailable +} + +func (p *mimecastParser) MakeFields(maprLine string) (map[string]string, error) { + return nil, ErrMimecastNotAvailable +} diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go index 2d9634d..aa6416c 100644 --- a/internal/mapr/logformat/parser.go +++ b/internal/mapr/logformat/parser.go @@ -31,6 +31,8 @@ func NewParser(logFormatName string, query *mapr.Query) (Parser, error) { return newGenericParser(hostname, timeZoneName, timeZoneOffset) case "generickv": return newGenericKVParser(hostname, timeZoneName, timeZoneOffset) + case "mimecast": + return newMimecastParser(hostname, timeZoneName, timeZoneOffset) default: return newDefaultParser(hostname, timeZoneName, timeZoneOffset) -- cgit v1.2.3 From bf39452c1f9a06d9f4e6eb3a06a23068a2451ca5 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 14:13:48 +0300 Subject: Update creating your own logformat docs, to reflect the recent changes. --- doc/logformats.md | 61 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/doc/logformats.md b/doc/logformats.md index c3f0c63..9d4e55d 100644 --- a/doc/logformats.md +++ b/doc/logformats.md @@ -21,15 +21,26 @@ By default, DTail will use the `default` log format. You can override the log fo % dmap --files /var/log/example.log --query 'from EXAMPLE select ....queryhere.... logformat generickv' ``` -Alternatively, you can override the default log format with `MapreduceLogFormat` in the Server section of `dtail.json`. +You can override the default log format with `MapreduceLogFormat` in the Server section of `dtail.json`. ## Under the hood: generickv As an example, let's have a look at the `generickv` log format's implementation. It's located at `internal/mapr/logformat/generickv.go`: -```shell -// MakeFieldsGENERIGKV is the generic key-value logfile parser. -func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error) { +```go +type genericKVParser struct { + defaultParser +} + +func newGenericKVParser(hostname, timeZoneName string, timeZoneOffset int) (*genericKVParser, error) { + defaultParser, err := newDefaultParser(hostname, timeZoneName, timeZoneOffset) + if err != nil { + return &genericKVParser{}, err + } + return &genericKVParser{defaultParser: *defaultParser}, nil +} + +func (p *genericKVParser) MakeFields(maprLine string) (map[string]string, error) { splitted := strings.Split(maprLine, protocol.FieldDelimiter) fields := make(map[string]string, len(splitted)) @@ -44,7 +55,7 @@ func (p *Parser) MakeFieldsGENERIGKV(maprLine string) (map[string]string, error) for _, kv := range splitted[0:] { keyAndValue := strings.SplitN(kv, "=", 2) if len(keyAndValue) != 2 { - // dlog.Common.Debug("Unable to parse key-value token, ignoring it", kv) + //dlog.Common.Debug("Unable to parse key-value token, ignoring it", kv) continue } fields[keyAndValue[0]] = keyAndValue[1] @@ -101,26 +112,40 @@ These variables may only exist in the DTail default log format (see `internal/ma * `$pid` - DTail server process ID * `$uptime` - DTail server uptime -## Implementing your own log format +## Implementing your own log format `Foo` -All what needs to be done is to place your own implementation into the `logformat` source directory. As a template, you can copy an existing format ... +What needs to be done is to place your own implementation into the `logformat` source directory. As a template, you can copy an existing format ... ```shell -% cp internal/mapr/logformat/generic.go internal/mapr/logformat/yourcustomformat.go +% cp internal/mapr/logformat/generic.go internal/mapr/logformat/foo.go ``` -... and replace `GENERIGKV` with your format's name in capital letters (the method name string is used by DTail to reflect the log format parser method, so it is important to name it correctly): +... and replace `generic` ` with your format's name `foo`: + +```go +package logformat + +type fooParser struct { + defaultParser +} + +func newFooParser(hostname, timeZoneName string, timeZoneOffset int) (*fooParser, error) { + defaultParser, err := newDefaultParser(hostname, timeZoneName, timeZoneOffset) + if err != nil { + return &fooParser{}, err + } + return &fooParser{defaultParser: *defaultParser}, nil +} + +func (p *fooParser) MakeFields(maprLine string) (map[string]string, error) { + fields := make(map[string]string, 3) + + .. + + .. -```shell -// MakeFieldsCUSTOMLOGFORMAT is your own custom log format. -func (p *Parser) MakeFieldsCUSTOMLOGFORMAT(maprLine string) (map[string]string, error) { - // .. Your own format implementation goes here - // .. you can parse maprLine and store values into the fields map. -.. -. -. return fields, nil } ``` -Once done, recompile DTail. DTail now understands `... logformat customlogformat` (see "Seleting a log format" above). +Once done, recompile DTail. DTail now understands `... logformat foo` (see "Seleting a log format" above). -- cgit v1.2.3 From d427574824a1a4b6a5a0e15a3696f3df1c698424 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 14:39:21 +0300 Subject: Add `custom1` and `custom2` log formats. --- doc/logformats.md | 2 ++ internal/mapr/logformat/custom1.go | 16 ++++++++++++++++ internal/mapr/logformat/custom2.go | 16 ++++++++++++++++ internal/mapr/logformat/parser.go | 16 ++++++++++++++-- 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 internal/mapr/logformat/custom1.go create mode 100644 internal/mapr/logformat/custom2.go diff --git a/doc/logformats.md b/doc/logformats.md index 9d4e55d..839b050 100644 --- a/doc/logformats.md +++ b/doc/logformats.md @@ -148,4 +148,6 @@ func (p *fooParser) MakeFields(maprLine string) (map[string]string, error) { } ``` +Next, `NewParser(...)` in `internal/mapr/logformat/parser.go` needs to be extended, so that the new log format is part of the switch statement. If you don't want to edit `parser.go` then you could instead use `custom1` or `custom2` log formats, there are ready templates available in the `logformat` package. + Once done, recompile DTail. DTail now understands `... logformat foo` (see "Seleting a log format" above). diff --git a/internal/mapr/logformat/custom1.go b/internal/mapr/logformat/custom1.go new file mode 100644 index 0000000..7229f3e --- /dev/null +++ b/internal/mapr/logformat/custom1.go @@ -0,0 +1,16 @@ +package logformat + +import "errors" + +var ErrCustom1NotImplemented error = errors.New("custom1 log format is not implemented") + +// Template for creating a custom log format. +type custom1Parser struct{} + +func newCustom1Parser(hostname, timeZoneName string, timeZoneOffset int) (*custom1Parser, error) { + return &custom1Parser{}, ErrCustom1NotImplemented +} + +func (p *custom1Parser) MakeFields(maprLine string) (map[string]string, error) { + return nil, ErrCustom1NotImplemented +} diff --git a/internal/mapr/logformat/custom2.go b/internal/mapr/logformat/custom2.go new file mode 100644 index 0000000..262c721 --- /dev/null +++ b/internal/mapr/logformat/custom2.go @@ -0,0 +1,16 @@ +package logformat + +import "errors" + +var ErrCustom2NotImplemented error = errors.New("custom2 log format is not implemented") + +// Template for creating a custom log format. +type custom2Parser struct{} + +func newCustom2Parser(hostname, timeZoneName string, timeZoneOffset int) (*custom2Parser, error) { + return &custom2Parser{}, ErrCustom2NotImplemented +} + +func (p *custom2Parser) MakeFields(maprLine string) (map[string]string, error) { + return nil, ErrCustom2NotImplemented +} diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go index aa6416c..24290ef 100644 --- a/internal/mapr/logformat/parser.go +++ b/internal/mapr/logformat/parser.go @@ -2,6 +2,7 @@ package logformat import ( "errors" + "fmt" "time" "github.com/mimecast/dtail/internal/config" @@ -26,6 +27,7 @@ func NewParser(logFormatName string, query *mapr.Query) (Parser, error) { now := time.Now() timeZoneName, timeZoneOffset := now.Zone() + // Extend this for adding more log formats! switch logFormatName { case "generic": return newGenericParser(hostname, timeZoneName, timeZoneOffset) @@ -33,8 +35,18 @@ func NewParser(logFormatName string, query *mapr.Query) (Parser, error) { return newGenericKVParser(hostname, timeZoneName, timeZoneOffset) case "mimecast": return newMimecastParser(hostname, timeZoneName, timeZoneOffset) - default: + case "default": return newDefaultParser(hostname, timeZoneName, timeZoneOffset) - + case "custom1": + return newCustom1Parser(hostname, timeZoneName, timeZoneOffset) + case "custom2": + return newCustom2Parser(hostname, timeZoneName, timeZoneOffset) + default: + p, err := newDefaultParser(hostname, timeZoneName, timeZoneOffset) + if err != nil { + return p, fmt.Errorf("No '%s' mapr log format and problem creating default one: %v", + logFormatName, err) + } + return p, fmt.Errorf("No '%s' mapr log format", logFormatName) } } -- cgit v1.2.3 From 7a5ffe7ddcde145c567ed424e600390122257551 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 15:35:37 +0300 Subject: Add CSV unit test --- internal/mapr/logformat/csv_test.go | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 internal/mapr/logformat/csv_test.go diff --git a/internal/mapr/logformat/csv_test.go b/internal/mapr/logformat/csv_test.go new file mode 100644 index 0000000..1baf032 --- /dev/null +++ b/internal/mapr/logformat/csv_test.go @@ -0,0 +1,54 @@ +package logformat + +import ( + "strings" + "testing" + + "github.com/mimecast/dtail/internal/protocol" +) + +func TestCSVLogFormat(t *testing.T) { + parser, err := NewParser("csv", nil) + if err != nil { + t.Errorf("Unable to create parser: %s", err.Error()) + } + + headers := []string{"name", "last_name", "color", "profession", "employee_number"} + dataLine1 := []string{"Paul", "Buetow", "Orange", "Site Reliability Engineer", "4242"} + dataLine2 := []string{"Peter", "Bauer", "Black", "CEO", "1"} + + inputs := []string{ + strings.Join(headers, protocol.CSVDelimiter), + strings.Join(dataLine1, protocol.CSVDelimiter), + strings.Join(dataLine2, protocol.CSVDelimiter), + } + + // First line is the header! + if _, err := parser.MakeFields(inputs[0]); err != ErrIgnoreFields { + t.Errorf("Unable to parse the CSV header") + } + + // First data line + fields, err := parser.MakeFields(inputs[1]) + if err != nil { + t.Errorf("Unable to parse first CSV data line: %s", err.Error()) + } + if val := fields["name"]; val != "Paul" { + t.Errorf("Expected 'name' to be 'Paul' but got '%s'", val) + } + if val := fields["employee_number"]; val != "4242" { + t.Errorf("Expected 'employee_number' to be '4242' but got '%s'", val) + } + + // Second data line + fields, err = parser.MakeFields(inputs[2]) + if err != nil { + t.Errorf("Unable to parse first CSV data line: %s", err.Error()) + } + if val := fields["last_name"]; val != "Bauer" { + t.Errorf("Expected 'last_name' to be 'Bauer' but got '%s'", val) + } + if val := fields["color"]; val != "Black" { + t.Errorf("Expected 'color' to be 'Black' but got '%s'", val) + } +} -- cgit v1.2.3 From 360f67bf536372cb6a78fe35c15ba6128fda290b Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 15:36:35 +0300 Subject: add CSV to parser --- internal/mapr/logformat/parser.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go index 24290ef..9888932 100644 --- a/internal/mapr/logformat/parser.go +++ b/internal/mapr/logformat/parser.go @@ -33,6 +33,8 @@ func NewParser(logFormatName string, query *mapr.Query) (Parser, error) { return newGenericParser(hostname, timeZoneName, timeZoneOffset) case "generickv": return newGenericKVParser(hostname, timeZoneName, timeZoneOffset) + case "csv": + return newCSVParser(hostname, timeZoneName, timeZoneOffset) case "mimecast": return newMimecastParser(hostname, timeZoneName, timeZoneOffset) case "default": -- cgit v1.2.3 From 9c77304550d65b8e7c2b724b991eef0dbc13694a Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 16:38:23 +0300 Subject: Can quote fields in select conditions, e.g. select `count($foo)`, .. --- internal/mapr/query.go | 10 ++++++++-- internal/mapr/query_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ internal/mapr/selectcondition.go | 19 ++++++++++--------- internal/mapr/token.go | 10 ++++++---- 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/internal/mapr/query.go b/internal/mapr/query.go index 247cdaf..ddcbc90 100644 --- a/internal/mapr/query.go +++ b/internal/mapr/query.go @@ -73,6 +73,13 @@ func NewQuery(queryStr string) (*Query, error) { Interval: time.Second * 5, Limit: -1, } + + // If log format is CSV, then use "." as the table. It means, that + // we don't do any file filtering, we process all lines of the CSV. + if q.LogFormat == "csv" { + q.Table = "." + } + return &q, q.parse(tokens) } @@ -87,8 +94,7 @@ func (q *Query) Has(what string) bool { } func (q *Query) parse(tokens []token) error { - tokens, err := q.parseTokens(tokens) - if err != nil { + if _, err := q.parseTokens(tokens); err != nil { return err } diff --git a/internal/mapr/query_test.go b/internal/mapr/query_test.go index f03ccba..f37b8d4 100644 --- a/internal/mapr/query_test.go +++ b/internal/mapr/query_test.go @@ -252,3 +252,43 @@ func TestParseQueryDeep(t *testing.T) { } } } + +func TestQuotedSelectCondition(t *testing.T) { + queryStr := "select `count($foo)`, foo, $foo, count($foo) logformat csv" + + q, err := NewQuery(queryStr) + if err != nil { + t.Errorf("Query parse error: %s\n%v: %v", queryStr, q, err) + } + if len(q.Select) != 4 { + t.Errorf("Expected three elements in 'select' clause but got '%v': %s\n%v", + q.Select, queryStr, q) + } + + if q.Select[0].Field != "count($foo)" { + t.Errorf("Expected 'num($foo)' as first element in 'select' clause but got '%v': %s\n%v", + q.Select[0].Field, queryStr, q) + } + if q.Select[0].Operation != Last { + t.Errorf("Expected 'Last' as aggregation function of first element in "+ + "'select' clause but got '%v': %s\n%v", q.Select[0].Operation, queryStr, q) + } + + if q.Select[1].Field != "foo" { + t.Errorf("Expected 'foo' as first element in 'select' clause but got '%v': %s\n%v", + q.Select[1].Field, queryStr, q) + } + if q.Select[2].Field != "$foo" { + t.Errorf("Expected '$foo' as first element in 'select' clause but got '%v': %s\n%v", + q.Select[2].Field, queryStr, q) + } + + if q.Select[3].Field != "$foo" { + t.Errorf("Expected '$foo' as first element in 'select' clause but got '%v': %s\n%v", + q.Select[3].Field, queryStr, q) + } + if q.Select[3].Operation != Count { + t.Errorf("Expected 'count' as aggregation function of thourth element in "+ + "'select' clause but got '%v': %s\n%v", q.Select[3].Operation, queryStr, q) + } +} diff --git a/internal/mapr/selectcondition.go b/internal/mapr/selectcondition.go index 45fc16b..78359c7 100644 --- a/internal/mapr/selectcondition.go +++ b/internal/mapr/selectcondition.go @@ -40,16 +40,18 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) { // Parse select aggregation, e.g. sum(foo) parse := func(token token) (selectCondition, error) { var sc selectCondition - tokenStr := token.str - if !strings.Contains(tokenStr, "(") && !strings.Contains(tokenStr, ")") { - sc.Field = tokenStr - sc.FieldStorage = tokenStr + // With quotes stripped: We got a quoted select expression, e.g. + // "select `count($foo)` ...", which will literaly look for field + // "count($foo)" without performing the count aggregation. + if token.quotesStripped || (!strings.Contains(token.str, "(") && !strings.Contains(token.str, ")")) { + sc.Field = token.str + sc.FieldStorage = token.str sc.Operation = Last return sc, nil } - a := strings.Split(tokenStr, "(") + a := strings.Split(token.str, "(") if len(a) != 2 { return sc, errors.New(invalidQuery + "Can't parse 'select' aggregation: " + token.str) @@ -61,8 +63,8 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) { return sc, errors.New(invalidQuery + "Can't parse 'select' field name " + "from aggregation: " + token.str) } - sc.Field = b[0] // Field name, e.g. 'foo' - sc.FieldStorage = tokenStr // e.g. 'sum(foo)' + sc.Field = b[0] // Field name, e.g. 'foo' + sc.FieldStorage = token.str // e.g. 'sum(foo)' switch agg { case "count": @@ -80,8 +82,7 @@ func makeSelectConditions(tokens []token) ([]selectCondition, error) { case "len": sc.Operation = Len default: - return sc, errors.New(invalidQuery + - "Unknown aggregation in 'select' clause: " + agg) + return sc, errors.New(invalidQuery + "Unknown aggregation in 'select' clause: " + agg) } return sc, nil } diff --git a/internal/mapr/token.go b/internal/mapr/token.go index 6ac7631..48d1192 100644 --- a/internal/mapr/token.go +++ b/internal/mapr/token.go @@ -9,8 +9,9 @@ var keywords = [...]string{"select", "from", "where", "set", "group", "rorder", // Represents a parsed token, used to parse the mapr query. type token struct { - str string - isBareword bool + str string + isBareword bool + quotesStripped bool } func (t token) isKeyword() bool { @@ -71,8 +72,9 @@ func tokensConsume(tokens []token) ([]token, []token) { stripped := t.str[1 : length-1] //dlog.Common.Trace("stripped", stripped) t := token{ - str: stripped, - isBareword: t.isBareword, + str: stripped, + isBareword: t.isBareword, + quotesStripped: true, } consumed = append(consumed, t) continue -- cgit v1.2.3 From f771066f175c7bde9fd5cbcf39ab855afd5d5786 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 17:17:38 +0300 Subject: add mapr aggregration on CSV integr test --- integrationtests/dmap5.csv.expected | 2 ++ integrationtests/dmap5.csv.in | 4 +++ integrationtests/dmap5.csv.query.expected | 1 + integrationtests/dmap_test.go | 57 +++++++++++++++++++++++++++++++ internal/mapr/logformat/csv.go | 53 ++++++++++++++++++++++++++++ internal/mapr/setcondition.go | 7 ++++ 6 files changed, 124 insertions(+) create mode 100644 integrationtests/dmap5.csv.expected create mode 100644 integrationtests/dmap5.csv.in create mode 100644 integrationtests/dmap5.csv.query.expected create mode 100644 internal/mapr/logformat/csv.go diff --git a/integrationtests/dmap5.csv.expected b/integrationtests/dmap5.csv.expected new file mode 100644 index 0000000..1323a61 --- /dev/null +++ b/integrationtests/dmap5.csv.expected @@ -0,0 +1,2 @@ +sum($timecount),last($time),min($min_goroutines) +63.000000,1002-071143,12.000000 diff --git a/integrationtests/dmap5.csv.in b/integrationtests/dmap5.csv.in new file mode 100644 index 0000000..98c7763 --- /dev/null +++ b/integrationtests/dmap5.csv.in @@ -0,0 +1,4 @@ +count($time),$time,max($goroutines),avg($goroutines),min($goroutines) +23,1002-071147,16.000000,14.391304,12.000000 +20,1002-071213,17.000000,14.100000,12.000000 +20,1002-071143,17.000000,15.000000,13.000000 diff --git a/integrationtests/dmap5.csv.query.expected b/integrationtests/dmap5.csv.query.expected new file mode 100644 index 0000000..2d1723c --- /dev/null +++ b/integrationtests/dmap5.csv.query.expected @@ -0,0 +1 @@ +select sum($timecount),last($time),min($min_goroutines), group by $hostname set $timecount = `count($time)`, $time = `$time`, $min_goroutines = `min($goroutines)` logformat csv outfile dmap5.csv.tmp \ No newline at end of file diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index a378fb5..f772243 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -245,3 +245,60 @@ func TestDMap4Append(t *testing.T) { os.Remove(csvFile) os.Remove(queryFile) } + +func TestDMap5CSV(t *testing.T) { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { + t.Log("Skipping") + return + } + inFile := "dmap5.csv.in" + outFile := "dmap5.stdout.tmp" + csvFile := "dmap5.csv.tmp" + expectedCsvFile := "dmap5.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap5.csv.query.expected" + + // Delete in case it exists already. Otherwise, test will fail. + os.Remove(csvFile) + + query := fmt.Sprintf("select sum($timecount),last($time),min($min_goroutines),"+ + " group by $hostname"+ + " set $timecount = `count($time)`, $time = `$time`, $min_goroutines = `min($goroutines)`"+ + " logformat csv outfile %s", csvFile) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Run dmap command twice, it should append in the 2nd iteration the new results to the already existing + // file as we specified "outfile append". That works transparently for any mapreduce query + // (e.g. also for the dtail command in streaming mode). But it is easier to test with the dmap + // command. + for i := 0; i < 2; i++ { + stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, + "", "../dmap", + "--query", query, + "--cfg", "none", + "--logger", "stdout", + "--logLevel", "info", + "--noColor", inFile) + + if err != nil { + t.Error(err) + return + } + waitForCommand(ctx, t, stdoutCh, stderrCh, cmdErrCh) + } + + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } + + os.Remove(outFile) + os.Remove(csvFile) + os.Remove(queryFile) +} diff --git a/internal/mapr/logformat/csv.go b/internal/mapr/logformat/csv.go new file mode 100644 index 0000000..ea85ca9 --- /dev/null +++ b/internal/mapr/logformat/csv.go @@ -0,0 +1,53 @@ +package logformat + +import ( + "fmt" + "strings" + + "github.com/mimecast/dtail/internal/protocol" +) + +type csvParser struct { + defaultParser + header []string + hasHeader bool +} + +func newCSVParser(hostname, timeZoneName string, timeZoneOffset int) (*csvParser, error) { + defaultParser, err := newDefaultParser(hostname, timeZoneName, timeZoneOffset) + if err != nil { + return &csvParser{}, err + } + return &csvParser{defaultParser: *defaultParser}, nil +} + +func (p *csvParser) MakeFields(maprLine string) (map[string]string, error) { + if !p.hasHeader { + p.parseHeader(maprLine) + return nil, ErrIgnoreFields + } + + fields := make(map[string]string, 7+len(p.header)) + fields["*"] = "*" + fields["$hostname"] = p.hostname + fields["$server"] = p.hostname + fields["$line"] = maprLine + fields["$empty"] = "" + fields["$timezone"] = p.timeZoneName + fields["$timeoffset"] = p.timeZoneOffset + + splitted := strings.Split(maprLine, protocol.CSVDelimiter) + for i, value := range splitted { + if i >= len(p.header) { + return fields, fmt.Errorf("CSV file seems corrupted, more fields than header values?") + } + fields[p.header[i]] = value + } + + return fields, nil +} + +func (p *csvParser) parseHeader(maprLine string) { + p.header = strings.Split(maprLine, protocol.CSVDelimiter) + p.hasHeader = true +} diff --git a/internal/mapr/setcondition.go b/internal/mapr/setcondition.go index 9dcd690..308a0f4 100644 --- a/internal/mapr/setcondition.go +++ b/internal/mapr/setcondition.go @@ -37,6 +37,13 @@ func makeSetConditions(tokens []token) (set []setCondition, err error) { return sc, nil, err } + // Seems like a quoted string? E.g.: "set $foo = `count(bar)`" + // So don't interpret `count` as a function! + if tokens[2].quotesStripped { + sc.rType = Field + return sc, tokens[3:], nil + } + // Seems like a function call? if strings.HasSuffix(sc.rString, ")") { functionStack, functionArg, err := funcs.NewFunctionStack(tokens[2].str) -- cgit v1.2.3 From 05ef7d56f945242fecb97cf03a3a9abab47013ee Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 5 Sep 2023 17:41:02 +0300 Subject: add CSV aggr example to docs --- doc/examples.md | 19 +++++++++++++++++++ doc/logformats.md | 4 +++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/examples.md b/doc/examples.md index 26ce002..4937cc5 100644 --- a/doc/examples.md +++ b/doc/examples.md @@ -151,6 +151,25 @@ You can also use a file input pipe as follows: dmap 'from STATS select $hostname,max($goroutines),max($cgocalls),$loadavg,lifetimeConnections group by $hostname order by max($cgocalls)' ``` +### Aggregating CSV files + +In essence, this works exactly like aggregating logs. All files operated on must be valid CSV files and the first line of the CSV must be the header. E.g.: + +```shell +% cat example.csv +name,lastname,age,profession +Michael,Jordan,40,Basketball player +Michael,Jackson,100,Singer +Albert,Einstein,200,Physician +% dmap --query 'select lastname,name where age > 40 logformat csv outfile result.csv' example.csv +% cat result.csv +lastname,name +Jackson,Michael +Einstein,Albert +``` + +DMap can also be used to query and aggregate CSV files from remote servers. + ### Other serverless commands The serverless mode works transparently with all other DTail commands. Here are some examples: diff --git a/doc/logformats.md b/doc/logformats.md index 839b050..dbf2051 100644 --- a/doc/logformats.md +++ b/doc/logformats.md @@ -10,8 +10,10 @@ You could either make your application follow the DTail default log format, or y The following log formats are currently available out of the box: * `default` - The default DTail log format -* `generic` - A generic log format with a very simple set of fields +* `generic` - A generic log format with a simple set of fields * `generickv` - A simple log format expecting all log lines in form of `field1=value1|field2=value2|...` +* `csv` - A simple CSV format expecting all files a comma separated CSV file. The first line of the file must be the CSV header. +* `custom1` and `custom2` - Customizable log formats. ### Selecting a log format -- cgit v1.2.3 From b518c88c108939510bd6523158619c3a54d8f53d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 7 Sep 2023 11:46:46 +0300 Subject: refactor go build tags --- Makefile | 45 ++++++++++++++++--------------------- internal/mapr/logformat/mimecast.go | 7 ++++++ internal/mapr/logformat/parser.go | 2 ++ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index d764051..717d701 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,31 @@ GO ?= go +ifdef DTAIL_USE_ACL +GO_TAGS=linuxacl +endif +ifdef DTAIL_USE_PROPRIETARY +GO_TAGS+=proprietary +endif all: build build: dserver dcat dgrep dmap dtail dtailhealth dserver: -ifndef DTAIL_USE_ACL - ${GO} build ${GO_FLAGS} -o dserver ./cmd/dserver/main.go -else - ${GO} build ${GO_FLAGS} -tags linuxacl -o dserver ./cmd/dserver/main.go -endif + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dserver ./cmd/dserver/main.go dcat: - ${GO} build ${GO_FLAGS} -o dcat ./cmd/dcat/main.go + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dcat ./cmd/dcat/main.go dgrep: - ${GO} build ${GO_FLAGS} -o dgrep ./cmd/dgrep/main.go + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dgrep ./cmd/dgrep/main.go dmap: - ${GO} build ${GO_FLAGS} -o dmap ./cmd/dmap/main.go + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dmap ./cmd/dmap/main.go dtail: - ${GO} build ${GO_FLAGS} -o dtail ./cmd/dtail/main.go + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dtail ./cmd/dtail/main.go dtailhealth: - ${GO} build ${GO_FLAGS} -o dtailhealth ./cmd/dtailhealth/main.go + ${GO} build ${GO_FLAGS} -tags '${GO_TAGS}' -o dtailhealth ./cmd/dtailhealth/main.go install: -ifndef DTAIL_USE_ACL - ${GO} install ./cmd/dserver/main.go -else - ${GO} install -tags linuxacl ./cmd/dserver/main.go -endif - ${GO} install ./cmd/dcat/main.go - ${GO} install ./cmd/dgrep/main.go - ${GO} install ./cmd/dmap/main.go - ${GO} install ./cmd/dtail/main.go - ${GO} install ./cmd/dtailhealth/main.go + ${GO} install -tags '${GO_TAGS}' ./cmd/dserver/main.go + ${GO} install -tags '${GO_TAGS}' ./cmd/dcat/main.go + ${GO} install -tags '${GO_TAGS}' ./cmd/dgrep/main.go + ${GO} install -tags '${GO_TAGS}' ./cmd/dmap/main.go + ${GO} install -tags '${GO_TAGS}' ./cmd/dtail/main.go + ${GO} install -tags '${GO_TAGS}' ./cmd/dtailhealth/main.go clean: ls ./cmd/ | while read cmd; do \ test -f $$cmd && rm $$cmd; \ @@ -47,10 +45,5 @@ lint: done | grep -F .go: test: ${GO} clean -testcache -ifndef DTAIL_USE_ACL set -e; find . -name '*_test.go' | while read file; do dirname $$file; done | \ - sort -u | while read dir; do ${GO} test --race -v $$dir || exit 2; done -else - set -e;find . -name '*_test.go' | while read file; do dirname $$file; done | \ - sort -u | while read dir; do ${GO} test --tags linuxacl --race -v $$dir || exit 2; done -endif + sort -u | while read dir; do ${GO} test -tags '${GO_TAGS}' --race -v $$dir || exit 2; done diff --git a/internal/mapr/logformat/mimecast.go b/internal/mapr/logformat/mimecast.go index bc6e30a..cf6b333 100644 --- a/internal/mapr/logformat/mimecast.go +++ b/internal/mapr/logformat/mimecast.go @@ -1,3 +1,6 @@ +//go:build !proprietary +// +build !proprietary + package logformat import "errors" @@ -11,6 +14,10 @@ func newMimecastParser(hostname, timeZoneName string, timeZoneOffset int) (*mime return &mimecastParser{}, ErrMimecastNotAvailable } +func newMimecastGenericParser(hostname, timeZoneName string, timeZoneOffset int) (*mimecastParser, error) { + return &mimecastParser{}, ErrMimecastNotAvailable +} + func (p *mimecastParser) MakeFields(maprLine string) (map[string]string, error) { return nil, ErrMimecastNotAvailable } diff --git a/internal/mapr/logformat/parser.go b/internal/mapr/logformat/parser.go index 9888932..37d7a63 100644 --- a/internal/mapr/logformat/parser.go +++ b/internal/mapr/logformat/parser.go @@ -37,6 +37,8 @@ func NewParser(logFormatName string, query *mapr.Query) (Parser, error) { return newCSVParser(hostname, timeZoneName, timeZoneOffset) case "mimecast": return newMimecastParser(hostname, timeZoneName, timeZoneOffset) + case "mimecastgeneric": + return newMimecastGenericParser(hostname, timeZoneName, timeZoneOffset) case "default": return newDefaultParser(hostname, timeZoneName, timeZoneOffset) case "custom1": -- cgit v1.2.3 From d6efc889f1dc9582ba8006633376c022c945a126 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Thu, 7 Sep 2023 15:48:29 +0300 Subject: update dependencies --- go.mod | 9 ++++++--- go.sum | 14 +++++++------- internal/version/version.go | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index fce2a35..edef243 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,10 @@ go 1.20 require ( github.com/DataDog/zstd v1.5.5 - golang.org/x/crypto v0.10.0 - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect + golang.org/x/crypto v0.13.0 +) + +require ( + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index 90a6d4b..f0d2329 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ -github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= diff --git a/internal/version/version.go b/internal/version/version.go index 90073b5..15ea50f 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ const ( // Name of DTail. Name string = "DTail" // Version of DTail. - Version string = "4.2.0" + Version string = "4.3.0" // Additional information for DTail Additional string = "Have a lot of fun!" ) -- cgit v1.2.3