From 7704c5ac705f17d5a8efb61fcdadfef3569a85fb Mon Sep 17 00:00:00 2001 From: boyska Date: Sat, 10 Nov 2018 17:50:51 +0100 Subject: [PATCH 01/10] tiny reformatting --- cmd/circologd/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/circologd/main.go b/cmd/circologd/main.go index 12faac1..ddde695 100644 --- a/cmd/circologd/main.go +++ b/cmd/circologd/main.go @@ -35,6 +35,7 @@ func main() { hub := circolog.NewHub(*bufsize) handler := syslog.NewChannelHandler(hub.LogMessages) + go hub.Run() server := syslog.NewServer() server.SetFormat(syslog.RFC5424) @@ -57,7 +58,6 @@ func main() { fmt.Fprintln(os.Stderr, "argh", err) os.Exit(1) } - go hub.Run() setupHTTP(hub) httpServer := http.Server{Handler: nil} @@ -94,7 +94,6 @@ func main() { fmt.Fprintln(os.Stderr, "Error closing http server:", err) } return - default: } } } From a990422953d786d27484fa0806d52a3ff3b66f20 Mon Sep 17 00:00:00 2001 From: boyska Date: Sat, 10 Nov 2018 18:00:35 +0100 Subject: [PATCH 02/10] HUP on circologd clears the buffer --- cmd/circologd/main.go | 20 +++++++++++++------- hub.go | 9 +++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/cmd/circologd/main.go b/cmd/circologd/main.go index ddde695..d3dfa10 100644 --- a/cmd/circologd/main.go +++ b/cmd/circologd/main.go @@ -31,7 +31,7 @@ func main() { flag.Parse() interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGHUP, syscall.SIGTERM) hub := circolog.NewHub(*bufsize) handler := syslog.NewChannelHandler(hub.LogMessages) @@ -84,16 +84,22 @@ func main() { } }() } - + // TODO: now we are ready for { select { case sig := <-interrupt: - log.Println("Quitting because of signal", sig) - server.Kill() - if err := httpServer.Close(); err != nil { - fmt.Fprintln(os.Stderr, "Error closing http server:", err) + if sig == syscall.SIGHUP { + log.Println("Clearing queue") + hub.Clear() + } + if sig == syscall.SIGTERM || sig == syscall.SIGINT { + log.Println("Quitting because of signal", sig) + server.Kill() + if err := httpServer.Close(); err != nil { + fmt.Fprintln(os.Stderr, "Error closing http server:", err) + } + return } - return } } } diff --git a/hub.go b/hub.go index 10bf8c1..eceb3be 100644 --- a/hub.go +++ b/hub.go @@ -101,3 +101,12 @@ func (h *Hub) Run() { } } } + +// Clear removes every all elements from the buffer +func (h *Hub) Clear() { + buf := h.circbuf + for i := 0; i < buf.Len(); i++ { + buf.Value = nil + buf = buf.Next() + } +} From 3f216f12f838f067454c0430326c0c40c468656e Mon Sep 17 00:00:00 2001 From: boyska Date: Sat, 10 Nov 2018 18:21:42 +0100 Subject: [PATCH 03/10] USR1 on circologd pause/resume --- cmd/circologd/main.go | 8 ++++++-- hub.go | 44 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/cmd/circologd/main.go b/cmd/circologd/main.go index d3dfa10..aed1266 100644 --- a/cmd/circologd/main.go +++ b/cmd/circologd/main.go @@ -31,7 +31,7 @@ func main() { flag.Parse() interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, syscall.SIGINT, syscall.SIGHUP, syscall.SIGTERM) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGTERM) hub := circolog.NewHub(*bufsize) handler := syslog.NewChannelHandler(hub.LogMessages) @@ -90,7 +90,11 @@ func main() { case sig := <-interrupt: if sig == syscall.SIGHUP { log.Println("Clearing queue") - hub.Clear() + hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandClear} + } + if sig == syscall.SIGUSR1 { + log.Println("Pause/resume") + hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandPauseToggle} } if sig == syscall.SIGTERM || sig == syscall.SIGINT { log.Println("Quitting because of signal", sig) diff --git a/hub.go b/hub.go index eceb3be..f4c746b 100644 --- a/hub.go +++ b/hub.go @@ -24,10 +24,25 @@ type ClientOptions struct { // The channel "register" and "unregister" can be seen as "command" // keep in mind that "registering" is what you do also to get messages in a one-time fashion. In fact, Client // has "options", such as Nofollow, to explain the Hub what should be given + +// An HubCommand is an "enum" of different commands +type HubCommand int + +const ( + CommandClear = iota + CommandPauseToggle = iota +) + +// An HubFullCommand is a Command, complete with arguments +type HubFullCommand struct { + Command HubCommand +} + type Hub struct { Register chan Client Unregister chan Client LogMessages chan format.LogParts + Commands chan HubFullCommand clients map[Client]bool circbuf *ring.Ring @@ -39,6 +54,7 @@ func NewHub(ringBufSize int) Hub { Register: make(chan Client), Unregister: make(chan Client), LogMessages: make(chan format.LogParts), + Commands: make(chan HubFullCommand), circbuf: ring.New(ringBufSize), } } @@ -77,6 +93,7 @@ func (h *Hub) register(cl Client) { // Run is hub main loop; keeps everything going func (h *Hub) Run() { + active := true for { select { case cl := <-h.Register: @@ -88,22 +105,31 @@ func (h *Hub) Run() { delete(h.clients, cl) } case msg := <-h.LogMessages: - h.circbuf.Value = msg - h.circbuf = h.circbuf.Next() - for client := range h.clients { - select { // send without blocking - case client.Messages <- msg: - break - default: - break + if active == true { + h.circbuf.Value = msg + h.circbuf = h.circbuf.Next() + for client := range h.clients { + select { // send without blocking + case client.Messages <- msg: + break + default: + break + } } } + case cmd := <-h.Commands: + if cmd.Command == CommandClear { + h.clear() + } + if cmd.Command == CommandPauseToggle { + active = !active + } } } } // Clear removes every all elements from the buffer -func (h *Hub) Clear() { +func (h *Hub) clear() { buf := h.circbuf for i := 0; i < buf.Len(); i++ { buf.Value = nil From 7c17c971a00e3c77d5b5a5f2410aca3324e2427b Mon Sep 17 00:00:00 2001 From: boyska Date: Sun, 11 Nov 2018 19:10:53 +0100 Subject: [PATCH 04/10] command responses --- cmd/circologd/http.go | 1 + cmd/circologd/main.go | 10 ++++++++-- hub.go | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/circologd/http.go b/cmd/circologd/http.go index 7962d8d..5e58bc9 100644 --- a/cmd/circologd/http.go +++ b/cmd/circologd/http.go @@ -13,6 +13,7 @@ import ( "gopkg.in/mcuadros/go-syslog.v2/format" ) +// TODO: return a server func setupHTTP(hub circolog.Hub) { http.HandleFunc("/", getHTTPHandler(hub)) http.HandleFunc("/ws", getWSHandler(hub)) diff --git a/cmd/circologd/main.go b/cmd/circologd/main.go index aed1266..e3d2b8d 100644 --- a/cmd/circologd/main.go +++ b/cmd/circologd/main.go @@ -89,12 +89,18 @@ func main() { select { case sig := <-interrupt: if sig == syscall.SIGHUP { - log.Println("Clearing queue") hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandClear} + <-hub.Responses + log.Println("Queue cleared") } if sig == syscall.SIGUSR1 { - log.Println("Pause/resume") hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandPauseToggle} + resp := <-hub.Responses + if resp.Value.(bool) { + log.Println("resumed") + } else { + log.Println("paused") + } } if sig == syscall.SIGTERM || sig == syscall.SIGINT { log.Println("Quitting because of signal", sig) diff --git a/hub.go b/hub.go index f4c746b..639612f 100644 --- a/hub.go +++ b/hub.go @@ -37,12 +37,16 @@ const ( type HubFullCommand struct { Command HubCommand } +type CommandResponse struct { + Value interface{} +} type Hub struct { Register chan Client Unregister chan Client LogMessages chan format.LogParts Commands chan HubFullCommand + Responses chan CommandResponse clients map[Client]bool circbuf *ring.Ring @@ -55,6 +59,7 @@ func NewHub(ringBufSize int) Hub { Unregister: make(chan Client), LogMessages: make(chan format.LogParts), Commands: make(chan HubFullCommand), + Responses: make(chan CommandResponse), circbuf: ring.New(ringBufSize), } } @@ -120,9 +125,11 @@ func (h *Hub) Run() { case cmd := <-h.Commands: if cmd.Command == CommandClear { h.clear() + h.Responses <- CommandResponse{Value: true} } if cmd.Command == CommandPauseToggle { active = !active + h.Responses <- CommandResponse{Value: active} } } } From 515e91068359f1865eadffe9a834ad2ce03d6f7d Mon Sep 17 00:00:00 2001 From: boyska Date: Sun, 11 Nov 2018 19:18:13 +0100 Subject: [PATCH 05/10] refactor httpd --- cmd/circologd/{http.go => http_log.go} | 9 +++++---- cmd/circologd/main.go | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) rename cmd/circologd/{http.go => http_log.go} (95%) diff --git a/cmd/circologd/http.go b/cmd/circologd/http_log.go similarity index 95% rename from cmd/circologd/http.go rename to cmd/circologd/http_log.go index 5e58bc9..d9d21ee 100644 --- a/cmd/circologd/http.go +++ b/cmd/circologd/http_log.go @@ -13,10 +13,11 @@ import ( "gopkg.in/mcuadros/go-syslog.v2/format" ) -// TODO: return a server -func setupHTTP(hub circolog.Hub) { - http.HandleFunc("/", getHTTPHandler(hub)) - http.HandleFunc("/ws", getWSHandler(hub)) +func setupHTTP(hub circolog.Hub) *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("/", getHTTPHandler(hub)) + mux.HandleFunc("/ws", getWSHandler(hub)) + return mux } func parseParameterL(r *http.Request) (int, error) { diff --git a/cmd/circologd/main.go b/cmd/circologd/main.go index e3d2b8d..c554114 100644 --- a/cmd/circologd/main.go +++ b/cmd/circologd/main.go @@ -59,8 +59,7 @@ func main() { os.Exit(1) } - setupHTTP(hub) - httpServer := http.Server{Handler: nil} + httpServer := http.Server{Handler: setupHTTP(hub)} if *querySocket != "" { fmt.Printf("Binding address `%s` [http]\n", *querySocket) unixListener, err := net.Listen("unix", *querySocket) From 1b08df0ce0fcd3d6faaf9f2f761009216cf1bbbe Mon Sep 17 00:00:00 2001 From: boyska Date: Sun, 11 Nov 2018 19:51:21 +0100 Subject: [PATCH 06/10] add control socket (HTTP server) also there is some refactoring on circologd: connection handling, closing, etc. Not as much as needed, though: shutdown is still unclean, and websocket clean shutdown is not tested --- cmd/circologd/http_ctl.go | 26 ++++++++++++++++++++++++++ cmd/circologd/main.go | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 cmd/circologd/http_ctl.go diff --git a/cmd/circologd/http_ctl.go b/cmd/circologd/http_ctl.go new file mode 100644 index 0000000..554869b --- /dev/null +++ b/cmd/circologd/http_ctl.go @@ -0,0 +1,26 @@ +package main + +import ( + "encoding/json" + "net/http" + + "git.lattuga.net/boyska/circolog" + "github.com/gorilla/mux" +) + +func setupHTTPCtl(hub circolog.Hub) *mux.Router { + mux := mux.NewRouter() + mux.HandleFunc("/pause/toggle", togglePause(hub)).Methods("POST") + return mux +} + +func togglePause(hub circolog.Hub) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandPauseToggle} + resp := <-hub.Responses + active := resp.Value.(bool) + w.Header().Set("content-type", "application/json") + enc := json.NewEncoder(w) + enc.Encode(map[string]interface{}{"paused": !active}) + } +} diff --git a/cmd/circologd/main.go b/cmd/circologd/main.go index c554114..054b226 100644 --- a/cmd/circologd/main.go +++ b/cmd/circologd/main.go @@ -16,7 +16,7 @@ import ( func cleanSocket(socket string) { if err := os.Remove(socket); err != nil { - fmt.Fprintln(os.Stderr, socket, ":", err) + fmt.Fprintln(os.Stderr, "Error cleaning", socket, ":", err) } } @@ -28,6 +28,7 @@ func main() { syslogAddr := flag.String("syslog-addr", "127.0.0.1:9514", "Address:port where to listen for syslog messages") queryAddr := flag.String("query-addr", "127.0.0.1:9080", "Address:port where to bind the query service") querySocket := flag.String("query-socket", "", "Path to a unix domain socket for the HTTP server; recommended for security reasons!") + ctlSocket := flag.String("ctl-socket", "/tmp/circologd-ctl.sock", "Path to a unix domain socket for the control server; leave empty to disable") flag.Parse() interrupt := make(chan os.Signal, 1) @@ -59,7 +60,7 @@ func main() { os.Exit(1) } - httpServer := http.Server{Handler: setupHTTP(hub)} + httpQueryServer := http.Server{Handler: setupHTTP(hub)} if *querySocket != "" { fmt.Printf("Binding address `%s` [http]\n", *querySocket) unixListener, err := net.Listen("unix", *querySocket) @@ -69,21 +70,39 @@ func main() { } defer cleanSocket(*querySocket) go func() { - if err := httpServer.Serve(unixListener); err != nil { - fmt.Fprintln(os.Stderr, "error binding:", err) + if err := httpQueryServer.Serve(unixListener); err != nil && err != http.ErrServerClosed { + fmt.Fprintln(os.Stderr, "error binding", *querySocket, ":", err) } }() } else { - httpServer.Addr = *queryAddr + httpQueryServer.Addr = *queryAddr fmt.Printf("Binding address `%s` [http]\n", *queryAddr) go func() { - err := httpServer.ListenAndServe() - if err != nil { + err := httpQueryServer.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + fmt.Fprintln(os.Stderr, "error binding", *queryAddr, ":", err) + } + }() + } + + httpCtlServer := http.Server{Handler: setupHTTPCtl(hub)} + if *ctlSocket != "" { + fmt.Printf("Binding address `%s` [http]\n", *ctlSocket) + unixListener, err := net.Listen("unix", *ctlSocket) + if err != nil { + fmt.Fprintln(os.Stderr, "Error binding HTTP unix domain socket", err) + return + } + defer cleanSocket(*ctlSocket) + go func() { + if err := httpCtlServer.Serve(unixListener); err != nil && err != http.ErrServerClosed { fmt.Fprintln(os.Stderr, "error binding:", err) } }() } + // TODO: now we are ready + for { select { case sig := <-interrupt: @@ -104,9 +123,12 @@ func main() { if sig == syscall.SIGTERM || sig == syscall.SIGINT { log.Println("Quitting because of signal", sig) server.Kill() - if err := httpServer.Close(); err != nil { + if err := httpQueryServer.Shutdown(nil); err != nil { fmt.Fprintln(os.Stderr, "Error closing http server:", err) } + if err := httpCtlServer.Shutdown(nil); err != nil { + fmt.Fprintln(os.Stderr, "Error closing control server:", err) + } return } } From eb66cb43077ac2b016689116900200fbe689dac0 Mon Sep 17 00:00:00 2001 From: boyska Date: Sun, 11 Nov 2018 19:58:49 +0100 Subject: [PATCH 07/10] clear logs on contrl socket --- cmd/circologd/http_ctl.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmd/circologd/http_ctl.go b/cmd/circologd/http_ctl.go index 554869b..cde8538 100644 --- a/cmd/circologd/http_ctl.go +++ b/cmd/circologd/http_ctl.go @@ -11,6 +11,7 @@ import ( func setupHTTPCtl(hub circolog.Hub) *mux.Router { mux := mux.NewRouter() mux.HandleFunc("/pause/toggle", togglePause(hub)).Methods("POST") + mux.HandleFunc("/logs/clear", clearQueue(hub)).Methods("POST") return mux } @@ -24,3 +25,14 @@ func togglePause(hub circolog.Hub) http.HandlerFunc { enc.Encode(map[string]interface{}{"paused": !active}) } } + +func clearQueue(hub circolog.Hub) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandClear} + resp := <-hub.Responses + success := resp.Value.(bool) + w.Header().Set("content-type", "application/json") + enc := json.NewEncoder(w) + enc.Encode(map[string]interface{}{"success": success}) + } +} From b2127fd3492122311ea37bf3970eb263f0ab192f Mon Sep 17 00:00:00 2001 From: boyska Date: Sun, 11 Nov 2018 20:06:10 +0100 Subject: [PATCH 08/10] update readme to recent enhancements --- README.md | 56 ++++++++++++++++++++++++++++++++++++++----- cmd/circologd/main.go | 7 +----- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3f3d6dd..8e79a89 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ A syslog daemon implementing circular buffer, in-memory storage. -This is useful when you want to keep some (heavy detailed) log available, but you don't want to log too many -things to disk. +This is useful when you want to keep some (heavily detailed) log available, but you don't want to log too many +things to disk. Remember: logging is useful, but can be dangerous to your users' privacy! -On your "main" syslog, send some message to this one! +On your "main" syslog, forward (part of the) messages to this one! ## Integration examples @@ -27,15 +27,59 @@ and run `circologd -syslogd-socket /run/circolog-syslog.sock -query-socket /run/ ## Client -`curl` might be enough of a client for most uses. +`circolog` has its own client: `circolog-tail`. It is intended to resemble `tail -f` for the most basic +options; however, it will include filtering options that are common when you want to read logs, because that's +very easy when you have structured logs available. + +However, one design point of circolog is to be usable without having a specific client: so the logs are +offered on both HTTP and websocket. This means that you can use `curl` if you want: curl --unix-socket /run/circolog-query.sock localhost/ -will give you everything that circologd has in memory +will give you everything that circologd has in memory. If you want to "follow" (as in `tail -f`) you need to use the websocket interface. However, I don't know of any websocket client supporting UNIX domain socket, so you have two options: - 1. wait until I write a proper `circolog-tail` client implementing it all + 1. Use `circolog-tail` 2. Use `circologd` with `-query-addr 127.0.0.1:9080`, add some iptables rule to prevent non-root to access that port, and run `ws ws://localhost:9080/ws`. You'll get all the "backlog", and will follow new log messages. + +### HTTP URLs and parameters + +When using HTTP, logs are served on `/`. Valid parameters are: + + * `l`. This is the amount of lines to send. This is essentially the same as the `-n` parameter on tail + Using `l=-1` (the default) means "give me every log message that you have + * `fmt`. This selects the output format. When `fmt=json` is used, each message is returned as JSON structured + data. The format of those JSON messages is still unstable. `fmt=syslog`, the default, outputs messages using "syslog style" (RFC XXXXXX) + +To use websocket, request path `/ws`. The same parameters of `/` are recognized. + +## Control daemon + +Circologd can be controlled, on some aspects, at run-time. It has 2 mechanisms for that: the easiest, and more +limited, is sending a signal with kill; the second, and more powerful, is a control socket, where you can give +commands to it. This control socket is just HTTP, so again `curl` is your friend. In the future a +`circolog-ctl` client will be developed. + +### Pause + +When circologd is paused, every new message it receives is immediately discarded. No exception. The backlog +is, however, preserved. This means that you can trigger the event that you want to investigate, pause +circolog, then analyze the logs. +Pausing might be the easiest way to make circologd only run "when needed". + +When circologd resumes, no previous message is lost. + + +To pause circologd with signals , send a `USR1` signal to the main pid. To "resume", send a `USR1` again. + +To pause with HTTP, send a `POST /pause/toggle` to your circologd control socket. + +### Clear + +When you clear the circologd's buffer, it will discard every message it has, but will keep collecting new +messages. + +You can do that with `POST /logs/clear` diff --git a/cmd/circologd/main.go b/cmd/circologd/main.go index 054b226..b40641c 100644 --- a/cmd/circologd/main.go +++ b/cmd/circologd/main.go @@ -32,7 +32,7 @@ func main() { flag.Parse() interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, syscall.SIGINT, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGTERM) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGTERM) hub := circolog.NewHub(*bufsize) handler := syslog.NewChannelHandler(hub.LogMessages) @@ -106,11 +106,6 @@ func main() { for { select { case sig := <-interrupt: - if sig == syscall.SIGHUP { - hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandClear} - <-hub.Responses - log.Println("Queue cleared") - } if sig == syscall.SIGUSR1 { hub.Commands <- circolog.HubFullCommand{Command: circolog.CommandPauseToggle} resp := <-hub.Responses From 2bf83b6c336766b7f88dd3016be9f7b80dd5a564 Mon Sep 17 00:00:00 2001 From: boyska Date: Sun, 11 Nov 2018 20:42:26 +0100 Subject: [PATCH 09/10] refactor syslog formatting the goal is making circolog-tail apply formatting "locally", receiving structured messages instead --- cmd/circologd/http_log.go | 15 ++++----- {cmd/circologd => formatter}/format.go | 43 ++++++++++++++++++++------ 2 files changed, 41 insertions(+), 17 deletions(-) rename {cmd/circologd => formatter}/format.go (61%) diff --git a/cmd/circologd/http_log.go b/cmd/circologd/http_log.go index d9d21ee..37077de 100644 --- a/cmd/circologd/http_log.go +++ b/cmd/circologd/http_log.go @@ -9,6 +9,7 @@ import ( "time" "git.lattuga.net/boyska/circolog" + "git.lattuga.net/boyska/circolog/formatter" "github.com/gorilla/websocket" "gopkg.in/mcuadros/go-syslog.v2/format" ) @@ -56,7 +57,7 @@ func parseParameters(r *http.Request) (circolog.ClientOptions, error) { } type renderOptions struct { // those are options relevant to the rendered (that is, the HTTP side of circologd) - Format renderFormat + Format formatter.Format } func parseRenderParameters(r *http.Request) (renderOptions, error) { @@ -70,11 +71,10 @@ func parseRenderParameters(r *http.Request) (renderOptions, error) { if len(val) != 1 { return opts, errors.New("Format repeated multiple times") } - format, err := parseRenderFormat(val[0]) + err := opts.Format.Set(val[0]) if err != nil { return opts, err } - opts.Format = format } return opts, nil } @@ -106,9 +106,10 @@ func getHTTPHandler(hub circolog.Hub) http.HandlerFunc { hub.Register <- client for x := range client.Messages { - writeFormatted(w, render_opts.Format, x) - if render_opts.Format != formatJSON { // bleah - w.Write([]byte("\n")) + if err := render_opts.Format.WriteFormatted(w, x); err == nil { + if render_opts.Format != formatter.FormatJSON { // bleah + w.Write([]byte("\n")) + } } } } @@ -166,7 +167,7 @@ func getWSHandler(hub circolog.Hub) http.HandlerFunc { if err != nil { return } - writeFormatted(w, render_opts.Format, message) + render_opts.Format.WriteFormatted(w, message) if err := w.Close(); err != nil { return diff --git a/cmd/circologd/format.go b/formatter/format.go similarity index 61% rename from cmd/circologd/format.go rename to formatter/format.go index 4818d2f..a386d5b 100644 --- a/cmd/circologd/format.go +++ b/formatter/format.go @@ -1,4 +1,4 @@ -package main +package formatter import ( "encoding/json" @@ -33,29 +33,52 @@ func init() { )) } -type renderFormat int +type Format int -func parseRenderFormat(s string) (renderFormat, error) { +func (rf *Format) Set(v string) error { + newvalue, err := parseFormat(v) + if err != nil { + return err + } + *rf = newvalue + return nil +} + +func (rf Format) String() string { + switch rf { + case FormatJSON: + return "json" + case FormatSyslog: + return "syslog" + } + return "" +} + +func (rf Format) WriteFormatted(w io.Writer, msg format.LogParts) error { + return WriteFormatted(w, rf, msg) +} + +func parseFormat(s string) (Format, error) { switch s { case "json": - return formatJSON, nil + return FormatJSON, nil case "syslog": - return formatSyslog, nil + return FormatSyslog, nil default: return 0, fmt.Errorf("Undefined format `%s`", s) } } const ( - formatSyslog = iota // 0 - formatJSON = iota + FormatSyslog = iota // 0 + FormatJSON = iota ) -func writeFormatted(w io.Writer, f renderFormat, msg format.LogParts) error { +func WriteFormatted(w io.Writer, f Format, msg format.LogParts) error { switch f { - case formatSyslog: + case FormatSyslog: return syslogTmpl.Execute(w, msg) - case formatJSON: + case FormatJSON: enc := json.NewEncoder(w) enc.SetIndent("", " ") return enc.Encode(msg) From b11c2edfc08f1d0a977abc4502117bd96f445f24 Mon Sep 17 00:00:00 2001 From: boyska Date: Sun, 11 Nov 2018 20:54:10 +0100 Subject: [PATCH 10/10] [tail] format locally JSON is actually unfit to send structured log messages, because date/time is not well supported. So we are using BSON, which supports those. BSON is, among the dozen serializers available, really a random choice and might be changed in the future. --- cmd/circolog-tail/main.go | 16 ++++++++++++++-- formatter/format.go | 13 +++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/cmd/circolog-tail/main.go b/cmd/circolog-tail/main.go index 5b25ba7..ce653a7 100644 --- a/cmd/circolog-tail/main.go +++ b/cmd/circolog-tail/main.go @@ -11,7 +11,10 @@ import ( "strconv" "time" + "git.lattuga.net/boyska/circolog/formatter" "github.com/gorilla/websocket" + "gopkg.in/mcuadros/go-syslog.v2/format" + "gopkg.in/mgo.v2/bson" ) func main() { @@ -28,6 +31,7 @@ func main() { Path: "/ws", } q := u.Query() + q.Set("fmt", "bson") if *backlogLimit >= 0 { q.Set("l", strconv.Itoa(*backlogLimit)) } @@ -58,12 +62,20 @@ func main() { go func() { defer close(done) for { - _, message, err := c.ReadMessage() + _, serialized, err := c.ReadMessage() if err != nil { log.Println("close:", err) return } - fmt.Println(string(message)) + var parsed format.LogParts + if err := bson.Unmarshal(serialized, &parsed); err != nil { + log.Println("invalid YAML", err) + continue + } + if err := formatter.WriteFormatted(os.Stdout, formatter.FormatSyslog, parsed); err != nil { + log.Println("error printing", err) + } + fmt.Println() } }() diff --git a/formatter/format.go b/formatter/format.go index a386d5b..94a4176 100644 --- a/formatter/format.go +++ b/formatter/format.go @@ -8,6 +8,7 @@ import ( "time" "gopkg.in/mcuadros/go-syslog.v2/format" + "gopkg.in/mgo.v2/bson" ) // Formatter is an interface, so that multiple implementations can exist @@ -50,6 +51,8 @@ func (rf Format) String() string { return "json" case FormatSyslog: return "syslog" + case FormatBSON: + return "bson" } return "" } @@ -64,6 +67,8 @@ func parseFormat(s string) (Format, error) { return FormatJSON, nil case "syslog": return FormatSyslog, nil + case "bson": + return FormatBSON, nil default: return 0, fmt.Errorf("Undefined format `%s`", s) } @@ -72,6 +77,7 @@ func parseFormat(s string) (Format, error) { const ( FormatSyslog = iota // 0 FormatJSON = iota + FormatBSON = iota ) func WriteFormatted(w io.Writer, f Format, msg format.LogParts) error { @@ -82,6 +88,13 @@ func WriteFormatted(w io.Writer, f Format, msg format.LogParts) error { enc := json.NewEncoder(w) enc.SetIndent("", " ") return enc.Encode(msg) + case FormatBSON: + enc, err := bson.Marshal(msg) + if err != nil { + return err + } + _, err = w.Write(enc) + return err } return nil }