Browse Source

Merge branch 'morefmts' of boyska/circolog into master

blallo 5 years ago
parent
commit
7dc3b5a7bb
8 changed files with 103 additions and 36 deletions
  1. 2 2
      cmd/circolog-tail/main.go
  2. 3 3
      cmd/circologd/http_log.go
  3. 1 1
      cmd/circologd/main.go
  4. 32 0
      data/data.go
  5. 47 4
      docs/docs/systemd.md
  6. 2 13
      filtering/filter.go
  7. 8 8
      formatter/format.go
  8. 8 5
      hub.go

+ 2 - 2
cmd/circolog-tail/main.go

@@ -11,12 +11,12 @@ import (
 	"strconv"
 	"strconv"
 	"time"
 	"time"
 
 
+	"git.lattuga.net/boyska/circolog/data"
 	"git.lattuga.net/boyska/circolog/filtering"
 	"git.lattuga.net/boyska/circolog/filtering"
 	"git.lattuga.net/boyska/circolog/formatter"
 	"git.lattuga.net/boyska/circolog/formatter"
 	"github.com/gorilla/websocket"
 	"github.com/gorilla/websocket"
 	isatty "github.com/mattn/go-isatty"
 	isatty "github.com/mattn/go-isatty"
 	"github.com/mgutz/ansi"
 	"github.com/mgutz/ansi"
-	"gopkg.in/mcuadros/go-syslog.v2/format"
 	"gopkg.in/mgo.v2/bson"
 	"gopkg.in/mgo.v2/bson"
 )
 )
 
 
@@ -112,7 +112,7 @@ func main() {
 				log.Println("close:", err)
 				log.Println("close:", err)
 				return
 				return
 			}
 			}
-			var parsed format.LogParts
+			var parsed data.Message
 			if err := bson.Unmarshal(serialized, &parsed); err != nil {
 			if err := bson.Unmarshal(serialized, &parsed); err != nil {
 				log.Println("invalid BSON", err)
 				log.Println("invalid BSON", err)
 				continue
 				continue

+ 3 - 3
cmd/circologd/http_log.go

@@ -9,9 +9,9 @@ import (
 	"time"
 	"time"
 
 
 	"git.lattuga.net/boyska/circolog"
 	"git.lattuga.net/boyska/circolog"
+	"git.lattuga.net/boyska/circolog/data"
 	"git.lattuga.net/boyska/circolog/formatter"
 	"git.lattuga.net/boyska/circolog/formatter"
 	"github.com/gorilla/websocket"
 	"github.com/gorilla/websocket"
-	"gopkg.in/mcuadros/go-syslog.v2/format"
 )
 )
 
 
 func setupHTTP(hub circolog.Hub) *http.ServeMux {
 func setupHTTP(hub circolog.Hub) *http.ServeMux {
@@ -100,7 +100,7 @@ func getHTTPHandler(hub circolog.Hub) http.HandlerFunc {
 		opts.Nofollow = true
 		opts.Nofollow = true
 
 
 		client := circolog.Client{
 		client := circolog.Client{
-			Messages: make(chan format.LogParts, 20),
+			Messages: make(chan data.Message, 20),
 			Options:  opts,
 			Options:  opts,
 		}
 		}
 		hub.Register <- client
 		hub.Register <- client
@@ -141,7 +141,7 @@ func getWSHandler(hub circolog.Hub) http.HandlerFunc {
 			return
 			return
 		}
 		}
 		client := circolog.Client{
 		client := circolog.Client{
-			Messages: make(chan format.LogParts, 20),
+			Messages: make(chan data.Message, 20),
 			Options:  opts,
 			Options:  opts,
 		}
 		}
 		hub.Register <- client
 		hub.Register <- client

+ 1 - 1
cmd/circologd/main.go

@@ -43,7 +43,7 @@ func main() {
 	go hub.Run()
 	go hub.Run()
 
 
 	server := syslog.NewServer()
 	server := syslog.NewServer()
-	server.SetFormat(syslog.RFC5424)
+	server.SetFormat(syslog.Automatic)
 	server.SetHandler(handler)
 	server.SetHandler(handler)
 	if *syslogSocketPath != "" {
 	if *syslogSocketPath != "" {
 		if err = server.ListenUnixgram(*syslogSocketPath); err != nil {
 		if err = server.ListenUnixgram(*syslogSocketPath); err != nil {

+ 32 - 0
data/data.go

@@ -0,0 +1,32 @@
+package data
+
+import "gopkg.in/mcuadros/go-syslog.v2/format"
+
+// Message is currently an alias for format.Logparts, but this is only temporary; sooner or later, a real struct will be used
+// The advantage of having an explicit Message is to clear out what data we are sending to circolog "readers"
+// This is not necessarily (and not in practice) the same structure that we receive from logging programs
+type Message format.LogParts
+
+// LogEntryToMessage converts messages received from writers to the format we promise to readers
+func LogEntryToMessage(orig format.LogParts) Message {
+	m := Message{}
+	if orig["version"] == 1 { // RFC5424
+		m["prog"] = orig["app_name"]
+		m["client"] = orig["client"]
+		m["host"] = orig["hostname"]
+		m["proc_id"] = orig["proc_id"]
+		m["msg"] = orig["message"]
+		m["facility"] = orig["facility"]
+		m["time"] = orig["timestamp"]
+		m["sev"] = orig["severity"]
+	} else { //RFC3164
+		m["prog"] = orig["tag"]
+		m["client"] = orig["client"]
+		m["host"] = orig["hostname"]
+		m["msg"] = orig["content"]
+		m["sev"] = orig["severity"]
+		m["time"] = orig["timestamp"]
+		m["proc_id"] = "-"
+	}
+	return m
+}

+ 47 - 4
docs/docs/systemd.md

@@ -1,7 +1,24 @@
-unit systemd con utente dinamico:
-    - no adduser/altre conf
-    - utente con pochi permessi
-    - accesso consentito al gruppo adm
+## A simple start
+
+The bare minimum you need to get circologd on a systemd-based system is this unit.
+Other options with more features or more security are provided below
+
+    [Unit]
+    Description=In-memory logging
+
+    [Service]
+    User=root
+    Group=adm
+    ExecStart=/usr/local/sbin/circologd -syslogd-socket /run/circolog/syslog.sock  -buffer-size 2000 -query-socket /run/circolog/query.sock
+
+    [Install]
+    WantedBy=multi-user.target
+
+
+## A better unit
+
+This is another unit, which has several security features, such as `DynamicUser`, filesystem restrictions, and
+more.
 
 
     [Unit]
     [Unit]
     Description=In-memory logging
     Description=In-memory logging
@@ -27,3 +44,29 @@ unit systemd con utente dinamico:
     [Install]
     [Install]
     WantedBy=multi-user.target
     WantedBy=multi-user.target
 
 
+## Journald
+
+None of those are integrated with journald, however. The simplest way to integrate with journald is the
+following.
+
+First of all, ensure `ForwardToSyslog=yes` in `/etc/systemd/journald.conf`.
+Then, you need to run circologd as root and bind it [to a special
+address](https://www.freedesktop.org/software/systemd/man/journald.conf.html#Forwarding%20to%20traditional%20syslog%20daemons).
+Ok, you don't strictly _need_ to run it as root, but that's the easiest way to run it.
+Here is a working unit for this:
+
+    [Unit]
+    Description=In-memory logging
+
+    [Service]
+    User=root
+    Group=adm
+    ExecStart=/usr/local/sbin/circologd -syslogd-socket /run/systemd/journal/syslog -buffer-size 2000 -query-socket /run/circolog/query.sock
+
+    [Install]
+    WantedBy=multi-user.target
+
+### journald with socket activation
+
+To run circologd as non-root, while listening on a root-owned socket (`/run/systemd/journal/syslog`) use
+socket activation

+ 2 - 13
filtering/filter.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"fmt"
 	"os"
 	"os"
 
 
+	"git.lattuga.net/boyska/circolog/data"
 	"github.com/araddon/qlbridge/datasource"
 	"github.com/araddon/qlbridge/datasource"
 	"github.com/araddon/qlbridge/expr"
 	"github.com/araddon/qlbridge/expr"
 	"github.com/araddon/qlbridge/value"
 	"github.com/araddon/qlbridge/value"
@@ -40,11 +41,10 @@ func (e *ExprValue) Set(value string) error {
 }
 }
 
 
 // Validate answers the question whether to include a log line or not.
 // Validate answers the question whether to include a log line or not.
-func (e *ExprValue) Validate(logLine map[string]interface{}) bool {
+func (e *ExprValue) Validate(line data.Message) bool {
 	if e.node == nil {
 	if e.node == nil {
 		return true
 		return true
 	}
 	}
-	line := translateMap(logLine)
 	context := datasource.NewContextSimpleNative(line)
 	context := datasource.NewContextSimpleNative(line)
 	val, ok := vm.Eval(context, e.node)
 	val, ok := vm.Eval(context, e.node)
 	if !ok || val == nil { // errors when evaluating
 	if !ok || val == nil { // errors when evaluating
@@ -56,14 +56,3 @@ func (e *ExprValue) Validate(logLine map[string]interface{}) bool {
 	fmt.Fprintln(os.Stderr, "WARNING: The 'where' expression doesn't return a boolean")
 	fmt.Fprintln(os.Stderr, "WARNING: The 'where' expression doesn't return a boolean")
 	return false
 	return false
 }
 }
-
-func translateMap(lineInput map[string]interface{}) map[string]interface{} {
-	lineOutput := make(map[string]interface{})
-	lineOutput["prog"] = lineInput["app_name"]
-	lineOutput["msg"] = lineInput["message"]
-	lineOutput["facility"] = lineInput["facility"]
-	lineOutput["host"] = lineInput["hostname"]
-	lineOutput["time"] = lineInput["timestamp"]
-	lineOutput["sev"] = lineInput["severity"]
-	return lineOutput
-}

+ 8 - 8
formatter/format.go

@@ -8,13 +8,13 @@ import (
 	"text/template"
 	"text/template"
 	"time"
 	"time"
 
 
+	"git.lattuga.net/boyska/circolog/data"
 	"github.com/mgutz/ansi"
 	"github.com/mgutz/ansi"
-	"gopkg.in/mcuadros/go-syslog.v2/format"
 	"gopkg.in/mgo.v2/bson"
 	"gopkg.in/mgo.v2/bson"
 )
 )
 
 
 // Formatter is an interface, so that multiple implementations can exist
 // Formatter is an interface, so that multiple implementations can exist
-type Formatter func(format.LogParts) string
+type Formatter func(data.Message) string
 
 
 var tmplFuncs template.FuncMap
 var tmplFuncs template.FuncMap
 var syslogTmpl *template.Template
 var syslogTmpl *template.Template
@@ -56,11 +56,11 @@ func init() {
 		},
 		},
 	}
 	}
 	syslogTmpl = template.Must(template.New("syslog").Funcs(tmplFuncs).Parse(
 	syslogTmpl = template.Must(template.New("syslog").Funcs(tmplFuncs).Parse(
-		"{{color \"yellow\" (rfc822 (index . \"timestamp\")) }} {{index . \"hostname\"}} " +
-			"{{index . \"app_name\" | autoColor}}" +
+		"{{color \"yellow\" (rfc822 (index . \"time\")) }} {{index . \"host\"}} " +
+			"{{index . \"prog\" | autoColor}}" +
 			"{{ if (ne (index . \"proc_id\") \"-\")}}[{{index . \"proc_id\"}}]{{end}}: " +
 			"{{ if (ne (index . \"proc_id\") \"-\")}}[{{index . \"proc_id\"}}]{{end}}: " +
-			"{{ sevName (index . \"severity\") }} " +
-			"{{index . \"message\"}}",
+			"{{ sevName (index . \"sev\") }} " +
+			"{{index . \"msg\"}}",
 	))
 	))
 }
 }
 
 
@@ -87,7 +87,7 @@ func (rf Format) String() string {
 	return ""
 	return ""
 }
 }
 
 
-func (rf Format) WriteFormatted(w io.Writer, msg format.LogParts) error {
+func (rf Format) WriteFormatted(w io.Writer, msg data.Message) error {
 	return WriteFormatted(w, rf, msg)
 	return WriteFormatted(w, rf, msg)
 }
 }
 
 
@@ -110,7 +110,7 @@ const (
 	FormatBSON   = iota
 	FormatBSON   = iota
 )
 )
 
 
-func WriteFormatted(w io.Writer, f Format, msg format.LogParts) error {
+func WriteFormatted(w io.Writer, f Format, msg data.Message) error {
 	switch f {
 	switch f {
 	case FormatSyslog:
 	case FormatSyslog:
 		return syslogTmpl.Execute(w, msg)
 		return syslogTmpl.Execute(w, msg)

+ 8 - 5
hub.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os"
 	"time"
 	"time"
 
 
+	"git.lattuga.net/boyska/circolog/data"
 	"git.lattuga.net/boyska/circolog/filtering"
 	"git.lattuga.net/boyska/circolog/filtering"
 	"gopkg.in/mcuadros/go-syslog.v2/format"
 	"gopkg.in/mcuadros/go-syslog.v2/format"
 )
 )
@@ -13,10 +14,11 @@ import (
 // Client represent a client connected via websocket. Its most important field is the messages channel, where
 // Client represent a client connected via websocket. Its most important field is the messages channel, where
 // new messages are sent.
 // new messages are sent.
 type Client struct {
 type Client struct {
-	Messages chan format.LogParts // only hub should write/close this
+	Messages chan data.Message // only hub should write/close this
 	Options  ClientOptions
 	Options  ClientOptions
 }
 }
 
 
+// ClientOptions is a struct containing connection options for every reader
 type ClientOptions struct {
 type ClientOptions struct {
 	BacklogLength int  // how many past messages the client wants to receive upon connection
 	BacklogLength int  // how many past messages the client wants to receive upon connection
 	Nofollow      bool // if Nofollow is true, the hub will not keep this client permanently. Rather, it will send every message to "Messages" and close the channel. Use this if you want to get the messages one-shot
 	Nofollow      bool // if Nofollow is true, the hub will not keep this client permanently. Rather, it will send every message to "Messages" and close the channel. Use this if you want to get the messages one-shot
@@ -100,7 +102,7 @@ func (h *Hub) register(cl Client) {
 			item := buf.Value
 			item := buf.Value
 			if item != nil {
 			if item != nil {
 				select { // send with short timeout
 				select { // send with short timeout
-				case cl.Messages <- item.(format.LogParts):
+				case cl.Messages <- item.(data.Message):
 					break
 					break
 				case <-time.After(500 * time.Millisecond):
 				case <-time.After(500 * time.Millisecond):
 					close(cl.Messages)
 					close(cl.Messages)
@@ -132,12 +134,13 @@ func (h *Hub) Run() {
 				delete(h.clients, cl)
 				delete(h.clients, cl)
 			}
 			}
 		case msg := <-h.LogMessages:
 		case msg := <-h.LogMessages:
-			if active == true && filter.Validate(msg) {
-				h.circbuf.Value = msg
+			newmsg := data.LogEntryToMessage(msg)
+			if active == true && filter.Validate(newmsg) {
+				h.circbuf.Value = newmsg
 				h.circbuf = h.circbuf.Next()
 				h.circbuf = h.circbuf.Next()
 				for client := range h.clients {
 				for client := range h.clients {
 					select { // send without blocking
 					select { // send without blocking
-					case client.Messages <- msg:
+					case client.Messages <- newmsg:
 						break
 						break
 					default:
 					default:
 						break
 						break