Merge branch 'morefmts' of boyska/circolog into master

also, align hostname to docs -> host

cmd/circolog-tail/main.go

@@ -11,12 +11,12 @@ import (
 	"strconv"
 	"time"
 
+	"git.lattuga.net/boyska/circolog/data"
 	"git.lattuga.net/boyska/circolog/filtering"
 	"git.lattuga.net/boyska/circolog/formatter"
 	"github.com/gorilla/websocket"
 	isatty "github.com/mattn/go-isatty"
 	"github.com/mgutz/ansi"
-	"gopkg.in/mcuadros/go-syslog.v2/format"
 	"gopkg.in/mgo.v2/bson"
 )
 
@@ -112,7 +112,7 @@ func main() {
 				log.Println("close:", err)
 				return
 			}
-			var parsed format.LogParts
+			var parsed data.Message
 			if err := bson.Unmarshal(serialized, &parsed); err != nil {
 				log.Println("invalid BSON", err)
 				continue

cmd/circologd/http_log.go

@@ -9,9 +9,9 @@ import (
 	"time"
 
 	"git.lattuga.net/boyska/circolog"
+	"git.lattuga.net/boyska/circolog/data"
 	"git.lattuga.net/boyska/circolog/formatter"
 	"github.com/gorilla/websocket"
-	"gopkg.in/mcuadros/go-syslog.v2/format"
 )
 
 func setupHTTP(hub circolog.Hub) *http.ServeMux {
@@ -100,7 +100,7 @@ func getHTTPHandler(hub circolog.Hub) http.HandlerFunc {
 		opts.Nofollow = true
 
 		client := circolog.Client{
-			Messages: make(chan format.LogParts, 20),
+			Messages: make(chan data.Message, 20),
 			Options:  opts,
 		}
 		hub.Register <- client
@@ -141,7 +141,7 @@ func getWSHandler(hub circolog.Hub) http.HandlerFunc {
 			return
 		}
 		client := circolog.Client{
-			Messages: make(chan format.LogParts, 20),
+			Messages: make(chan data.Message, 20),
 			Options:  opts,
 		}
 		hub.Register <- client

cmd/circologd/main.go

@@ -43,7 +43,7 @@ func main() {
 	go hub.Run()
 
 	server := syslog.NewServer()
-	server.SetFormat(syslog.RFC5424)
+	server.SetFormat(syslog.Automatic)
 	server.SetHandler(handler)
 	if *syslogSocketPath != "" {
 		if err = server.ListenUnixgram(*syslogSocketPath); err != nil {

data/data.go

@@ -0,0 +1,32 @@
+package data
+
+import "gopkg.in/mcuadros/go-syslog.v2/format"
+
+// Message is currently an alias for format.Logparts, but this is only temporary; sooner or later, a real struct will be used
+// The advantage of having an explicit Message is to clear out what data we are sending to circolog "readers"
+// This is not necessarily (and not in practice) the same structure that we receive from logging programs
+type Message format.LogParts
+
+// LogEntryToMessage converts messages received from writers to the format we promise to readers
+func LogEntryToMessage(orig format.LogParts) Message {
+	m := Message{}
+	if orig["version"] == 1 { // RFC5424
+		m["prog"] = orig["app_name"]
+		m["client"] = orig["client"]
+		m["host"] = orig["hostname"]
+		m["proc_id"] = orig["proc_id"]
+		m["msg"] = orig["message"]
+		m["facility"] = orig["facility"]
+		m["time"] = orig["timestamp"]
+		m["sev"] = orig["severity"]
+	} else { //RFC3164
+		m["prog"] = orig["tag"]
+		m["client"] = orig["client"]
+		m["host"] = orig["hostname"]
+		m["msg"] = orig["content"]
+		m["sev"] = orig["severity"]
+		m["time"] = orig["timestamp"]
+		m["proc_id"] = "-"
+	}
+	return m
+}

docs/docs/hacking.md

@@ -7,10 +7,14 @@ that is meaningful in itself. Fortunately `go` allows to pull packages from any
 git server that is publicly available. Therefore do not be afraid of go-getting
 from git.lattuga.net.
 
-[Spiegare meglio la natura autogestita di lattuga]
+lattuga.net is a self-managed server, run with clear principles of not-for-profit and antifascist nature.
+
+We think using situated, decentralized tools and networks is fundamental to get a technological landscape that
+doesn't lead to oppression.
 
 ### So how to collaborate
 
+Pull request is not the only paradigm for collaboration :)
+
 If you want to collaborate to the software, either drop a mail to
-<indirizzo@scamuffo.com> or open an account on git.lattuga.net to open a PR. We
+<indirizzo@scamuffo.com>. We are also reachable at [IRC irc.mufhd0.net]
-are also reachable at [IRC irc.mufhd0.net]

docs/docs/systemd.md

@@ -1,7 +1,24 @@
-unit systemd con utente dinamico:
+## A simple start
-    - no adduser/altre conf
+
-    - utente con pochi permessi
+The bare minimum you need to get circologd on a systemd-based system is this unit.
-    - accesso consentito al gruppo adm
+Other options with more features or more security are provided below
+
+    [Unit]
+    Description=In-memory logging
+
+    [Service]
+    User=root
+    Group=adm
+    ExecStart=/usr/local/sbin/circologd -syslogd-socket /run/circolog/syslog.sock  -buffer-size 2000 -query-socket /run/circolog/query.sock
+
+    [Install]
+    WantedBy=multi-user.target
+
+
+## A better unit
+
+This is another unit, which has several security features, such as `DynamicUser`, filesystem restrictions, and
+more.
 
     [Unit]
     Description=In-memory logging
@@ -27,3 +44,29 @@ unit systemd con utente dinamico:
     [Install]
     WantedBy=multi-user.target
 
+## Journald
+
+None of those are integrated with journald, however. The simplest way to integrate with journald is the
+following.
+
+First of all, ensure `ForwardToSyslog=yes` in `/etc/systemd/journald.conf`.
+Then, you need to run circologd as root and bind it [to a special
+address](https://www.freedesktop.org/software/systemd/man/journald.conf.html#Forwarding%20to%20traditional%20syslog%20daemons).
+Ok, you don't strictly _need_ to run it as root, but that's the easiest way to run it.
+Here is a working unit for this:
+
+    [Unit]
+    Description=In-memory logging
+
+    [Service]
+    User=root
+    Group=adm
+    ExecStart=/usr/local/sbin/circologd -syslogd-socket /run/systemd/journal/syslog -buffer-size 2000 -query-socket /run/circolog/query.sock
+
+    [Install]
+    WantedBy=multi-user.target
+
+### journald with socket activation
+
+To run circologd as non-root, while listening on a root-owned socket (`/run/systemd/journal/syslog`) use
+socket activation

filtering/filter.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 
+	"git.lattuga.net/boyska/circolog/data"
 	"github.com/araddon/qlbridge/datasource"
 	"github.com/araddon/qlbridge/expr"
 	"github.com/araddon/qlbridge/value"
@@ -40,11 +41,10 @@ func (e *ExprValue) Set(value string) error {
 }
 
 // Validate answers the question whether to include a log line or not.
-func (e *ExprValue) Validate(logLine map[string]interface{}) bool {
+func (e *ExprValue) Validate(line data.Message) bool {
 	if e.node == nil {
 		return true
 	}
-	line := translateMap(logLine)
 	context := datasource.NewContextSimpleNative(line)
 	val, ok := vm.Eval(context, e.node)
 	if !ok || val == nil { // errors when evaluating
@@ -56,14 +56,3 @@ func (e *ExprValue) Validate(logLine map[string]interface{}) bool {
 	fmt.Fprintln(os.Stderr, "WARNING: The 'where' expression doesn't return a boolean")
 	return false
 }
-
-func translateMap(lineInput map[string]interface{}) map[string]interface{} {
-	lineOutput := make(map[string]interface{})
-	lineOutput["prog"] = lineInput["app_name"]
-	lineOutput["msg"] = lineInput["message"]
-	lineOutput["facility"] = lineInput["facility"]
-	lineOutput["host"] = lineInput["hostname"]
-	lineOutput["time"] = lineInput["timestamp"]
-	lineOutput["sev"] = lineInput["severity"]
-	return lineOutput
-}

formatter/format.go

@@ -8,13 +8,13 @@ import (
 	"text/template"
 	"time"
 
+	"git.lattuga.net/boyska/circolog/data"
 	"github.com/mgutz/ansi"
-	"gopkg.in/mcuadros/go-syslog.v2/format"
 	"gopkg.in/mgo.v2/bson"
 )
 
 // Formatter is an interface, so that multiple implementations can exist
-type Formatter func(format.LogParts) string
+type Formatter func(data.Message) string
 
 var tmplFuncs template.FuncMap
 var syslogTmpl *template.Template
@@ -56,11 +56,11 @@ func init() {
 		},
 	}
 	syslogTmpl = template.Must(template.New("syslog").Funcs(tmplFuncs).Parse(
-		"{{color \"yellow\" (rfc822 (index . \"timestamp\")) }} {{index . \"hostname\"}} " +
+		"{{color \"yellow\" (rfc822 (index . \"time\")) }} {{index . \"host\"}} " +
-			"{{index . \"app_name\" | autoColor}}" +
+			"{{index . \"prog\" | autoColor}}" +
 			"{{ if (ne (index . \"proc_id\") \"-\")}}[{{index . \"proc_id\"}}]{{end}}: " +
-			"{{ sevName (index . \"severity\") }} " +
+			"{{ sevName (index . \"sev\") }} " +
-			"{{index . \"message\"}}",
+			"{{index . \"msg\"}}",
 	))
 }
 
@@ -87,7 +87,7 @@ func (rf Format) String() string {
 	return ""
 }
 
-func (rf Format) WriteFormatted(w io.Writer, msg format.LogParts) error {
+func (rf Format) WriteFormatted(w io.Writer, msg data.Message) error {
 	return WriteFormatted(w, rf, msg)
 }
 
@@ -110,7 +110,7 @@ const (
 	FormatBSON   = iota
 )
 
-func WriteFormatted(w io.Writer, f Format, msg format.LogParts) error {
+func WriteFormatted(w io.Writer, f Format, msg data.Message) error {
 	switch f {
 	case FormatSyslog:
 		return syslogTmpl.Execute(w, msg)

hub.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	"git.lattuga.net/boyska/circolog/data"
 	"git.lattuga.net/boyska/circolog/filtering"
 	"gopkg.in/mcuadros/go-syslog.v2/format"
 )
@@ -13,10 +14,11 @@ import (
 // Client represent a client connected via websocket. Its most important field is the messages channel, where
 // new messages are sent.
 type Client struct {
-	Messages chan format.LogParts // only hub should write/close this
+	Messages chan data.Message // only hub should write/close this
 	Options  ClientOptions
 }
 
+// ClientOptions is a struct containing connection options for every reader
 type ClientOptions struct {
 	BacklogLength int  // how many past messages the client wants to receive upon connection
 	Nofollow      bool // if Nofollow is true, the hub will not keep this client permanently. Rather, it will send every message to "Messages" and close the channel. Use this if you want to get the messages one-shot
@@ -100,7 +102,7 @@ func (h *Hub) register(cl Client) {
 			item := buf.Value
 			if item != nil {
 				select { // send with short timeout
-				case cl.Messages <- item.(format.LogParts):
+				case cl.Messages <- item.(data.Message):
 					break
 				case <-time.After(500 * time.Millisecond):
 					close(cl.Messages)
@@ -132,12 +134,13 @@ func (h *Hub) Run() {
 				delete(h.clients, cl)
 			}
 		case msg := <-h.LogMessages:
-			if active == true && filter.Validate(msg) {
+			newmsg := data.LogEntryToMessage(msg)
-				h.circbuf.Value = msg
+			if active == true && filter.Validate(newmsg) {
+				h.circbuf.Value = newmsg
 				h.circbuf = h.circbuf.Next()
 				for client := range h.clients {
 					select { // send without blocking
-					case client.Messages <- msg:
+					case client.Messages <- newmsg:
 						break
 					default:
 						break