Implement syslog and google stackdriver logging

cli/cli.go

@@ -43,8 +43,22 @@ func init() {
 	mainCmd.PersistentFlags().StringVar(&opts.Logging.Level, "log-level", "error", "Specify log verbosity")
 	mainCmd.PersistentFlags().StringVar(&opts.Logging.Output, "log-output", "stderr", "Specify log output destination")
 	mainCmd.PersistentFlags().StringVar(&opts.Logging.Format, "log-format", "text", "Specify log output format (text, json)")
-	mainCmd.PersistentFlags().StringVar(&opts.Logging.Newrelic, "log-newrelic", "", "Log to Newrelic using the specified license key")
+
-	mainCmd.PersistentFlags().MarkHidden("log-newrelic")
+	mainCmd.PersistentFlags().StringVar(&opts.Logging.Google.Name, "log-driver-google-name", "surreal", "")
+	mainCmd.PersistentFlags().StringVar(&opts.Logging.Google.Project, "log-driver-google-project", "", "")
+	mainCmd.PersistentFlags().StringVar(&opts.Logging.Google.Credentials, "log-driver-google-credentials", "", "")
+	mainCmd.PersistentFlags().MarkHidden("log-driver-google-name")
+	mainCmd.PersistentFlags().MarkHidden("log-driver-google-project")
+	mainCmd.PersistentFlags().MarkHidden("log-driver-google-credentials")
+
+	mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Tag, "log-driver-syslog-tag", "cirrius", "Specify a tag for the syslog driver")
+	mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Host, "log-driver-syslog-host", "localhost:514", "Specify a remote host:port for the syslog driver")
+	mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Protocol, "log-driver-syslog-protocol", "", "Specify the protocol to use for the syslog driver")
+	mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Priority, "log-driver-syslog-priority", "debug", "Specify the syslog priority for the syslog driver")
+	mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-tag")
+	mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-host")
+	mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-protocol")
+	mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-priority")
 
 	cobra.OnInitialize(setup)
 

cli/setup.go

@@ -230,12 +230,14 @@ func setup() {
 	// Logging
 	// --------------------------------------------------
 
+	var chk map[string]bool
+
 	// Ensure that the specified
 	// logging level is allowed
 
 	if opts.Logging.Level != "" {
 
-		chk := map[string]bool{
+		chk = map[string]bool{
 			"debug":   true,
 			"info":    true,
 			"warning": true,
@@ -252,31 +254,12 @@ func setup() {
 
 	}
 
-	// Ensure that the specified
-	// logging output is allowed
-
-	if opts.Logging.Output != "" {
-
-		chk := map[string]bool{
-			"none":   true,
-			"stdout": true,
-			"stderr": true,
-		}
-
-		if _, ok := chk[opts.Logging.Output]; !ok {
-			log.Fatal("Incorrect log output specified")
-		}
-
-		log.SetOutput(opts.Logging.Output)
-
-	}
-
 	// Ensure that the specified
 	// logging format is allowed
 
 	if opts.Logging.Format != "" {
 
-		chk := map[string]bool{
+		chk = map[string]bool{
 			"text": true,
 			"json": true,
 		}
@@ -289,6 +272,93 @@ func setup() {
 
 	}
 
+	// Ensure that the specified
+	// logging output is allowed
+
+	if opts.Logging.Output != "" {
+
+		chk = map[string]bool{
+			"none":   true,
+			"stdout": true,
+			"stderr": true,
+			"google": true,
+			"syslog": true,
+		}
+
+		if _, ok := chk[opts.Logging.Output]; !ok {
+			log.Fatal("Incorrect log output specified")
+		}
+
+		if opts.Logging.Output == "google" {
+
+			hook, err := log.NewGoogleHook(
+				opts.Logging.Level,
+				opts.Logging.Google.Name,
+				opts.Logging.Google.Project,
+				opts.Logging.Google.Credentials,
+			)
+
+			if err != nil {
+				log.Fatal("There was a problem configuring the google logging driver")
+			}
+
+			log.Instance().Hooks.Add(hook)
+
+			opts.Logging.Output = "none"
+
+		}
+
+		if opts.Logging.Output == "syslog" {
+
+			chk = map[string]bool{
+				"":    true,
+				"tcp": true,
+				"udp": true,
+			}
+
+			if _, ok := chk[opts.Logging.Syslog.Protocol]; !ok {
+				log.Fatal("Incorrect log protocol specified for syslog logging driver")
+			}
+
+			chk = map[string]bool{
+				"debug":   true,
+				"info":    true,
+				"notice":  true,
+				"warning": true,
+				"err":     true,
+				"crit":    true,
+				"alert":   true,
+				"emerg":   true,
+			}
+
+			if _, ok := chk[opts.Logging.Syslog.Priority]; !ok {
+				log.Fatal("Incorrect log priority specified for syslog logging driver")
+			}
+
+			hook, err := log.NewSyslogHook(
+				opts.Logging.Level,
+				opts.Logging.Syslog.Protocol,
+				opts.Logging.Syslog.Host,
+				opts.Logging.Syslog.Priority,
+				opts.Logging.Syslog.Tag,
+			)
+
+			if err != nil {
+				log.Fatal("There was a problem configuring the syslog logging driver")
+			}
+
+			log.Instance().Hooks.Add(hook)
+
+			opts.Logging.Output = "none"
+
+		}
+
+		log.SetOutput(opts.Logging.Output)
+
+	}
+
+	// Enable global options object
+
 	cnf.Settings = opts
 
 }

cli/start.go

@@ -88,7 +88,6 @@ func init() {
 	startCmd.PersistentFlags().StringVarP(&opts.DB.Code, "key", "k", "", flag("key"))
 
 	startCmd.PersistentFlags().StringVarP(&opts.Node.Host, "bind", "b", "0.0.0.0", "The hostname or ip address to listen for connections on.")
-
 	startCmd.PersistentFlags().StringVarP(&opts.Node.Name, "name", "n", host, "The name of this node, used for logs and statistics.")
 
 	startCmd.PersistentFlags().IntVar(&opts.Port.Tcp, "port-tcp", 33693, "The port on which to serve the tcp server.")

cnf/cnf.go

@@ -83,9 +83,19 @@ type Options struct {
 	}
 
 	Logging struct {
-		Level    string // Stores the configured logging level
+		Level  string // Stores the configured logging level
-		Output   string // Stores the configured logging output
+		Output string // Stores the configured logging output
-		Format   string // Stores the configured logging format
+		Format string // Stores the configured logging format
-		Newrelic string // Stores the configured newrelic license key
+		Google struct {
+			Name        string // Stores the GCE logging name
+			Project     string // Stores the GCE logging project
+			Credentials string // Store the path to the credentials file
+		}
+		Syslog struct {
+			Tag      string // Stores the syslog tag name
+			Host     string // Stores the syslog remote host:port
+			Protocol string // Stores the syslog protocol to use
+			Priority string // Stores the syslog logging priority
+		}
 	}
 }

log/google.go

@@ -0,0 +1,160 @@
+// Copyright © 2016 Abcum Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package log
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Sirupsen/logrus"
+
+	"cloud.google.com/go/compute/metadata"
+	"cloud.google.com/go/logging"
+	"google.golang.org/api/option"
+	"google.golang.org/genproto/googleapis/api/monitoredres"
+)
+
+type HookGoogle struct {
+	name        string
+	project     string
+	credentials string
+	level       logrus.Level
+	levels      []logrus.Level
+	logclient   *logging.Client
+	logbuffer   *logging.Logger
+}
+
+func NewGoogleHook(level, name, project, credentials string) (hook *HookGoogle, err error) {
+
+	hook = &HookGoogle{}
+
+	// Ensure that we only send the
+	// specified log levels to the
+	// Google Stackdriver endpoint.
+
+	switch level {
+	case "debug":
+		hook.level = logrus.DebugLevel
+	case "info":
+		hook.level = logrus.InfoLevel
+	case "warning":
+		hook.level = logrus.WarnLevel
+	case "error":
+		hook.level = logrus.ErrorLevel
+	case "fatal":
+		hook.level = logrus.FatalLevel
+	case "panic":
+		hook.level = logrus.PanicLevel
+	default:
+		return nil, fmt.Errorf("Please specify a valid google logging level")
+	}
+
+	for l := logrus.PanicLevel; l <= hook.level; l++ {
+		hook.levels = append(hook.levels, l)
+	}
+
+	// Specify the log name that all
+	// logs should be stored under in
+	// Google Stackdriver.
+
+	hook.name = name
+
+	// If no project id has been set
+	// then attempt to pull this from
+	// machine metadata if on GCE.
+
+	if project == "" {
+
+		if project, err = metadata.ProjectID(); err != nil {
+			return nil, err
+		}
+
+	}
+
+	// Otherwise set the log name to
+	// the project name which has been
+	// specified on the command line.
+
+	hook.project = project
+
+	// Connect to Stackdriver using a
+	// credentials file if one has been
+	// specified, or metadata if not.
+
+	switch credentials {
+	case "":
+		hook.logclient, err = logging.NewClient(
+			context.Background(),
+			hook.project,
+		)
+	default:
+		hook.logclient, err = logging.NewClient(
+			context.Background(),
+			hook.project,
+			option.WithServiceAccountFile(credentials),
+		)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Attempt to ping the Stackdriver
+	// endpoint to ensure the settings
+	// and authentication are correct.
+
+	err = hook.logclient.Ping(context.Background())
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup the asynchronous buffering
+	// logger, which we can use to send
+	// logs to the Stackdriver client.
+
+	hook.logbuffer = hook.logclient.Logger(
+		hook.name,
+		logging.CommonResource(&monitoredres.MonitoredResource{
+			Type: "logging_log",
+		}),
+	)
+
+	return hook, err
+
+}
+
+func (h *HookGoogle) Levels() []logrus.Level {
+	return h.levels
+}
+
+func (h *HookGoogle) Fire(entry *logrus.Entry) error {
+
+	e := logging.Entry{
+		Timestamp: entry.Time,
+		Labels:    make(map[string]string),
+		Payload:   entry.Message,
+		Severity:  logging.ParseSeverity(entry.Level.String()),
+	}
+
+	for k, v := range entry.Data {
+		e.Labels[k] = fmt.Sprintf("%v", v)
+	}
+
+	h.logbuffer.Log(e)
+
+	return nil
+
+}

log/syslog.go

@@ -0,0 +1,111 @@
+// Copyright © 2016 Abcum Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package log
+
+import (
+	"fmt"
+	"log/syslog"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type HookSyslog struct {
+	host     string
+	protocol string
+	endpoint *syslog.Writer
+}
+
+func NewSyslogHook(level, protocol, host, priority, tag string) (hook *HookSyslog, err error) {
+
+	hook = &HookSyslog{}
+
+	var endpoint *syslog.Writer
+	var severity syslog.Priority
+
+	// Convert the passed priority to
+	// one of the expected syslog
+	// severity levels.
+
+	switch priority {
+	case "debug":
+		severity = syslog.LOG_DEBUG
+	case "info":
+		severity = syslog.LOG_INFO
+	case "notice":
+		severity = syslog.LOG_NOTICE
+	case "warning":
+		severity = syslog.LOG_WARNING
+	case "err":
+		severity = syslog.LOG_ERR
+	case "crit":
+		severity = syslog.LOG_CRIT
+	case "alert":
+		severity = syslog.LOG_ALERT
+	case "emerg":
+		severity = syslog.LOG_EMERG
+	default:
+		return nil, fmt.Errorf("Please specify a valid syslog priority")
+	}
+
+	// Attempt to dial the syslog
+	// endpoint, or exit if there is
+	// a problem connecting.
+
+	if endpoint, err = syslog.Dial(protocol, host, severity, tag); err != nil {
+		return nil, err
+	}
+
+	// Finish setting up the logrus
+	// hook with the configuration
+	// options which were specified.
+
+	hook.host = host
+	hook.protocol = protocol
+	hook.endpoint = endpoint
+
+	return hook, err
+
+}
+
+func (h *HookSyslog) Levels() []logrus.Level {
+	return logrus.AllLevels
+}
+
+func (h *HookSyslog) Fire(entry *logrus.Entry) error {
+
+	line := entry.Message
+
+	for k, v := range entry.Data {
+		line += fmt.Sprintf(" %s=%v", k, v)
+	}
+
+	switch entry.Level {
+	case logrus.PanicLevel:
+		h.endpoint.Crit(line)
+	case logrus.FatalLevel:
+		h.endpoint.Crit(line)
+	case logrus.ErrorLevel:
+		h.endpoint.Err(line)
+	case logrus.WarnLevel:
+		h.endpoint.Warning(line)
+	case logrus.InfoLevel:
+		h.endpoint.Notice(line)
+	case logrus.DebugLevel:
+		h.endpoint.Info(line)
+	}
+
+	return nil
+
+}