noth/surrealpatch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove google and syslog logging

Browse source
This commit is contained in:
Tobie Morgan Hitchcock 2018-04-05 08:51:40 +01:00
parent d163db967f
commit be832029a7
10 changed files with 0 additions and 570 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
cli/cli.go
View file

@ -45,22 +45,6 @@ func init() {
mainCmd.PersistentFlags().StringVar(&opts.Logging.Output, "log-output", "stderr", "Specify log output destination") mainCmd.PersistentFlags().StringVar(&opts.Logging.Output, "log-output", "stderr", "Specify log output destination")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Format, "log-format", "text", "Specify log output format (text, json)") mainCmd.PersistentFlags().StringVar(&opts.Logging.Format, "log-format", "text", "Specify log output format (text, json)")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Google.Name, "log-driver-google-name", "surreal", "")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Google.Project, "log-driver-google-project", "", "")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Google.Credentials, "log-driver-google-credentials", "", "")
mainCmd.PersistentFlags().MarkHidden("log-driver-google-name")
mainCmd.PersistentFlags().MarkHidden("log-driver-google-project")
mainCmd.PersistentFlags().MarkHidden("log-driver-google-credentials")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Tag, "log-driver-syslog-tag", "surreal", "Specify a tag for the syslog driver")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Host, "log-driver-syslog-host", "localhost:514", "Specify a remote host:port for the syslog driver")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Protocol, "log-driver-syslog-protocol", "", "Specify the protocol to use for the syslog driver")
mainCmd.PersistentFlags().StringVar(&opts.Logging.Syslog.Priority, "log-driver-syslog-priority", "debug", "Specify the syslog priority for the syslog driver")
mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-tag")
mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-host")
mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-protocol")
mainCmd.PersistentFlags().MarkHidden("log-driver-syslog-priority")
cobra.OnInitialize(setup) cobra.OnInitialize(setup)
} }

66
cli/setup.go
View file

@ -294,78 +294,12 @@ func setup() {
"none": true, "none": true,
"stdout": true, "stdout": true,
"stderr": true, "stderr": true,
"google": true,
"syslog": true,
} }
if _, ok := chk[opts.Logging.Output]; !ok { if _, ok := chk[opts.Logging.Output]; !ok {
log.Fatal("Incorrect log output specified") log.Fatal("Incorrect log output specified")
} }
if opts.Logging.Output == "google" {
hook, err := log.NewGoogleHook(
opts.Logging.Level,
opts.Logging.Google.Name,
opts.Logging.Google.Project,
opts.Logging.Google.Credentials,
)
if err != nil {
log.Fatal("There was a problem configuring the google logging driver")
}
log.Instance().Hooks.Add(hook)
opts.Logging.Output = "none"
}
if opts.Logging.Output == "syslog" {
chk = map[string]bool{
"": true,
"tcp": true,
"udp": true,
}
if _, ok := chk[opts.Logging.Syslog.Protocol]; !ok {
log.Fatal("Incorrect log protocol specified for syslog logging driver")
}
chk = map[string]bool{
"debug": true,
"info": true,
"notice": true,
"warning": true,
"err": true,
"crit": true,
"alert": true,
"emerg": true,
}
if _, ok := chk[opts.Logging.Syslog.Priority]; !ok {
log.Fatal("Incorrect log priority specified for syslog logging driver")
}
hook, err := log.NewSyslogHook(
opts.Logging.Level,
opts.Logging.Syslog.Protocol,
opts.Logging.Syslog.Host,
opts.Logging.Syslog.Priority,
opts.Logging.Syslog.Tag,
)
if err != nil {
log.Fatal("There was a problem configuring the syslog logging driver")
}
log.Instance().Hooks.Add(hook)
opts.Logging.Output = "none"
}
log.SetOutput(opts.Logging.Output) log.SetOutput(opts.Logging.Output)
} }

11
cnf/cnf.go
View file

@ -82,16 +82,5 @@ type Options struct {
Level string // Stores the configured logging level Level string // Stores the configured logging level
Output string // Stores the configured logging output Output string // Stores the configured logging output
Format string // Stores the configured logging format Format string // Stores the configured logging format
Google struct {
Name string // Stores the GCE logging name
Project string // Stores the GCE logging project
Credentials string // Store the path to the credentials file
}
Syslog struct {
Tag string // Stores the syslog tag name
Host string // Stores the syslog remote host:port
Protocol string // Stores the syslog protocol to use
Priority string // Stores the syslog logging priority
}
} }
} }

12
db/db.go
View file

@ -30,8 +30,6 @@ import (
"github.com/abcum/surreal/util/data" "github.com/abcum/surreal/util/data"
"github.com/abcum/surreal/util/uuid" "github.com/abcum/surreal/util/uuid"
"cloud.google.com/go/trace"
_ "github.com/abcum/surreal/kvs/mysql" _ "github.com/abcum/surreal/kvs/mysql"
_ "github.com/abcum/surreal/kvs/rixxdb" _ "github.com/abcum/surreal/kvs/rixxdb"
) )
@ -101,11 +99,6 @@ func Socket(fib *fibre.Context, id string) (beg, end func()) {
// the underlying data layer. // the underlying data layer.
func Execute(fib *fibre.Context, txt interface{}, vars map[string]interface{}) (out []*Response, err error) { func Execute(fib *fibre.Context, txt interface{}, vars map[string]interface{}) (out []*Response, err error) {
span := trace.FromContext(fib.Context()).NewChild("db.Execute")
nctx := trace.NewContext(fib.Context(), span)
fib = fib.WithContext(nctx)
defer span.Finish()
// Parse the received SQL batch query strings // Parse the received SQL batch query strings
// into SQL ASTs, using any immutable preset // into SQL ASTs, using any immutable preset
// variables if set. // variables if set.
@ -127,11 +120,6 @@ func Execute(fib *fibre.Context, txt interface{}, vars map[string]interface{}) (
// data layer. // data layer.
func Process(fib *fibre.Context, ast *sql.Query, vars map[string]interface{}) (out []*Response, err error) { func Process(fib *fibre.Context, ast *sql.Query, vars map[string]interface{}) (out []*Response, err error) {
span := trace.FromContext(fib.Context()).NewChild("db.Process")
nctx := trace.NewContext(fib.Context(), span)
fib = fib.WithContext(nctx)
defer span.Finish()
// If no preset variables have been defined // If no preset variables have been defined
// then ensure that the variables is // then ensure that the variables is
// instantiated for future use. // instantiated for future use.

11
db/executor.go
View file

@ -15,15 +15,12 @@
package db package db
import ( import (
"fmt"
"time" "time"
"context" "context"
"runtime/debug" "runtime/debug"
"cloud.google.com/go/trace"
"github.com/abcum/surreal/kvs" "github.com/abcum/surreal/kvs"
"github.com/abcum/surreal/log" "github.com/abcum/surreal/log"
"github.com/abcum/surreal/mem" "github.com/abcum/surreal/mem"
@ -109,9 +106,6 @@ func (e *executor) execute(ctx context.Context, ast *sql.Query) {
default: default:
trc := trace.FromContext(ctx).NewChild(fmt.Sprint(stm))
ctx := trace.NewContext(ctx, trc)
// When in debugging mode, log every sql // When in debugging mode, log every sql
// query, along with the query execution // query, along with the query execution
// speed, so we can analyse slow queries. // speed, so we can analyse slow queries.
@ -145,7 +139,6 @@ func (e *executor) execute(ctx context.Context, ast *sql.Query) {
switch stm.(type) { switch stm.(type) {
case *sql.BeginStatement: case *sql.BeginStatement:
err = e.begin(ctx, true) err = e.begin(ctx, true)
trc.Finish()
continue continue
case *sql.CancelStatement: case *sql.CancelStatement:
err, buf = e.cancel(buf, err, e.send) err, buf = e.cancel(buf, err, e.send)
@ -154,7 +147,6 @@ func (e *executor) execute(ctx context.Context, ast *sql.Query) {
} else { } else {
clear(id) clear(id)
} }
trc.Finish()
continue continue
case *sql.CommitStatement: case *sql.CommitStatement:
err, buf = e.commit(buf, err, e.send) err, buf = e.commit(buf, err, e.send)
@ -163,7 +155,6 @@ func (e *executor) execute(ctx context.Context, ast *sql.Query) {
} else { } else {
flush(id) flush(id)
} }
trc.Finish()
continue continue
} }
@ -205,8 +196,6 @@ func (e *executor) execute(ctx context.Context, ast *sql.Query) {
} }
} }
trc.Finish()
} }
} }

234
log/google.go
View file

@ -1,234 +0,0 @@
// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package log
import (
"context"
"fmt"
"net/http"
"time"
"runtime/debug"
"github.com/abcum/fibre"
"github.com/Sirupsen/logrus"
"github.com/abcum/surreal/util/build"
"cloud.google.com/go/compute/metadata"
"cloud.google.com/go/errorreporting"
"cloud.google.com/go/logging"
"google.golang.org/api/option"
"google.golang.org/genproto/googleapis/api/monitoredres"
)
type HookGoogle struct {
name string
project string
credentials string
level logrus.Level
levels []logrus.Level
logclient *logging.Client
logbuffer *logging.Logger
errclient *errorreporting.Client
}
func NewGoogleHook(level, name, project, credentials string) (hook *HookGoogle, err error) {
hook = &HookGoogle{}
// Ensure that we only send the
// specified log levels to the
// Google Stackdriver endpoint.
switch level {
case "debug":
hook.level = logrus.DebugLevel
case "info":
hook.level = logrus.InfoLevel
case "warning":
hook.level = logrus.WarnLevel
case "error":
hook.level = logrus.ErrorLevel
case "fatal":
hook.level = logrus.FatalLevel
case "panic":
hook.level = logrus.PanicLevel
default:
return nil, fmt.Errorf("Please specify a valid google logging level")
}
for l := logrus.PanicLevel; l <= hook.level; l++ {
hook.levels = append(hook.levels, l)
}
// Specify the log name that all
// logs should be stored under in
// Google Stackdriver.
hook.name = name
// If no project id has been set
// then attempt to pull this from
// machine metadata if on GCE.
if project == "" {
if project, err = metadata.ProjectID(); err != nil {
return nil, err
}
}
// Otherwise set the log name to
// the project name which has been
// specified on the command line.
hook.project = project
// Connect to Stackdriver using a
// credentials file if one has been
// specified, or metadata if not.
switch credentials {
case "":
hook.logclient, err = logging.NewClient(
context.Background(),
hook.project,
)
default:
hook.logclient, err = logging.NewClient(
context.Background(),
hook.project,
option.WithServiceAccountFile(credentials),
)
}
if err != nil {
return nil, err
}
// Attempt to ping the Stackdriver
// endpoint to ensure the settings
// and authentication are correct.
err = hook.logclient.Ping(context.Background())
if err != nil {
return nil, err
}
// Attempt to ping the Stackdriver
// endpoint to ensure the settings
// and authentication are correct.
hook.errclient, err = errorreporting.NewClient(
context.Background(),
hook.project,
errorreporting.Config{
ServiceName: hook.name,
ServiceVersion: build.GetInfo().Ver,
},
)
if err != nil {
return nil, err
}
// Setup the asynchronous buffering
// logger, which we can use to send
// logs to the Stackdriver client.
hook.logbuffer = hook.logclient.Logger(
hook.name,
logging.CommonResource(&monitoredres.MonitoredResource{
Type: "logging_log",
}),
)
return hook, err
}
func (h *HookGoogle) Levels() []logrus.Level {
return h.levels
}
func (h *HookGoogle) Fire(entry *logrus.Entry) error {
// If we receive an error, fatal, or
// panic - then log the error to GCE
// with a full stack trace.
switch entry.Level {
case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
ctx := context.Background()
err := fmt.Errorf("%s\n%s", entry.Message, debug.Stack())
go func() {
for _, v := range entry.Data {
switch i := v.(type) {
case *http.Request:
h.errclient.ReportSync(ctx, errorreporting.Entry{
Error: err, Req: i,
})
return
case *fibre.Context:
h.errclient.ReportSync(ctx, errorreporting.Entry{
Error: err, Req: i.Request().Request,
})
return
}
}
}()
}
// Otherwise just log the error to
// GCE as a log entry, and attach any
// http request data to it.
msg := logging.Entry{
Timestamp: entry.Time,
Labels: make(map[string]string),
Payload: entry.Message,
Severity: logging.ParseSeverity(entry.Level.String()),
}
for k, v := range entry.Data {
switch i := v.(type) {
default:
msg.Labels[k] = fmt.Sprintf("%v", i)
case *http.Request:
msg.HTTPRequest = &logging.HTTPRequest{
Request: i,
RemoteIP: i.RemoteAddr,
}
case *fibre.Context:
msg.HTTPRequest = &logging.HTTPRequest{
RemoteIP: i.IP().String(),
Request: i.Request().Request,
Status: i.Response().Status(),
RequestSize: i.Request().Size(),
ResponseSize: i.Response().Size(),
Latency: time.Since(i.Request().Start()),
}
}
}
h.logbuffer.Log(msg)
return nil
}

111
log/syslog.go
View file

@ -1,111 +0,0 @@
// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package log
import (
"fmt"
"log/syslog"
"github.com/Sirupsen/logrus"
)
type HookSyslog struct {
host string
protocol string
endpoint *syslog.Writer
}
func NewSyslogHook(level, protocol, host, priority, tag string) (hook *HookSyslog, err error) {
hook = &HookSyslog{}
var endpoint *syslog.Writer
var severity syslog.Priority
// Convert the passed priority to
// one of the expected syslog
// severity levels.
switch priority {
case "debug":
severity = syslog.LOG_DEBUG
case "info":
severity = syslog.LOG_INFO
case "notice":
severity = syslog.LOG_NOTICE
case "warning":
severity = syslog.LOG_WARNING
case "err":
severity = syslog.LOG_ERR
case "crit":
severity = syslog.LOG_CRIT
case "alert":
severity = syslog.LOG_ALERT
case "emerg":
severity = syslog.LOG_EMERG
default:
return nil, fmt.Errorf("Please specify a valid syslog priority")
}
// Attempt to dial the syslog
// endpoint, or exit if there is
// a problem connecting.
if endpoint, err = syslog.Dial(protocol, host, severity, tag); err != nil {
return nil, err
}
// Finish setting up the logrus
// hook with the configuration
// options which were specified.
hook.host = host
hook.protocol = protocol
hook.endpoint = endpoint
return hook, err
}
func (h *HookSyslog) Levels() []logrus.Level {
return logrus.AllLevels
}
func (h *HookSyslog) Fire(entry *logrus.Entry) error {
line := entry.Message
for k, v := range entry.Data {
line += fmt.Sprintf(" %s=%v", k, v)
}
switch entry.Level {
case logrus.PanicLevel:
h.endpoint.Crit(line)
case logrus.FatalLevel:
h.endpoint.Crit(line)
case logrus.ErrorLevel:
h.endpoint.Err(line)
case logrus.WarnLevel:
h.endpoint.Warning(line)
case logrus.InfoLevel:
h.endpoint.Notice(line)
case logrus.DebugLevel:
h.endpoint.Info(line)
}
return nil
}

4
sql/parser.go
View file

@ -20,8 +20,6 @@ import (
"strings" "strings"
"github.com/abcum/fibre" "github.com/abcum/fibre"
"cloud.google.com/go/trace"
) )
// parser represents a parser. // parser represents a parser.
@ -42,8 +40,6 @@ type parser struct {
// Parse parses sql from a []byte, string, or io.Reader. // Parse parses sql from a []byte, string, or io.Reader.
func Parse(c *fibre.Context, i interface{}) (*Query, error) { func Parse(c *fibre.Context, i interface{}) (*Query, error) {
defer trace.FromContext(c.Context()).NewChild("sql.Parse").Finish()
switch x := i.(type) { switch x := i.(type) {
default: default:
return nil, &EmptyError{} return nil, &EmptyError{}

101
web/trace.go
View file

@ -1,101 +0,0 @@
// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package web
import (
"context"
"github.com/abcum/fibre"
"github.com/abcum/surreal/cnf"
"cloud.google.com/go/compute/metadata"
"cloud.google.com/go/trace"
"google.golang.org/api/option"
)
var client *trace.Client
// tracer returns a middleware function for stackdriver trace monitoring.
func tracer() fibre.MiddlewareFunc {
var e error
var s trace.SamplingPolicy
var p string = cnf.Settings.Logging.Google.Project
var c string = cnf.Settings.Logging.Google.Credentials
// If no project id has been set
// then attempt to pull this from
// machine metadata if on GCE.
if p == "" {
if p, e = metadata.ProjectID(); e != nil {
return fibre.MiddlewareSkip
}
}
// Connect to Stackdriver using a
// credentials file if one has been
// specified, or metadata if not.
switch c {
case "":
client, e = trace.NewClient(
context.Background(),
p,
)
default:
client, e = trace.NewClient(
context.Background(),
p,
option.WithServiceAccountFile(c),
)
}
if e != nil {
return fibre.MiddlewareSkip
}
// Attempt to setup the Stackdriver
// client policy so that a fraction
// of requests are sent to google.
if s, e = trace.NewLimitedSampler(1, 5); e != nil {
return fibre.MiddlewareSkip
}
client.SetSamplingPolicy(s)
return func(h fibre.HandlerFunc) fibre.HandlerFunc {
return func(c *fibre.Context) error {
if c.Request().Header().Get("Upgrade") == "websocket" {
return h(c)
}
span := client.SpanFromRequest(c.Request().Request)
span.SetLabel("http/id", c.Get("id").(string))
ctx := trace.NewContext(c.Context(), span)
c = c.WithContext(ctx)
defer span.Finish()
return h(c)
}
}
}

4
web/web.go
View file

@ -43,10 +43,6 @@ func Setup(opts *cnf.Options) (err error) {
s.Use(mw.Logs()) // Log requests s.Use(mw.Logs()) // Log requests
s.Use(mw.Sock()) // Setup sockets s.Use(mw.Sock()) // Setup sockets
// Add trace information
s.Use(tracer())
// Add cors headers // Add cors headers
s.Use(mw.Cors(&mw.CorsOpts{ s.Use(mw.Cors(&mw.CorsOpts{