Initial cloud-services repo - gateway service + pkg modules

pkg/logger/limiter.go

@@ -0,0 +1,45 @@
+package logger
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ReneKroon/ttlcache/v2"
+)
+
+func NewLogLimiter(duration time.Duration, limit int) *LogLimiter {
+	return &LogLimiter{
+		duration: duration,
+		limit:    limit,
+	}
+}
+
+type LogLimiter struct {
+	duration  time.Duration
+	limit     int
+	cache     *ttlcache.Cache
+	onceCache sync.Once
+}
+
+func (l *LogLimiter) Check(message string) bool {
+	_, err := l.logCache().Get(message)
+	if err == ttlcache.ErrNotFound {
+		l.logCache().Set(message, true)
+		return true
+	}
+
+	return false
+}
+
+func (l *LogLimiter) logCache() *ttlcache.Cache {
+	l.onceCache.Do(func() {
+		if l.cache == nil {
+			cache := ttlcache.NewCache()
+			cache.SetTTL(l.duration)
+			cache.SetCacheSizeLimit(l.limit)
+			l.cache = cache
+		}
+	})
+
+	return l.cache
+}

pkg/logger/limiter_test.go

@@ -0,0 +1,40 @@
+package logger_test
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestLimiter(t *testing.T) {
+	err := errors.New("this is a test")
+	err2 := errors.New("this is a different test")
+	limiter := logger.NewLogLimiter(time.Millisecond*1, 5)
+
+	result := limiter.Check(err.Error())
+	if !result {
+		t.Errorf(testhelper.TestErrorTemplate, "check 1", true, result)
+	}
+
+	result = limiter.Check(err2.Error())
+	if !result {
+		t.Errorf(testhelper.TestErrorTemplate, "check different error", true, result)
+	}
+
+	for i := 0; i < 6; i++ {
+		result = limiter.Check(err.Error())
+		if result {
+			t.Errorf(testhelper.TestErrorTemplate, fmt.Sprintf("check %d", i+2), false, result)
+		}
+	}
+
+	time.Sleep(time.Millisecond * 1)
+	result = limiter.Check(err.Error())
+	if !result {
+		t.Errorf(testhelper.TestErrorTemplate, "check expired", true, result)
+	}
+}

pkg/logger/log_config

@@ -0,0 +1 @@
+info

pkg/logger/logger.go

@@ -0,0 +1,258 @@
+package logger
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"sync"
+
+	"fiskerinc.com/modules/utils/envtool"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/diode"
+	"github.com/rs/zerolog/pkgerrors"
+)
+
+const RING_BUFFER_SIZE = 1000
+
+// Logger is the global logger
+var Logger *zerolog.Logger
+var once sync.Once
+var wr diode.Writer
+
+func GetLogger() *zerolog.Logger {
+	once.Do(func() {
+		envLogLevel := envtool.GetEnv("LOG_LEVEL", "info")
+		SetGlobalLevel(envLogLevel)
+		zerolog.ErrorFieldName = "message"
+		zerolog.ErrorStackMarshaler = MarshalStack
+
+		wr = diode.NewWriter(os.Stdout, RING_BUFFER_SIZE, 0, func(missed int) {
+			Logger.Error().Msgf("logger dropped %d messages", missed)
+		})
+		logger := zerolog.New(wr).With().Timestamp().Caller().Logger()
+		Logger = &logger
+	})
+	return Logger
+}
+
+// Cleans up the diode logger
+func Close() {
+	wr.Close()
+}
+
+func MarshalStack(err error) interface{} {
+	stack := pkgerrors.MarshalStack(err)
+	if stack == nil {
+		return nil
+	}
+
+	stackS, ok := stack.([]map[string]string)
+	if !ok {
+		return stack
+	}
+
+	res := ""
+	for _, v := range stackS {
+		s := fmt.Sprintf("%v", v)
+		s = strings.TrimPrefix(s, "map[")
+		s = strings.TrimSuffix(s, "]")
+		res += s + "\n"
+	}
+
+	return res
+}
+
+// SetGlobalLevel provides static function to zerolog
+func SetGlobalLevel(level string) error {
+	l, err := zerolog.ParseLevel(level)
+	if err != nil {
+		return err
+	}
+
+	zerolog.SetGlobalLevel(l)
+	return nil
+}
+
+// Event is a custom event
+type Event struct {
+	*zerolog.Event
+}
+
+// Confidential logs a struct with potentially sensitive fields that should
+// be masked
+//
+// You must call Msg on the returned event in order to send the event
+func (e *Event) Confidential(i interface{}) *Event {
+	v := reflect.ValueOf(i)
+	if v.Type().Kind() != reflect.Ptr {
+		v = reflect.New(reflect.TypeOf(i))
+	}
+	v = v.Elem()
+
+	m := recursiveMask(v)
+	for i := 0; i < m.NumField(); i++ {
+		varName := m.Type().Field(i).Name
+		varValue := m.Field(i).Interface()
+		e.Interface(varName, varValue)
+	}
+
+	return e
+}
+
+// recursiveMask goes through reflect.Value and masks fields
+func recursiveMask(v reflect.Value) reflect.Value {
+	for i := 0; i < v.NumField(); i++ {
+		varName := v.Type().Field(i).Name
+		varType := v.Type().Field(i).Type
+		varValue := v.Field(i).Interface()
+
+		if varType.Kind() == reflect.Struct {
+			v.Field(i).Set(recursiveMask(v.Field(i)))
+		} else if varType.Kind() == reflect.Ptr {
+			if v.Field(i).IsNil() {
+				continue
+			}
+			o := v.Field(i).Elem()
+			if o.Type().Kind() == reflect.Struct {
+				recursiveMask(o)
+			}
+		} else {
+			switch varName {
+			case "Email":
+				v.Field(i).Set(reflect.ValueOf(MaskEmail(varValue.(string))))
+			case "Password":
+				v.Field(i).Set(reflect.ValueOf(MaskPassword(varValue.(string))))
+			case "Phone":
+				v.Field(i).Set(reflect.ValueOf(MaskPhoneNumber(varValue.(string))))
+			}
+		}
+	}
+
+	return v
+}
+
+// github.com/zerolog/log
+
+// Output duplicates the global logger and sets w as its output.
+func Output(w io.Writer) zerolog.Logger {
+	return GetLogger().Output(w)
+}
+
+// With creates a child logger with the field added to its context.
+func With() zerolog.Context {
+	return GetLogger().With()
+}
+
+// Level creates a child logger with the minimum accepted level set to level.
+func Level(level zerolog.Level) zerolog.Logger {
+	return GetLogger().Level(level)
+}
+
+// Sample returns a logger with the s sampler.
+func Sample(s zerolog.Sampler) zerolog.Logger {
+	return GetLogger().Sample(s)
+}
+
+// Hook returns a logger with the h Hook.
+func Hook(h zerolog.Hook) zerolog.Logger {
+	return GetLogger().Hook(h)
+}
+
+// Err starts a new message with error level with err as a field if not nil or
+// with info level if err is nil.
+//
+// You must call Msg on the returned event in order to send the event.
+func Err(err error) *Event {
+	return &Event{GetLogger().Err(err)}
+}
+
+// Trace starts a new message with trace level.
+//
+// You must call Msg on the returned event in order to send the event.
+func Trace() *Event {
+	return &Event{GetLogger().Trace()}
+}
+
+// Debug starts a new message with debug level.
+//
+// You must call Msg on the returned event in order to send the event.
+func Debug() *Event {
+	return &Event{GetLogger().Debug()}
+}
+
+// Info starts a new message with info level.
+//
+// You must call Msg on the returned event in order to send the event.
+func Info() *Event {
+	return &Event{GetLogger().Info()}
+}
+
+// Warn starts a new message with warn level.
+//
+// You must call Msg on the returned event in order to send the event.
+func Warn() *Event {
+	return &Event{GetLogger().Warn().Stack()}
+}
+
+// Error starts a new message with error level.
+//
+// You must call Msg on the returned event in order to send the event.
+func Error() *Event {
+	return &Event{GetLogger().Error().Stack()}
+}
+
+// Fatal starts a new message with fatal level. The os.Exit(1) function
+// is called by the Msg method.
+//
+// You must call Msg on the returned event in order to send the event.
+func Fatal() *Event {
+	return &Event{GetLogger().Fatal().Stack()}
+}
+
+// Panic starts a new message with panic level. The message is also sent
+// to the panic function.
+//
+// You must call Msg on the returned event in order to send the event.
+func Panic() *Event {
+	return &Event{GetLogger().Panic().Stack()}
+}
+
+// WithLevel starts a new message with level.
+//
+// You must call Msg on the returned event in order to send the event.
+func WithLevel(level zerolog.Level) *Event {
+	return &Event{GetLogger().WithLevel(level)}
+}
+
+// Log starts a new message with no level. Setting zerolog.GlobalLevel to
+// zerolog.Disabled will still disable events produced by this method.
+//
+// You must call Msg on the returned event in order to send the event.
+func Log() *Event {
+	return &Event{GetLogger().Log()}
+}
+
+// Print sends a log event using debug level and no extra field.
+// Arguments are handled in the manner of fmt.Print.
+func Print(v ...interface{}) {
+	GetLogger().Print(v...)
+}
+
+// Printf sends a log event using debug level and no extra field.
+// Arguments are handled in the manner of fmt.Printf.
+func Printf(format string, v ...interface{}) {
+	GetLogger().Printf(format, v...)
+}
+
+// Ctx returns the Logger associated with the ctx. If no logger
+// is associated, a disabled logger is returned.
+func Ctx(ctx context.Context) *zerolog.Logger {
+	return zerolog.Ctx(ctx)
+}
+
+func At(level *Event, id string, logtype string) *zerolog.Event {
+	return level.Str("id", id).Str("type", logtype)
+}

pkg/logger/logger_test.go

@@ -0,0 +1,191 @@
+package logger
+
+import (
+	"fmt"
+	"io"
+	stdlog "log"
+	"os"
+	"testing"
+
+	"fiskerinc.com/modules/testhelper"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/diode"
+)
+
+func setupTestLogOutput() diode.Writer {
+	wr := diode.NewWriter(os.Stdout, RING_BUFFER_SIZE, 0, func(missed int) {
+		Logger.Error().Msgf("logger dropped %d messages", missed)
+	})
+	logger := zerolog.New(wr)
+	Logger = &logger
+
+	return wr
+}
+
+func TestGetLogger(t *testing.T) {
+	GetLogger()
+	logLevel := zerolog.GlobalLevel().String()
+	expectedLevel := "info"
+	if logLevel != expectedLevel {
+		t.Errorf(testhelper.TestErrorTemplate, "TestLoggerInit", expectedLevel, logLevel)
+	}
+}
+
+func TestLoggerSetGlobalLevel(t *testing.T) {
+	logger := zerolog.New(io.Discard)
+	Logger = &logger
+
+	err := SetGlobalLevel("debug")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestLoggerSetGlobalLevel", nil, err)
+	}
+
+	logLevel := zerolog.GlobalLevel().String()
+	expectedLevel := "debug"
+
+	if logLevel != expectedLevel {
+		t.Errorf(testhelper.TestErrorTemplate, "TestLoggerSetGlobalLevel", expectedLevel, logLevel)
+	}
+}
+
+func TestLoggerSetGlobalLevelUnknown(t *testing.T) {
+	err := SetGlobalLevel("fisker")
+	if err == nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestLoggerSetGlobalLevelUnknown", "err", err)
+	}
+}
+
+func ExampleDebug() {
+	wr := setupTestLogOutput()
+	defer wr.Close()
+
+	Debug().Msg("Use this to record individual events")
+	// Output: {"level":"debug","message":"Use this to record individual events"}
+}
+
+func ExampleInfo() {
+	wr := setupTestLogOutput()
+	defer wr.Close()
+
+	Info().Msg("This is the default level")
+	// Output: {"level":"info","message":"This is the default level"}
+}
+
+func ExampleWarn() {
+	wr := setupTestLogOutput()
+	defer wr.Close()
+
+	Warn().Msg("Use this to notify of abnormal events/triggers")
+	// Output: {"level":"warn","message":"Use this to notify of abnormal events/triggers"}
+}
+
+func ExampleError() {
+	wr := setupTestLogOutput()
+	defer wr.Close()
+
+	Error().Err(fmt.Errorf("Use this as a substitute for panic/fatal")).Send()
+	// Output: {"level":"error","message":"Use this as a substitute for panic/fatal"}
+}
+
+func ExampleConfidential() {
+	wr := setupTestLogOutput()
+	defer wr.Close()
+
+	type User struct {
+		Name     string
+		Password string
+		Phone    string
+	}
+	type VehicleData struct {
+		Email string
+		VIN   string
+		Miles int
+		User1 User
+		User2 *User
+	}
+
+	confObject := &VehicleData{
+		Email: "dtaylor@fiskerinc.com",
+		VIN:   "5VST414",
+		Miles: 29500,
+		User1: User{
+			Name:     "Henrik Fisker",
+			Password: "ocean123!",
+			Phone:    "+45(123)456-7890",
+		},
+		User2: &User{
+			Name:     "Drew Taylor",
+			Password: "Emotion123!",
+			Phone:    "+1(123)456-7890",
+		},
+	}
+	Info().Confidential(confObject).Send()
+	// Output: {"level":"info","Email":"d******@fiskerinc.com","VIN":"5VST414","Miles":29500,"User1":{"Name":"Henrik Fisker","Password":"*********","Phone":"+4*(1**)4**-7***"},"User2":{"Name":"Drew Taylor","Password":"***********","Phone":"+1(1**)4**-7***"}}
+}
+
+func ExampleContext() {
+	wr := setupTestLogOutput()
+	defer wr.Close()
+
+	Info().
+		Str("vehicle", "Ocean").
+		Str("vin", "ABC123").
+		Int("miles", 8077).
+		Send()
+	// Output: {"level":"info","vehicle":"Ocean","vin":"ABC123","miles":8077}
+}
+
+func ExampleSublog() {
+	wr := setupTestLogOutput()
+	defer wr.Close()
+
+	sublogger := Logger.With().
+		Str("component", "test").
+		Logger()
+	sublogger.Info().Msg("Branch sublogs off root logger")
+	// Output: {"level":"info","component":"test","message":"Branch sublogs off root logger"}
+}
+
+func BenchmarkZeroLogDefault(b *testing.B) {
+	logger := zerolog.New(io.Discard)
+	for i := 0; i < b.N; i++ {
+		logger.Info().Msg("hello fisker")
+	}
+}
+
+func BenchmarkZeroLogWithTime(b *testing.B) {
+	logger := zerolog.New(io.Discard).With().Timestamp().Logger()
+	for i := 0; i < b.N; i++ {
+		logger.Info().Msg("hello fisker")
+	}
+}
+
+func BenchmarkZeroLogWithCaller(b *testing.B) {
+	logger := zerolog.New(io.Discard).With().Caller().Logger()
+	for i := 0; i < b.N; i++ {
+		logger.Info().Msg("hello fisker")
+	}
+}
+
+func BenchmarkZeroLogWithCallerAndTime(b *testing.B) {
+	logger := zerolog.New(io.Discard).With().Timestamp().Caller().Logger()
+	for i := 0; i < b.N; i++ {
+		logger.Info().Msg("hello fisker")
+	}
+}
+
+func BenchmarkNativeLogDefault(b *testing.B) {
+	stdlog.SetOutput(io.Discard)
+	for i := 0; i < b.N; i++ {
+		stdlog.Print("hello fisker")
+	}
+}
+
+func BenchmarkNativeLogWithFields(b *testing.B) {
+	stdlog.SetOutput(io.Discard)
+	stdlog.SetFlags(stdlog.Ldate | stdlog.Ltime | stdlog.Lmicroseconds | stdlog.Llongfile)
+	for i := 0; i < b.N; i++ {
+		stdlog.Print("hello fisker")
+	}
+}

pkg/logger/mask.go

@@ -0,0 +1,30 @@
+package logger
+
+import (
+	"regexp"
+	"strings"
+)
+
+// MaskEmail removes all identifying chars except first from email username
+func MaskEmail(s string) string {
+	tmp := strings.Split(s, "@")
+	addr := tmp[0]
+	domain := tmp[1]
+	if len(addr) <= 0 || len(domain) <= 0 {
+		return ""
+	}
+
+	addr = addr[:1] + strings.Repeat("*", len(addr)-1)
+	return addr + "@" + domain
+}
+
+// MaskPassword replaces all characters
+func MaskPassword(s string) string {
+	return strings.Repeat("*", len(s))
+}
+
+// MaskPhoneNumber replaces all nums except first from number string
+func MaskPhoneNumber(s string) string {
+	m := regexp.MustCompile(`\B\d`)
+	return m.ReplaceAllString(s, "*")
+}

pkg/logger/mask_test.go

@@ -0,0 +1,77 @@
+package logger
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestMaskEmailEasy(t *testing.T) {
+	email := "dtaylor@fiskerinc.com"
+	expected := "d******@fiskerinc.com"
+	masked := MaskEmail(email)
+
+	if masked != expected {
+		t.Errorf(testhelper.TestErrorTemplate, "TestMaskEmailEasy", expected, masked)
+	}
+}
+
+func TestMaskEmailMed(t *testing.T) {
+	email := "d.taylor@fiskerinc.com"
+	expected := "d*******@fiskerinc.com"
+	masked := MaskEmail(email)
+
+	if masked != expected {
+		t.Errorf(testhelper.TestErrorTemplate, "TestMaskEmailMed", expected, masked)
+	}
+}
+
+func TestMaskEmailHard(t *testing.T) {
+	email := "d!-taylor@fiskerinc.com"
+	expected := "d********@fiskerinc.com"
+	masked := MaskEmail(email)
+
+	if masked != expected {
+		t.Errorf(testhelper.TestErrorTemplate, "TestMaskEmailHard", expected, masked)
+	}
+}
+
+func TestMaskPassword(t *testing.T) {
+	password := "abc123!"
+	expected := "*******"
+	masked := MaskPassword(password)
+
+	if masked != expected {
+		t.Errorf(testhelper.TestErrorTemplate, "TestMaskPassword", expected, masked)
+	}
+}
+
+func TestMaskPhoneNumberEasy(t *testing.T) {
+	phone := "1234567890"
+	expected := "1*********"
+	masked := MaskPhoneNumber(phone)
+
+	if masked != expected {
+		t.Errorf(testhelper.TestErrorTemplate, "TestMaskPhoneNumberEasy", expected, masked)
+	}
+}
+
+func TestMaskPhoneNumberMed(t *testing.T) {
+	phone := "+1234567890"
+	expected := "+1*********"
+	masked := MaskPhoneNumber(phone)
+
+	if masked != expected {
+		t.Errorf(testhelper.TestErrorTemplate, "TestMaskPhoneNumberMed", expected, masked)
+	}
+}
+
+func TestMaskPhoneNumberHard(t *testing.T) {
+	phone := "+1(123)456-7890"
+	expected := "+1(1**)4**-7***"
+	masked := MaskPhoneNumber(phone)
+
+	if masked != expected {
+		t.Errorf(testhelper.TestErrorTemplate, "TestMaskPhoneNumberHard", expected, masked)
+	}
+}