Initial cloud-services repo - gateway service + pkg modules

pkg/dbc/models/can_message.go

@@ -0,0 +1,143 @@
+package models
+
+import (
+	"fmt"
+
+	"fiskerinc.com/modules/dbc/state"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/digitaltwin"
+)
+
+type CANMessageInterface interface {
+	AddSignal(signal CANSignal) CANMessageInterface
+	GetID() int
+	GetLength() int
+	ParseState(keyvaluesOut map[string]interface{}, vin string, timestampUSec int, incomingSignals []common.CANSignal, cached digitaltwin.DigitalTwinCacheInterface) error
+	Signal(position int, name string, convert func(value float64) interface{}) CANMessageInterface
+}
+
+// NewCANMessage creates a top level instance for parsing a command. The id is the can signal in which individual signals exists
+// length is the number of messages inside the message, i.e. 1318 TBOX_0x526 has 2 different signals: TBOX_GPSLongi and TBOX_GPSLati
+func NewCANMessage(id int, length int) CANMessageInterface {
+	return &CANMessage{
+		id:     id,
+		length: length,
+	}
+}
+
+type CANMessage struct {
+	id         int
+	length     int
+	signals    map[int]CANSignal
+	diagnostic map[int]DiagnosticSignal
+}
+
+func (c *CANMessage) GetID() int {
+	return c.id
+}
+
+func (c *CANMessage) GetLength() int {
+	return c.length
+}
+
+// position is the index of the signal in its list. Starting at zero
+func (c *CANMessage) Signal(position int, name string, convert func(value float64) interface{}) CANMessageInterface {
+	c.AddSignal(NewCANSignal(position, name, convert))
+	return c
+}
+
+func (c *CANMessage) AddSignal(signal CANSignal) CANMessageInterface {
+	m := c.getSignalsMap()
+	if _, ok := m[signal.Position]; ok {
+		panic(fmt.Sprintf("signal position %d already exists", signal.Position))
+	}
+
+	m[signal.Position] = signal
+
+	return c
+}
+
+func (c *CANMessage) Diagnostic(position int, name string, alert float64) CANMessageInterface {
+	c.AddDiagnostic(NewDiagnosticSignal(position, name, alert))
+	return c
+}
+
+func (c *CANMessage) AddDiagnostic(signal DiagnosticSignal) CANMessageInterface {
+	m := c.getDiagnosticMap()
+	if _, ok := m[signal.Position]; ok {
+		panic(fmt.Sprintf("diagnositic position %d already exists", signal.Position))
+	}
+
+	m[signal.Position] = signal
+
+	return c
+}
+
+func (c *CANMessage) getDiagnosticMap() map[int]DiagnosticSignal {
+	if c.diagnostic == nil {
+		c.diagnostic = map[int]DiagnosticSignal{}
+	}
+
+	return c.diagnostic
+}
+
+func (c *CANMessage) getSignalsMap() map[int]CANSignal {
+	if c.signals == nil {
+		c.signals = map[int]CANSignal{}
+	}
+
+	return c.signals
+}
+
+// cached: Checks if the new signal has the same value as it did the last time, if so don't update it
+func (c *CANMessage) ParseState(keyvaluesOut map[string]interface{}, vin string, timestampUSec int, incomingSignals []common.CANSignal, cached digitaltwin.DigitalTwinCacheInterface) error {
+	var can CANSignal
+	var ok bool
+
+	s := c.getSignalsMap()
+
+	for i, signal := range incomingSignals {
+		if can, ok = s[i]; !ok {
+			continue
+		}
+
+		if can.Name != signal.Name {
+			return IncorrectSignalError(can.Name, signal.Name)
+		}
+
+		val := can.ParseState(incomingSignals[i])
+
+		if cached != nil && cached.Exists(vin, signal.Name, val) {
+			continue
+		}
+		keyvaluesOut[signal.Name] = val
+
+		// Added updated time for each signals. Key format -> "${signal_name}:updated"
+		updatedAt, _ := state.SerializeTimestampUSec(timestampUSec)
+		keyvaluesOut[fmt.Sprintf("%s:%s", signal.Name, "updated")] = updatedAt
+	}
+	return nil
+}
+
+/*
+func (c *CANMessage) DiagnosticAlert(vin string, timestampUSec int, signals []common.CANSignal) ([]DiagnosticSignal, error) {
+	var flags []DiagnosticSignal
+
+	m := c.getDiagnosticMap()
+
+	for i, signal := range signals {
+		if diag, ok := m[i]; !ok {
+			continue
+		}
+
+		if diag.Name != signal.Name {
+			return nil, IncorrectSignalError(diag.Name, signal.Name)
+		}
+
+		append(flags, diag)
+	}
+
+	return nil, nil
+}
+*/

pkg/dbc/models/can_signal.go

@@ -0,0 +1,33 @@
+package models
+
+import (
+	"fiskerinc.com/modules/common"
+)
+
+func NewCANSignal(position int, name string, convert func(value float64) interface{}) CANSignal {
+	return CANSignal{
+		Position:     position,
+		Name:         name,
+		RedisKey:     name,
+		ConvertValue: convert,
+	}
+}
+
+type CANSignal struct {
+	Position     int
+	Name         string
+	RedisKey     string
+	ConvertValue func(value float64) interface{}
+}
+
+func (c *CANSignal) ParseState(signal common.CANSignal) interface{} {
+	return c.getValue(signal.Value)
+}
+
+func (c *CANSignal) getValue(value float64) interface{} {
+	if c.ConvertValue == nil {
+		return value
+	}
+
+	return c.ConvertValue(value)
+}

pkg/dbc/models/dbc_collection.go

@@ -0,0 +1,191 @@
+package models
+
+import (
+	"fmt"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/dbc/state"
+	"fiskerinc.com/modules/digitaltwin"
+	"fiskerinc.com/modules/grpc/kafka_grpc"
+	"fiskerinc.com/modules/logger"
+	"github.com/pkg/errors"
+)
+
+func GetShortKey(key string) string {
+	// the version (hash) is represented as hex
+	// we agreed to use 2 first bytes, so we need 4 characters of hex string
+	const length = 4
+
+	if len(key) < length {
+		return key
+	}
+
+	return key[:length]
+}
+
+type DBCCollectionInterface interface {
+	Get(dbcHash string) (DBCVersionInterface, error)
+	AddVersion(dbcHash string, dbcVersion DBCVersionInterface) DBCCollectionInterface
+	ParseState(vin string, payload *kafka_grpc.GRPC_BatchPayload, cached digitaltwin.DigitalTwinCacheInterface) (map[string]interface{}, error)
+	GetOrigKey(short string) (string, error)
+	GetHashesList() []string
+}
+
+func NewDBCCollection(c map[string]DBCVersionInterface) DBCCollectionInterface {
+	return &DBCCollection{
+		collections: c,
+	}
+}
+
+type DBCCollection struct {
+	collections map[string]DBCVersionInterface
+	keys        map[string]string
+	options     []DBCOption
+}
+
+func (d *DBCCollection) getCollectionMap() map[string]DBCVersionInterface {
+	if d.collections == nil {
+		d.collections = map[string]DBCVersionInterface{}
+	}
+
+	return d.collections
+}
+
+func (d *DBCCollection) getKeyMap() map[string]string {
+	if d.keys == nil {
+		d.keys = map[string]string{}
+	}
+
+	return d.keys
+}
+
+func (d *DBCCollection) GetHashesList() []string {
+	var hashs []string
+	for _, hash := range d.keys {
+		hashs = append(hashs, hash)
+	}
+
+	return hashs
+}
+
+func (d *DBCCollection) Get(dbcHash string) (DBCVersionInterface, error) {
+	c := d.getCollectionMap()
+	key := GetShortKey(dbcHash)
+	if db, ok := c[key]; ok {
+		return db, nil
+	}
+
+	return nil, errors.Errorf("DBC %s does not exists", dbcHash)
+}
+
+func (d *DBCCollection) AddVersion(dbcHash string, dbcVersion DBCVersionInterface) DBCCollectionInterface {
+	c := d.getCollectionMap()
+	m := d.getKeyMap()
+	key := GetShortKey(dbcHash)
+	if _, ok := c[key]; ok {
+		panic(fmt.Sprintf("DBC %s already exists", dbcHash))
+	}
+
+	c[key] = dbcVersion
+	m[key] = dbcHash
+
+	return d
+}
+
+func (d *DBCCollection) GetOrigKey(short string) (string, error) {
+	m := d.getKeyMap()
+	short = GetShortKey(short)
+	if key, ok := m[short]; ok {
+		return key, nil
+	}
+
+	return "", ErrInvalidDBC(short)
+}
+
+func (d *DBCCollection) ParseState(vin string, payload *kafka_grpc.GRPC_BatchPayload, cached digitaltwin.DigitalTwinCacheInterface) (map[string]interface{}, error) {
+	keyvalues := make(map[string]interface{})
+
+	dbc, err := d.Get(payload.Version)
+	if err != nil {
+		return nil, err
+	}
+
+	updated_at := d.processFrames(dbc, keyvalues, vin, payload.Data.Frames, cached)
+	d.processOptions(keyvalues, vin)
+
+	if len(keyvalues) > 0 {
+		serialized, err := state.SerializeTimestampUSec(updated_at)
+		if err != nil {
+			return keyvalues, err
+		}
+		keyvalues[state.UPDATED_AT] = serialized
+	}
+
+	return keyvalues, nil
+}
+
+// cached: Local memory and digital twin, check to see if the value for that digital twin value is already saved in its given value
+func (d *DBCCollection) processFrames(dbc DBCVersionInterface, keyvaluesOut map[string]interface{}, vin string, frames []*kafka_grpc.GRPC_CANFrame, cached digitaltwin.DigitalTwinCacheInterface) int {
+	var updated_at int
+
+	for _, msg := range frames {
+		incomingSignals, err := dbc.GenerateCANSignals(vin, msg)
+		if err != nil {
+			logger.At(logger.Warn(), common.TRex.Key(vin), "dbc").
+				Str("signal", fmt.Sprintf("%v", msg)).
+				Str("dbc", dbc.Hash()).
+				Err(err).Send()
+			continue
+		}
+
+		updated_at = int(msg.Epoch)
+		err = dbc.ParseState(int(msg.ID), keyvaluesOut, vin, updated_at, incomingSignals, cached)
+		if err != nil {
+			logger.At(logger.Warn(), common.TRex.Key(vin), "dbc").
+				Str("signal", fmt.Sprintf("%v", msg)).
+				Str("dbc", dbc.Hash()).
+				Err(err).Send()
+		}
+	}
+
+	return updated_at
+}
+
+func (d *DBCCollection) processOptions(values map[string]interface{}, vin string) error {
+	for _, option := range d.options {
+		err := option.Handler(values, vin)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+/*
+func (d *DBCCollection) DiagnosticAlert(vin string, payload *kafka_grpc.GRPC_BatchPayload) ([]DiagnosticSignal, error) {
+	var flags = make([]DiagnosticSignal, 0)
+
+	dbc, err := d.Get(payload.Version)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, msg := range payload.Data.Frames {
+		signals, err := dbc.GenerateCANSignals(vin, msg)
+		if err != nil {
+			logger.Warn().Str("id", vin).Err(err).Msgf("%+v", msg)
+			continue
+		}
+
+		alerts, err := dbc.DiagnosticAlert(int(msg.ID), vin, int(msg.Epoch), signals)
+		if err != nil {
+			logger.Warn().Str("id", vin).Err(err).Msgf("%+v", msg)
+		}
+
+		append(flags, alerts...)
+	}
+
+	return flags, err
+}
+*/

pkg/dbc/models/dbc_collection_test.go

@@ -0,0 +1,26 @@
+package models
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestDBCCollection_AddVersion(t *testing.T) {
+	hash := "abcdef1234562"
+	dbc := &DBCVersion{
+		diagnosticFlags: map[int]map[string]float64{},
+		state:           map[int]CANMessageInterface{},
+	}
+	col := DBCCollection{}
+	col.AddVersion(hash, dbc)
+	d, err := col.Get(hash)
+	assert.Equal(t, dbc, d)
+	assert.Equal(t, nil, err)
+
+	hList := col.GetHashesList()
+	assert.Equal(t, []string{hash}, hList)
+
+	oKey, err := col.GetOrigKey(GetShortKey(hash))
+	assert.Equal(t, hash, oKey)
+	assert.Equal(t, nil, err)
+}

pkg/dbc/models/dbc_options.go

@@ -0,0 +1,5 @@
+package models
+
+type DBCOption struct {
+	Handler func(values map[string]interface{}, vin string) error
+}

pkg/dbc/models/dbc_version.go

@@ -0,0 +1,186 @@
+package models
+
+import (
+	"encoding/base64"
+	"fmt"
+	"math"
+	"strings"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/digitaltwin"
+	"fiskerinc.com/modules/grpc/kafka_grpc"
+
+	can "github.com/Fisker-Inc/project-ai-can-go"
+	"github.com/Fisker-Inc/project-ai-can-go/pkg/descriptor"
+	"github.com/pkg/errors"
+)
+
+type DBCVersionInterface interface {
+	AddMsg(msg CANMessageInterface) DBCVersionInterface
+	DecodeMessage(msg *kafka_grpc.GRPC_CANFrame) ([]descriptor.DecodedSignal, error)
+	GenerateCANSignals(vin string, msg *kafka_grpc.GRPC_CANFrame) ([]common.CANSignal, error)
+	GetActiveDiagnosticFlags(vin string, msg *kafka_grpc.GRPC_CANFrame) ([]common.CANSignal, bool, error)
+	SetDiagnosticFlags(flags map[int]map[string]float64) DBCVersionInterface
+	ParseState(id int, keyvaluesOut map[string]interface{}, vin string, timestampUSec int, incomingSignals []common.CANSignal, cached digitaltwin.DigitalTwinCacheInterface) error
+	GetDatabase() *descriptor.Database
+	Hash() string
+}
+
+func NewDBCVersion(dbc *descriptor.Database, hash string) DBCVersionInterface {
+	if len(hash) < 3 {
+		panic("NewDBCVersion hash cannot be less than 3 characters")
+	}
+
+	return &DBCVersion{
+		dbc:  dbc,
+		hash: hash,
+	}
+}
+
+type DBCVersion struct {
+	state           map[int]CANMessageInterface
+	dbc             *descriptor.Database
+	diagnosticFlags map[int]map[string]float64
+	hash            string
+}
+
+func (d *DBCVersion) AddMsg(msg CANMessageInterface) DBCVersionInterface {
+	m := d.getMsgMap()
+	if _, ok := m[msg.GetID()]; ok {
+		panic(fmt.Sprintf("CAN message %d already exists", msg.GetID()))
+	}
+
+	m[msg.GetID()] = msg
+
+	return d
+}
+
+func (d *DBCVersion) getMsgMap() map[int]CANMessageInterface {
+	if d.state == nil {
+		d.state = map[int]CANMessageInterface{}
+	}
+
+	return d.state
+}
+
+func (d *DBCVersion) ParseState(id int, keyvaluesOut map[string]interface{}, vin string, timestampUSec int, incomingSignals []common.CANSignal, cached digitaltwin.DigitalTwinCacheInterface) error {
+	m := d.getMsgMap()
+	var msg CANMessageInterface
+	var ok bool
+
+	if msg, ok = m[id]; !ok {
+		return nil
+	}
+
+	if i := len(incomingSignals); i != msg.GetLength() {
+		return IncorrectSignalsLength(id, msg.GetLength(), i)
+	}
+
+	err := msg.ParseState(keyvaluesOut, vin, timestampUSec, incomingSignals, cached)
+
+	// TODO run options here
+
+	return err
+}
+
+func (d *DBCVersion) GenerateCANSignals(vin string, msg *kafka_grpc.GRPC_CANFrame) ([]common.CANSignal, error) {
+	var signals []common.CANSignal
+
+	decodedSignals, err := d.DecodeMessage(msg)
+	if err != nil {
+		return signals, err
+	}
+
+	for _, signal := range decodedSignals {
+		signals = append(signals, common.CANSignal{
+			VIN:       vin,
+			Timestamp: float64(msg.Epoch) * math.Pow(10, -6),
+			ID:        int(msg.ID),
+			Name:      signal.Signal.Name,
+			Value:     signal.Value,
+		})
+	}
+
+	return signals, nil
+}
+
+func (d *DBCVersion) DecodeMessage(msg *kafka_grpc.GRPC_CANFrame) ([]descriptor.DecodedSignal, error) {
+	var decodedSignals []descriptor.DecodedSignal
+
+	if msg.ID >= 2048 || msg.ID < 0 {
+		if GetInvalidCANIDCache().Exists(d.hash, msg.ID) {
+			return decodedSignals, nil
+		}
+		return decodedSignals, errors.Errorf("message id %v out of bounds", msg.ID)
+	}
+
+	message, ok := d.dbc.Message(uint32(msg.ID))
+	if !ok {
+		if GetInvalidCANIDCache().Exists(d.hash, msg.ID) {
+			return decodedSignals, nil
+		}
+
+		return decodedSignals, errors.Errorf(
+			"message id %v not found in %s, version: %s",
+			msg.ID, d.dbc.SourceFile, d.hash)
+	}
+
+	strippedData := strings.TrimRight(string(msg.Value), "=")
+	data, err := base64.RawStdEncoding.DecodeString(strippedData)
+	if err != nil {
+		return decodedSignals, errors.WithStack(err)
+	}
+
+	p := can.Payload{Data: data}
+	decodedSignals = message.Decode(&p)
+	return decodedSignals, nil
+}
+
+func (d *DBCVersion) GetActiveDiagnosticFlags(vin string, msg *kafka_grpc.GRPC_CANFrame) ([]common.CANSignal, bool, error) {
+	var flags []common.CANSignal
+
+	diagnosticSignals, ok := d.diagnosticFlags[int(msg.ID)]
+	if !ok {
+		return flags, false, nil
+	}
+
+	decodedSignals, err := d.DecodeMessage(msg)
+	if err != nil {
+		return flags, false, err
+	}
+
+	flags = make([]common.CANSignal, 0)
+	for _, signal := range decodedSignals {
+		faultVal, ok := diagnosticSignals[signal.Signal.Name]
+
+		if !ok {
+			continue
+		} else if signal.Value != faultVal {
+			continue
+		}
+
+		flags = append(flags, common.CANSignal{
+			VIN:       vin,
+			Timestamp: float64(msg.Epoch) * math.Pow(10, -6),
+			ID:        int(msg.ID),
+			Name:      signal.Signal.Name,
+			Value:     signal.Value,
+		})
+	}
+
+	return flags, len(flags) != 0, nil
+}
+
+func (d *DBCVersion) Hash() string {
+	return d.hash
+}
+
+// TODO this should be replaced by generated diagnostic alerts from the DBC
+func (d *DBCVersion) SetDiagnosticFlags(flags map[int]map[string]float64) DBCVersionInterface {
+	d.diagnosticFlags = flags
+	return d
+}
+
+func (d *DBCVersion) GetDatabase() *descriptor.Database {
+	return d.dbc
+}

pkg/dbc/models/diagnostic_signal.go

@@ -0,0 +1,16 @@
+package models
+
+func NewDiagnosticSignal(position int, name string, alert float64) DiagnosticSignal {
+	return DiagnosticSignal{
+		AlertValue: alert,
+		CANSignal: CANSignal{
+			Name:     name,
+			Position: position,
+		},
+	}
+}
+
+type DiagnosticSignal struct {
+	AlertValue float64
+	CANSignal
+}

pkg/dbc/models/errors.go

@@ -0,0 +1,21 @@
+package models
+
+import (
+	"github.com/pkg/errors"
+)
+
+func IncorrectSignalsLength(msgid int, expected int, received int) error {
+	return errors.Errorf("msg id %d incorrect number of signals in message expected %d received %d", msgid, expected, received)
+}
+
+func IncorrectSignalError(expected string, received string) error {
+	return errors.Errorf("wrong signal in message expected %s received %s", expected, received)
+}
+
+func IncorrectSignalValue(expected interface{}, received interface{}) error {
+	return errors.Errorf("wrong value in message expected one of %v received %v", expected, received)
+}
+
+func ErrInvalidDBC(version string) error {
+	return errors.Errorf("could not find %s within collection", version)
+}

pkg/dbc/models/invalid_id_cache.go

@@ -0,0 +1,47 @@
+package models
+
+import (
+	"fmt"
+	"sync"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/utils/envtool"
+)
+
+var (
+	max_invalid_cache    = envtool.GetEnvInt("MAX_INVALID_CANID_CACHE", 100)
+	invalidCANIDOnce     sync.Once
+	invalidCANIDInstance InvalidCANIDCacheInterface
+)
+
+func NewInvalidCANIDCache() InvalidCANIDCacheInterface {
+	return &InvalidCANIDCache{
+		ringMap: cache.NewRingMap(max_invalid_cache),
+	}
+}
+
+func GetInvalidCANIDCache() InvalidCANIDCacheInterface {
+	invalidCANIDOnce.Do(func() {
+		if invalidCANIDInstance == nil {
+			invalidCANIDInstance = NewInvalidCANIDCache()
+		}
+	})
+
+	return invalidCANIDInstance
+}
+
+func SetInvalidCANIDCache(instance InvalidCANIDCacheInterface) {
+	invalidCANIDInstance = instance
+}
+
+type InvalidCANIDCacheInterface interface {
+	Exists(dbc string, can_id int32) bool
+}
+
+type InvalidCANIDCache struct {
+	ringMap *cache.RingMap
+}
+
+func (i *InvalidCANIDCache) Exists(dbc string, can_id int32) bool {
+	return !i.ringMap.Set(fmt.Sprintf("%s:%d", dbc[0:4], can_id), false)
+}