Initial cloud-services repo - gateway service + pkg modules

.envrc

@@ -0,0 +1,25 @@
+# Auto-activate devbox
+eval "$(devbox generate direnv --print-envrc)"
+
+# Mini cluster defaults
+export DB_HOST=cloud-dev-rw.cnpg-system.svc.cluster.local
+export DB_NAME=cloud_dev
+export DB_USER=cloud_dev
+export DB_PASSWORD=cloud_dev_password
+export DB_SSLMODE=disable
+
+export MONGO_HOST=cloud-dev-svc.mongodb.svc.cluster.local
+export MONGO_PORT=27017
+export MONGO_USER=cloud_dev
+export MONGO_PASSWORD=cloud_dev_password
+export MONGO_DB_NAME=db
+
+export REDIS_HOST=cloud-dev.redis.svc.cluster.local
+export REDIS_PORT=6379
+
+export KAFKA_HOSTS=cloud-dev-kafka-bootstrap.kafka.svc.cluster.local:9092
+
+export OIDC_ISSUER=https://keycloak.mini.cloud.fiskerinc.com/realms/compute-auth
+export OIDC_JWK_URL=https://keycloak.mini.cloud.fiskerinc.com/realms/compute-auth/protocol/openid-connect/certs
+
+export VAULT_URL=http://vault.vault.svc.cluster.local:8200/v1

.gitignore

@@ -0,0 +1,35 @@
+# Binaries
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin/
+dist/
+
+# Test
+*.test
+*.out
+coverage.txt
+
+# Go
+vendor/
+go.work.sum
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Devbox
+.devbox/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Env
+.env
+.env.local
+*.local

Dockerfile

@@ -0,0 +1,35 @@
+# Multi-service Dockerfile with build caching
+# Usage: docker build --build-arg SERVICE=gateway -t cloud-gateway .
+
+ARG SERVICE=gateway
+
+# Build stage
+FROM golang:1.24-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /src
+
+# Cache dependencies first (changes less often)
+COPY go.work go.work.sum* ./
+COPY pkg/go.mod pkg/go.sum* ./pkg/
+COPY services/${SERVICE}/go.mod services/${SERVICE}/go.sum* ./services/${SERVICE}/
+
+RUN go mod download
+
+# Copy source and build
+COPY pkg/ ./pkg/
+COPY services/${SERVICE}/ ./services/${SERVICE}/
+
+WORKDIR /src/services/${SERVICE}
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /app .
+
+# Runtime stage
+FROM alpine:3.20
+
+RUN apk add --no-cache ca-certificates tzdata
+
+COPY --from=builder /app /app
+
+USER nobody:nobody
+ENTRYPOINT ["/app"]

README.md

@@ -0,0 +1,49 @@
+# cloud-services
+
+Go microservices for the vehicle cloud platform.
+
+## Quick Start
+```bash
+# Install devbox (if needed)
+curl -fsSL https://get.jetify.com/devbox | bash
+
+# Enter dev environment
+devbox shell
+
+# Run a service locally
+cd services/gateway
+go run .
+```
+
+## Structure
+```
+services/       # Individual Go microservices
+shared/         # Shared Go modules
+deploy/         # Kubernetes manifests (kustomize)
+  base/         # Base configs
+  overlays/     # Environment-specific (development, etc.)
+```
+
+## Services
+| Service | Description |
+|---------|-------------|
+| gateway | API gateway, routes requests |
+| auth | Authentication (Keycloak integration) |
+| ota | OTA update management |
+| depot | Vehicle registration & management |
+| attendant | Event processing |
+| cargo | Data ingestion to storage |
+| ditto | Digital twin state |
+| manufacture | Manufacturing integration |
+| aftersales | Aftersales/diagnostic services |
+
+## Local Development
+Services connect to:
+- PostgreSQL: `cloud-dev-rw.cnpg-system.svc:5432`
+- MongoDB: `cloud-dev-svc.mongodb.svc:27017`
+- Redis: `cloud-dev.redis.svc:6379`
+- Kafka: `cloud-dev-kafka-bootstrap.kafka.svc:9092`
+- Keycloak: `https://keycloak.mini.cloud.fiskerinc.com`
+
+## Deployment
+ArgoCD syncs from this repo. Push to main → auto-deploy to mini cluster.

deploy/base/configmap-common.yaml

@@ -0,0 +1,46 @@
+# Common environment config shared by all services
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloud-common-config
+  namespace: cloud-services
+data:
+  # PostgreSQL
+  DB_HOST: cloud-dev-rw.cnpg-system.svc.cluster.local
+  DB_PORT: "5432"
+  DB_NAME: cloud_dev
+  DB_USER: cloud_dev
+  DB_SSLMODE: disable
+  DB_POOLSIZE: "10"
+  
+  # MongoDB
+  MONGO_HOST: cloud-dev-svc.mongodb.svc.cluster.local
+  MONGO_PORT: "27017"
+  MONGO_DB_NAME: db
+  MONGO_ODX_DB_NAME: odx_db
+  MONGO_CLIENT_TIMEOUT: "60"
+  
+  # Redis
+  REDIS_HOST: cloud-dev.redis.svc.cluster.local
+  REDIS_PORT: "6379"
+  REDIS_IDLETIMEOUT_MS: "3600000"
+  REDIS_MAXIDLECONN: "10"
+  REDIS_MAXACTIVECONN: "10"
+  
+  # Kafka (Strimzi)
+  KAFKA_HOSTS: cloud-dev-kafka-bootstrap.kafka.svc.cluster.local:9092
+  KAFKA_SECURITY_PROTOCOL: PLAINTEXT
+  KAFKA_GO_BATCH_CONSUMER: "true"
+  KAFKA_BATCH_NUM_MESSAGES: "50000"
+  KAFKA_BATCH_SIZE: "1000000"
+  KAFKA_LINGER_MS: "50"
+  
+  # Auth (Keycloak)
+  OIDC_ISSUER: https://keycloak.mini.cloud.fiskerinc.com/realms/compute-auth
+  OIDC_JWK_URL: https://keycloak.mini.cloud.fiskerinc.com/realms/compute-auth/protocol/openid-connect/certs
+  
+  # Vault
+  VAULT_URL: http://vault.vault.svc.cluster.local:8200/v1
+  
+  # Logging
+  LOG_LEVEL: info

deploy/base/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - configmap-common.yaml

deploy/base/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cloud-services
+  labels:
+    app.kubernetes.io/part-of: cloud-platform

deploy/overlays/development/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: cloud-services
+
+resources:
+  - ../../base
+  - secrets.yaml
+  # Services (uncomment as migrated)
+  # - services/gateway/
+  # - services/auth/
+
+commonLabels:
+  environment: development

deploy/overlays/development/secrets.yaml

@@ -0,0 +1,21 @@
+# Dev secrets - in prod use external-secrets with Vault
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloud-db-credentials
+  namespace: cloud-services
+type: Opaque
+stringData:
+  DB_PASSWORD: cloud_dev_password
+  MONGO_PASSWORD: cloud_dev_password
+  REDIS_PASSWORD: ""
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloud-auth-credentials
+  namespace: cloud-services
+type: Opaque
+stringData:
+  OIDC_CLIENT_ID: ota-portal
+  OIDC_CLIENT_SECRET: ota-portal-secret-change-me

devbox.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/main/.schema/devbox.schema.json",
+  "packages": [
+    "go@1.24",
+    "gopls@latest",
+    "golangci-lint@latest",
+    "kubectl@latest",
+    "kustomize@latest",
+    "k9s@latest"
+  ],
+  "shell": {
+    "init_hook": [
+      "echo 'Cloud Services dev environment ready'",
+      "export GOWORK=off"
+    ],
+    "scripts": {
+      "test": "go test ./...",
+      "lint": "golangci-lint run ./...",
+      "build": "go build ./..."
+    }
+  }
+}

go.work

@@ -0,0 +1,6 @@
+go 1.24
+
+use (
+	./pkg
+	./services/gateway
+)

pkg/adminroles/roles.go

@@ -0,0 +1,37 @@
+package adminroles
+
+import "fiskerinc.com/modules/utils/envtool"
+
+// RoleID for groups
+type RoleID string
+type RoleMap map[string][]RoleID
+
+var (
+	RoleCreate              RoleID = RoleID(envtool.GetEnv("ROLE_CREATE", "REPLACE_ME"))
+	RoleReadOnly            RoleID = RoleID(envtool.GetEnv("ROLE_READ_ONLY", "REPLACE_ME"))
+	RoleDelete              RoleID = RoleID(envtool.GetEnv("ROLE_DELETE", "REPLACE_ME"))
+	RoleGenerateCertificate RoleID = RoleID(envtool.GetEnv("ROLE_GENERATE_CERTIFICATE", "REPLACE_ME"))
+	RoleManufacture         RoleID = RoleID(envtool.GetEnv("ROLE_MANUFACTURE", "REPLACE_ME"))
+	RoleCarDiagnostic       RoleID = RoleID(envtool.GetEnv("ROLE_CAR_DIAGNOSTIC", "REPLACE_ME"))
+	RoleSupplier            RoleID = RoleID(envtool.GetEnv("ROLE_SUPPLIER", "REPLACE_ME"))
+	RoleSupplierApprover    RoleID = RoleID(envtool.GetEnv("ROLE_SUPPLIER_APPROVER", "REPLACE_ME"))
+	RoleAfterSalesAccess    RoleID = RoleID(envtool.GetEnv("ROLE_AFTER_SALES_ACCESS", "REPLACE_ME"))
+	RoleAfterSalesAccessFSP RoleID = RoleID(envtool.GetEnv("ROLE_AFTER_SALES_ACCESS_FSP", "REPLACE_ME"))
+	RoleSAPIntegration      RoleID = RoleID(envtool.GetEnv("ROLE_SAP_INTEGRATION", "REPLACE_ME"))
+	RoleMagna               RoleID = RoleID(envtool.GetEnv("MAGNA_GROUP_ID", "REPLACE_ME"))
+	RoleManifestMigration   RoleID = RoleID(envtool.GetEnv("ROLE_MANIFEST_MIGRATION", "REPLACE_ME"))
+	RoleUpdateDeploy        RoleID = RoleID(envtool.GetEnv("ROLE_UPDATE_DEPLOY", "REPLACE_ME"))
+)
+
+func (r RoleMap) CopyAndMerge(m RoleMap) RoleMap {
+	nMap := make(RoleMap)
+	for k, v := range r {
+		nMap[k] = v
+	}
+
+	for k, v := range m {
+		nMap[k] = v
+	}
+
+	return nMap
+}

pkg/adminroles/roles_checker.go

@@ -0,0 +1,109 @@
+package adminroles
+
+import (
+	"strings"
+
+	"fiskerinc.com/modules/validator"
+
+	"github.com/pkg/errors"
+)
+
+const MissingPermissionError = "missing permission"
+
+type RolesChecker struct {
+	RequiredRoles []string
+}
+
+func (rc *RolesChecker) Check(roles []string) error {
+	if len(rc.RequiredRoles) != 0 {
+		return rc.HasRole(roles)
+	}
+
+	return nil
+}
+
+func (rc *RolesChecker) CheckGroups(groups interface{}) error {
+	if len(rc.RequiredRoles) != 0 {
+		roles, err := rc.parseRolesFromGroups(groups)
+		if err != nil {
+			return errors.New(MissingPermissionError)
+		}
+
+		return rc.HasRole(roles)
+	}
+
+	return nil
+}
+
+func (rc *RolesChecker) HasRole(roles []string) error {
+	err := validator.ValidateField(roles, "max=1024,dive,uuid")
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	for _, required := range rc.RequiredRoles {
+		if rc.containsRole(required, roles) {
+			return nil
+		}
+	}
+
+	return errors.New(MissingPermissionError)
+}
+
+func (rc *RolesChecker) parseRolesFromGroups(groups interface{}) ([]string, error) {
+
+	if str, ok := groups.(string); ok {
+		return rc.parseStringRoles(str)
+	}
+
+	if items, ok := groups.([]interface{}); ok && len(items) > 0 {
+		if _, ok := items[0].(string); ok {
+			return rc.parseSliceRoles(items)
+		}
+	}
+	return nil, errors.New(MissingPermissionError)
+}
+
+func (rc *RolesChecker) parseSliceRoles(groups []interface{}) ([]string, error) {
+	items := make([]string, len(groups))
+
+	for i, item := range groups {
+		items[i] = item.(string)
+	}
+
+	return items, nil
+}
+
+func (rc *RolesChecker) parseStringRoles(groups string) ([]string, error) {
+	clean := strings.Trim(strings.ReplaceAll(groups, " ", ""), "[]")
+	if len(clean) == 0 {
+		return nil, errors.New(MissingPermissionError)
+	}
+
+	items := strings.Split(clean, ",")
+	if items == nil || len(items) == 0 {
+		return nil, errors.New(MissingPermissionError)
+	}
+
+	return items, nil
+}
+
+func (rc *RolesChecker) containsRole(role string, groups []string) bool {
+	for _, group := range groups {
+		if role == group {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (rc *RolesChecker) SetRequiredRoles(roles []RoleID) {
+	result := make([]string, len(roles))
+
+	for i, role := range roles {
+		result[i] = string(role)
+	}
+
+	rc.RequiredRoles = result
+}

pkg/adminroles/roles_checker_test.go

@@ -0,0 +1,117 @@
+package adminroles_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/adminroles"
+	"fiskerinc.com/modules/testhelper"
+)
+
+const testRole = "7bcdcdb2-3279-44bf-a998-771bab4b33e1"
+const missingPermission = "missing permission"
+
+func TestCheck(t *testing.T) {
+	type testCase struct {
+		Name          string
+		Roles         []string
+		ExpectedError string
+	}
+
+	tests := []testCase{
+		{
+			Name:          "Nil roles",
+			Roles:         nil,
+			ExpectedError: missingPermission,
+		},
+		{
+			Name:          "Empty roles",
+			Roles:         []string{},
+			ExpectedError: missingPermission,
+		},
+		{
+			Name:          "Bad role",
+			Roles:         []string{"XXXXXXXXXXXXX"},
+			ExpectedError: "Key: '[0]' Error:Field validation for '[0]' failed on the 'uuid' tag",
+		},
+		{
+			Name:  "Bad role 2",
+			Roles: []string{testRole, "YYYYYY", "ZZZZZZZ"},
+			ExpectedError: `Key: '[1]' Error:Field validation for '[1]' failed on the 'uuid' tag
+Key: '[2]' Error:Field validation for '[2]' failed on the 'uuid' tag`,
+		},
+		{
+			Name:          "Good",
+			Roles:         []string{testRole},
+			ExpectedError: "",
+		},
+	}
+
+	checker := adminroles.RolesChecker{
+		RequiredRoles: []string{testRole},
+	}
+	for _, test := range tests {
+		err := checker.Check(test.Roles)
+		if err != nil && err.Error() != test.ExpectedError {
+			t.Errorf(testhelper.TestErrorTemplate, test.Name, test.ExpectedError, err.Error())
+		}
+		if test.ExpectedError == "" && err != nil {
+			t.Errorf(testhelper.TestErrorTemplate, test.Name, test.ExpectedError, err.Error())
+		}
+	}
+}
+
+func TestCheckGroup(t *testing.T) {
+
+	type testCase struct {
+		Name          string
+		Groups        string
+		ExpectedError string
+	}
+
+	tests := []testCase{
+		{
+			Name:          "No groups",
+			Groups:        "",
+			ExpectedError: missingPermission,
+		},
+		{
+			Name:          "No groups 2",
+			Groups:        "    ",
+			ExpectedError: missingPermission,
+		},
+		{
+			Name:          "Does not have group",
+			Groups:        "[8d8278a5-9c0e-4c7f-918a-811fd1d236e4, 6c3cf98d-0ada-48c6-ae94-b171cfa275fc, 56ef4bec-d739-4ddf-a003-ecc813085b8d, efcc3025-e2d8-4212-8227-805c7be39d2c, 5515a98f-4668-4121-8e8d-fee2825699cf, 86956a2f-8d46-47ff-9b29-f99079ae3c1d, c4d4361c-8882-47b4-8641-fd3ab68ae722]",
+			ExpectedError: missingPermission,
+		},
+		{
+			Name:          "Partial role id",
+			Groups:        "[7bcdcdb2-3279-44bf-a998]",
+			ExpectedError: "Key: '[0]' Error:Field validation for '[0]' failed on the 'uuid' tag",
+		},
+		{
+			Name:   "Bad group ids",
+			Groups: "[[8d8278a59c0e4c7f918a811fd1d236e4, 6c3cf98d-0ada-48c6-ae94-b171cfa275fcXXXXXXX]",
+			ExpectedError: `Key: '[0]' Error:Field validation for '[0]' failed on the 'uuid' tag
+Key: '[1]' Error:Field validation for '[1]' failed on the 'uuid' tag`,
+		},
+		{
+			Name:          "Has permission",
+			Groups:        "[8d8278a5-9c0e-4c7f-918a-811fd1d236e4, 6c3cf98d-0ada-48c6-ae94-b171cfa275fc, 56ef4bec-d739-4ddf-a003-ecc813085b8d, efcc3025-e2d8-4212-8227-805c7be39d2c, 5515a98f-4668-4121-8e8d-fee2825699cf, 86956a2f-8d46-47ff-9b29-f99079ae3c1d, c4d4361c-8882-47b4-8641-fd3ab68ae722, 7bcdcdb2-3279-44bf-a998-771bab4b33e1]",
+			ExpectedError: "",
+		},
+	}
+
+	checker := adminroles.RolesChecker{
+		RequiredRoles: []string{testRole},
+	}
+	for _, test := range tests {
+		err := checker.CheckGroups(test.Groups)
+		if err != nil && err.Error() != test.ExpectedError {
+			t.Errorf(testhelper.TestErrorTemplate, test.Name, test.ExpectedError, err.Error())
+		}
+		if test.ExpectedError == "" && err != nil {
+			t.Errorf(testhelper.TestErrorTemplate, test.Name, test.ExpectedError, err.Error())
+		}
+	}
+}

pkg/americanlease/americanLeaseFilter_test.go

@@ -0,0 +1,26 @@
+package americanlease
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/validator"
+	"fiskerinc.com/modules/vindecoder"
+)
+
+
+func TestCheckForInvalidVINs(t *testing.T){
+	invalidVINList := []string{}
+	for vin := range VINList {
+		if !validator.ValidateVINSimple(vin){
+			invalidVINList = append(invalidVINList, vin)
+			continue
+		}
+		if !vindecoder.VerifyVinCheckDigit(vin){
+			invalidVINList = append(invalidVINList, vin)
+		}
+	}
+	if len(invalidVINList) > 0 {
+		t.Fail()
+		t.Logf("%+v\n", invalidVINList)
+	}
+}

pkg/americanlease/invalidVINResponse.go

@@ -0,0 +1,25 @@
+package americanlease
+
+import (
+	"net/http"
+)
+
+// Write that the VIN was invalid if so
+func ValidVIN(vin string, w http.ResponseWriter)(blocked bool){
+	if !IsAL(vin){
+		w.WriteHeader(http.StatusForbidden)
+		blocked = true
+	}
+	return
+}
+
+func ValidVINs(vins []string, w http.ResponseWriter)(blocked bool){
+	for _, v := range vins {
+		if !IsAL(v){
+			w.WriteHeader(http.StatusForbidden)
+			blocked = true
+			break
+		}
+	}
+	return
+}

pkg/auth/get_users_list.go

@@ -0,0 +1,92 @@
+package auth
+
+import (
+	"strings"
+	"sync"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/utils/envtool"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/cognitoidentityprovider"
+)
+
+var (
+	ConsumerPoolId  string
+	cognitoOnce     sync.Once
+	cognitoInstance *cognitoidentityprovider.CognitoIdentityProvider
+)
+
+func GetUsersList(users []string) ([]common.JSONUserProfile, error) {
+	var userList []common.JSONUserProfile
+
+	for _, userid := range users {
+		cognitoClient := getAWS()
+
+		filter := strings.Replace("username = \"userId\"", "userId", userid, -1)
+
+		request := &cognitoidentityprovider.ListUsersInput{
+			Filter:     &filter,
+			UserPoolId: &ConsumerPoolId,
+		}
+		resp, err := cognitoClient.ListUsers(request)
+		if err != nil {
+			return nil, err
+		}
+
+		userList = append(userList, convertAWSUsers(resp.Users)...)
+	}
+
+	return userList, nil
+}
+
+func convertAWSUsers(users []*cognitoidentityprovider.UserType) []common.JSONUserProfile {
+	var userList []common.JSONUserProfile
+
+	for _, user := range users {
+		userList = append(userList, findUserAttributes(user))
+	}
+
+	return userList
+}
+
+func findUserAttributes(awsUser *cognitoidentityprovider.UserType) common.JSONUserProfile {
+	attributes := awsUser.Attributes
+	user := common.JSONUserProfile{}
+
+	user.UserName = *awsUser.Username
+	for _, attribute := range attributes {
+		switch *attribute.Name {
+		case "email":
+			user.Email = *attribute.Value
+		case "phone_number":
+			user.Phone = *attribute.Value
+		case "given_name":
+			user.FirstName = *attribute.Value
+		case "family_name":
+			user.LastName = *attribute.Value
+		}
+	}
+
+	return user
+}
+
+func getAWS() *cognitoidentityprovider.CognitoIdentityProvider {
+	cognitoOnce.Do(func() {
+		if cognitoInstance != nil {
+			return
+		}
+		logger.Info().Msg("Init cognito provider instance")
+		setPoolId()
+		mySession := session.Must(session.NewSession())
+		cognitoInstance = cognitoidentityprovider.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
+	})
+
+	return cognitoInstance
+}
+
+func setPoolId() {
+	//default to dev pool
+	ConsumerPoolId = envtool.GetEnv("CONSUMER_COGNITO_CLIENT_ID", "us-west-2_c7Qu91m3J")
+}

pkg/auth/user.go

@@ -0,0 +1,144 @@
+package auth
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+
+	"fiskerinc.com/modules/httpclient"
+	"fiskerinc.com/modules/jwt"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/utils/envtool"
+	"fiskerinc.com/modules/utils"
+)
+
+var getUserURL string = envtool.GetEnv("AUTH_GET_USER", "https://dev-auth.fiskerdps.com/auth/me")
+
+func AppendUserMiddleware(next http.HandlerFunc, apiCalls queries.APICallsInterface) http.HandlerFunc {
+	wrapper := func(w http.ResponseWriter, r *http.Request) {
+		_, err := jwt.GetAuthorizationHeader(r)
+		if err != nil {
+			logger.Warn().Err(err).Msgf("token invalid %s %s", r.Method, r.RequestURI)
+			utils.RespError(w, http.StatusUnauthorized, err.Error())
+			return
+		}
+
+		// go to auth to get user information
+		req, _ := http.NewRequest("GET", getUserURL, nil)
+		req.Header.Set("Authorization", r.Header.Get("Authorization"))
+		resp, err := httpclient.Do(req)
+		if err != nil {
+			logger.Warn().Err(err).Msgf("Unable to fetch user %s %s", r.Method, r.RequestURI)
+			utils.RespError(w, http.StatusUnauthorized, err.Error())
+			return
+		}
+		defer resp.Body.Close()
+
+		if resp != nil && resp.StatusCode != 200 {
+			logger.Warn().Err(err).Msgf("Unable to fetch user %s %s", r.Method, r.RequestURI)
+			if err != nil {
+				utils.RespError(w, http.StatusUnauthorized, err.Error())
+			} else {
+				utils.RespError(w, http.StatusUnauthorized, resp.Status)
+			}
+			return
+		}
+
+		user, err := extractUserAttributes(resp)
+		if err != nil {
+			logger.Warn().Err(err).Msgf("Unable to parse user response %s %s", r.Method, r.RequestURI)
+			utils.RespError(w, http.StatusUnauthorized, err.Error())
+			return
+		}
+
+		err = logAPICall(user, r.RequestURI, r.Method, apiCalls)
+		if err != nil {
+			logger.Warn().Msgf("Call log %s %s '%v'", r.Method, r.RequestURI, err)
+		}
+
+		ctx := r.Context()
+		ctx = context.WithValue(ctx, "identity", user)
+
+		next.ServeHTTP(w, r.WithContext(ctx))
+	}
+	return wrapper
+}
+
+func AppendUserTokenMiddleware(next http.HandlerFunc, apiCalls queries.APICallsInterface) http.HandlerFunc {
+	wrapper := func(w http.ResponseWriter, r *http.Request) {
+		token, err := jwt.GetAuthorizationHeader(r)
+		if err != nil {
+			logger.Warn().Err(err).Msgf("token invalid %s %s", r.Method, r.RequestURI)
+			utils.RespError(w, http.StatusUnauthorized, err.Error())
+			return
+		}
+
+		valid := jwt.NewJWTValidator("")
+		_, err = valid.ValidateToken(token.Token)
+		if err != nil {
+			logger.Warn().Err(err).Msgf("token invalid %s %s", r.Method, r.RequestURI)
+			utils.RespError(w, http.StatusUnauthorized, err.Error())
+			return
+		}
+
+		payload, err := jwt.GetPayload(token.Token)
+		if err != nil {
+			logger.Warn().Err(err).Msgf("token invalid %s %s", r.Method, r.RequestURI)
+			utils.RespError(w, http.StatusUnauthorized, err.Error())
+			return
+		}
+
+		err = logAPICall(payload, r.RequestURI, r.Method, apiCalls)
+		if err != nil {
+			logger.Warn().Msgf("Call log %s %s '%v'", r.Method, r.RequestURI, err)
+		}
+
+		ctx := r.Context()
+		ctx = context.WithValue(ctx, "identity", payload)
+
+		next.ServeHTTP(w, r.WithContext(ctx))
+	}
+	return wrapper
+}
+
+func extractUserAttributes(resp *http.Response) (map[string]interface{}, error) {
+	user := make(map[string]interface{})
+	err := json.NewDecoder(resp.Body).Decode(&user)
+	if err != nil {
+		return nil, err
+	}
+
+	username, ok := user["id"]
+	if !ok {
+		return nil, errors.New("invalid user token")
+	}
+
+	user["username"] = username
+	return user, nil
+}
+
+func logAPICall(payload map[string]interface{}, uri string, method string, apiCalls queries.APICallsInterface) error {
+	var (
+		username string
+		ok       bool
+	)
+
+	if username, ok = payload["username"].(string); !ok || username == "" {
+		return nil
+	}
+
+	endpoint, _, _ := strings.Cut(uri, "?")
+	_, err := apiCalls.Insert(common.APICall{
+		ClientID:   username,
+		AccessType: common.AccessTypeJWT,
+		Endpoint:   endpoint,
+		Method:     method,
+	})
+
+	return err
+}

pkg/auth/user_consent_token.go

@@ -0,0 +1,140 @@
+package auth
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"fiskerinc.com/modules/httpclient"
+	"fiskerinc.com/modules/jwt"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/utils/envtool"
+	"github.com/pkg/errors"
+)
+
+var cognitoUserConsentClientID = envtool.GetEnv("COGNITO_USER_CONSENT_CLIENT_ID", "REPLACE_ME")
+var cognitoUserConsentClientSecret = envtool.GetEnv("COGNITO_USER_CONSENT_CLIENT_SECRET", "REPLACE_ME")
+var cognitoUserConsentAuthURL = envtool.GetEnv("COGNITO_USER_CONSENT_AUTH_URL", "REPLACE_ME")
+
+var CognitoUserConsentJWT CognitoJWT
+
+type CognitoJWT struct {
+	token      string
+	expiration *time.Time
+	mu         sync.RWMutex
+}
+
+func (c *CognitoJWT) GetUserConsentToken() (string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// check for an unexpired token. if it is there, return it
+	if c.token != "" && time.Now().Before(*c.expiration) {
+		return c.token, nil
+	}
+
+	// make a request to Cognito to get the token
+	authResponse, err := RequestUserConsentToken()
+	if err != nil {
+		return "", err
+	}
+
+	// cache the token and expire time
+	c.token = authResponse.AccessToken.JWTToken
+	c.expiration = &authResponse.ExpireTime
+
+	return c.token, nil
+}
+
+func RequestUserConsentToken() (AuthResponse, error) {
+	var resp AuthTokens
+	var tokens AuthResponse
+
+	tokenReq, err := getUserConsentTokenRequest()
+	if err != nil {
+		return tokens, err
+	}
+
+	tokenRes, err := httpclient.Do(tokenReq)
+	if err != nil {
+		return tokens, err
+	}
+	defer tokenRes.Body.Close()
+
+	err = json.NewDecoder(tokenRes.Body).Decode(&resp)
+	if err != nil {
+		logger.Error().Err(err).Send()
+	}
+
+	if len(resp.Error) > 0 {
+		return tokens, errors.New(resp.Error)
+	}
+
+	return getAuthResponse(&resp), nil
+}
+
+// getUserConsentTokenRequest returns http request to exchange code for a user consent token
+func getUserConsentTokenRequest() (*http.Request, error) {
+	basicAuth := base64.StdEncoding.EncodeToString([]byte(strings.Join([]string{cognitoUserConsentClientID, ":", cognitoUserConsentClientSecret}, "")))
+
+	body := url.Values{
+		"client_id":  {cognitoUserConsentClientID},
+		"grant_type": {"client_credentials"},
+	}
+
+	r, err := http.NewRequest(http.MethodPost, getUserConsentTokenURL(), strings.NewReader(body.Encode()))
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	r.Header.Add("Authorization", fmt.Sprintf("Basic %s", basicAuth))
+	r.Header.Add("Content-type", "application/x-www-form-urlencoded")
+
+	return r, nil
+}
+
+// GetAuthResponse converts AuthTokens response from Cognito to include decoded payload
+func getAuthResponse(data *AuthTokens) AuthResponse {
+	var result AuthResponse
+
+	result.AccessToken.JWTToken = data.AcessToken
+	result.IDToken.JWTToken = data.IDToken
+	result.RefreshToken.Token = data.RefreshToken
+
+	// calculate the expire time of the token, leaving off one minute of wiggle room
+	result.ExpireTime = time.Now().Add(time.Duration(data.ExpiresIn)*time.Second - time.Minute)
+
+	return result
+}
+
+func getUserConsentTokenURL() string {
+	return cognitoUserConsentAuthURL + "/oauth2/token"
+}
+
+// AuthTokens json response for auth tokens
+type AuthTokens struct {
+	AcessToken   string `json:"access_token"`
+	RefreshToken string `json:"refresh_token,omitempty"`
+	IDToken      string `json:"id_token,omitempty"`
+	TokenType    string `json:"token_type"`
+	ExpiresIn    int64  `json:"expires_in"`
+	Error        string `json:"error,omitempty"`
+}
+
+// JWTResponse json response
+type JWTResponse struct {
+	JWTToken string `json:"jwtToken"`
+}
+
+// AuthResponse json response
+type AuthResponse struct {
+	AccessToken  JWTResponse   `json:"accessToken,omitempty"`
+	IDToken      JWTResponse   `json:"idToken,omitempty"`
+	RefreshToken jwt.AuthToken `json:"refreshToken,omitempty"`
+	ExpireTime   time.Time     `json:"expire_time"`
+}

pkg/auth/user_consent_token_test.go

@@ -0,0 +1,57 @@
+package auth_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	auth "fiskerinc.com/modules/auth"
+	"fiskerinc.com/modules/httpclient"
+	"fiskerinc.com/modules/httpclient/mock"
+	"fiskerinc.com/modules/testhelper"
+)
+
+const responseUserConsentJSON = `{"access_token":"eyJraWQiOiJqSXowUVRjc0tDVCtoeEd6MlMwK0NoUHlON3c4cmlQXC9sNm1xekFYUmw2bz0iLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIyZGQ2ZmVkOS1lNTgyLTQ1MWItYTkzYi01Yjk0MTBkZmJjNDMiLCJjb2duaXRvOmdyb3VwcyI6WyJ1cy13ZXN0LTJfQVd3akxYeW0yX0F6dXJlQUQiXSwidG9rZW5fdXNlIjoiYWNjZXNzIiwic2NvcGUiOiJodHRwczpcL1wvZmlza2VyaW5jLmNvbVwvb3RhdXBkYXRlLnJlYWQgaHR0cHM6XC9cL2Zpc2tlcmluYy5jb21cL290YXVwZGF0ZS5jcmVhdGUgb3BlbmlkIGVtYWlsIiwiYXV0aF90aW1lIjoxNjEzNjA3NTc2LCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAudXMtd2VzdC0yLmFtYXpvbmF3cy5jb21cL3VzLXdlc3QtMl9BV3dqTFh5bTIiLCJleHAiOjE2MTM2MTExNzYsImlhdCI6MTYxMzYwNzU3NiwidmVyc2lvbiI6MiwianRpIjoiYzUyNjI0YjItYmJkYi00N2RiLTllNTgtOGU5ZmU3Yjg1ODMxIiwiY2xpZW50X2lkIjoiN2NrMnRmb3FhdmM3MmM0NWhoN3RnZTQya2QiLCJ1c2VybmFtZSI6ImF6dXJlYWRfand1QGZpc2tlcmluYy5jb20ifQ.FvlES5AgjhymQKnHP41D2Ude0Ten6L8REBRXTyu5dyWGrG4vTfBGoxlkGE2-MEFc0s6uhbdST_E2Mc5QNlXG47ibK14tFl6kOqDd74TCfg5sWghb_nSjC-M769eUHQSQcs4L8jcnEt0bjqMmPtt8lZwu3VS7mkSRXD6_hX43rPLGUpMaz5RqKlfHX8YUyD6UnENW9Gg3zonPRsPWVtupc494B_pSZGuFs-jVzBDgb_SdrGt5wb3GazsNcB8KeAf0m0QoEiApsCYxKGUG9eQZw_CAUrhCj9mFT-xJuyvEp0t6B8HDHrdW4mIHblKqhZok1mPwCntJmOfyOs3niNaILg","id_token":"eyJraWQiOiJlUTNuZFJLaUVcL084VUZ5RHFsYjN0S1RzWG00SzVPMlc4NXd3VWkzT2tNZz0iLCJhbGciOiJSUzI1NiJ9.eyJhdF9oYXNoIjoiMHFmcmdyVlZfOW1XRWp5MVdOZDl5QSIsInN1YiI6IjJkZDZmZWQ5LWU1ODItNDUxYi1hOTNiLTViOTQxMGRmYmM0MyIsImNvZ25pdG86Z3JvdXBzIjpbInVzLXdlc3QtMl9BV3dqTFh5bTJfQXp1cmVBRCJdLCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsImlzcyI6Imh0dHBzOlwvXC9jb2duaXRvLWlkcC51cy13ZXN0LTIuYW1hem9uYXdzLmNvbVwvdXMtd2VzdC0yX0FXd2pMWHltMiIsImNvZ25pdG86dXNlcm5hbWUiOiJhenVyZWFkX2p3dUBmaXNrZXJpbmMuY29tIiwiYXVkIjoiN2NrMnRmb3FhdmM3MmM0NWhoN3RnZTQya2QiLCJpZGVudGl0aWVzIjpbeyJ1c2VySWQiOiJqd3VAZmlza2VyaW5jLmNvbSIsInByb3ZpZGVyTmFtZSI6IkF6dXJlQUQiLCJwcm92aWRlclR5cGUiOiJTQU1MIiwiaXNzdWVyIjoiaHR0cHM6XC9cL3N0cy53aW5kb3dzLm5ldFwvNWFhNGI2NDAtYzlmYy00YTliLWIzYTMtZDRhN2QwMDhmYjVlXC8iLCJwcmltYXJ5IjoidHJ1ZSIsImRhdGVDcmVhdGVkIjoiMTYxMjkwMjQxMzM4MyJ9XSwidG9rZW5fdXNlIjoiaWQiLCJhdXRoX3RpbWUiOjE2MTM2MDc1NzYsImV4cCI6MTYxMzYxMTE3NiwiaWF0IjoxNjEzNjA3NTc2LCJlbWFpbCI6Imp3dUBmaXNrZXJpbmMuY29tIn0.NbEWEgX48Z-zz3gREEH44OpnvhoYDcm9RlVdqKVoSJ777g0A0LDpGwz7UGcqvZLeQLPsHaMyV8-sblLvKQvpsenJfq81XddVWCAqI55VCdbnouCphIDYOEPNbWs9ORdrXxciALTt1AAehsF0dTDG0V5fce5Vku2qZZbpELdq9r4CBJQXWtFiV8lUaZPEMNJbZVdh1KjwJSpeF8CtJGKUXIIm6tAYVVKc27YWgxe2fh3zhke5MUnGYrb98-RLmDUwpUQ4eBnXu9gtA-9qIpOumXkftogWpeNZ7Rc0tAI8ZvOmG8plFyYoRMrKuC4kECeUdrsRJlCv4ijpK_L7GwEL9Q","refresh_token":"eyJjdHkiOiJKV1QiLCJlbmMiOiJBMjU2R0NNIiwiYWxnIjoiUlNBLU9BRVAifQ.LHo8ysGz7T3sJtwf8aHpWFzH8B84yDvfL4Q0YuRd0kfKSA51z4hKFPSLpo4PiFodJ-VPugJQWfSYXXpe4Tjd3bdTH-oYDJcJvRHIV3ZIID0EApt53lkxsFWV_9b33bltLYyJ7DnclQq1GnfgohDhD9F2CpnN3Xa-ntVmF9ntLe6wxZvk_zdBlbhIPwCc4FuPDIB_skNaciWCzU9LUfzvcfZJAR8KztM8ofDm3YJGZrRJltz6In78ZlN1sIlFuPSIRy56sg3yG3wMfe9Lrst0VacG_2fy6Ccg9VuLqD_xnzMmzjwMF9PGdnO5DlCblWwrHsDE6FkTuDy7ojnPJpPJlw.gzFbDwAmMKp-4eiE.AfY0IecXygmkDUkUGIIG7JmhBSzk4VD_sEAwuTeOufKD_duvNXFTQYNU_QvDc7M-9Vssbbb35dMMw3KLxW2IbC7fll8lNvHHMm1gkxlVxK1h5uRhmgt0q7tyMwLw1iKUDqOa177faHZJISN_gvfh-rlbNswswDGU061dyFh-w6Ck8SXoPnWfp9GxZJBgxzZ5uBV1D7_1bAghqWYNMsMUTSvOYyeWvVJHap-gjtGc491Vf97z6mh9PDBvIi734D90NbV5idZ11CCW7liI5L7kgRwuHZVxiu_NpkPED7dWcaBhOATur4r3P28U39JC5P5FD4JXlqyPl9FXVBkW049E1vdJrrkV3IbiqUMVXlkUeq6G87YUTdmt8qRPgiOc-G6g84RxSPQE55uojbuSSlON2CKZYmSmFVM0X7bBU42wP1wNP7Jq3LTjHcj4rOaN1ozffJxyGs54r7NP4D9u3nt2ozNkjk_DNK3UmxDPaQtZAtFO1d-T7UXv2BvzoCN2LGilzxVi04p9LcvoTDzI5GUY9OsjGUsdSZJvISylHAMMDi8nSxsBBSPD18fzV0tLhdjGM0-XljiM4bjZWNR4Nvraus33p8U4k5lmn2bx13JfHvDa3Zqf_aK57lam6Zf_6mvAK4I7A40WmiolJCxeEeDD54ljF0kAluT4sw6sxVY8It80A95TGFd0lm5e-tGrFKIoqRyPV5uwzzz3XT1HVPJda1ufdGhSUj8slsyqTUrnphh6JWbRfA9mrLdKQKuqM3xEslAwYZOhX6qOzADbo5WQMneTwn34QixMT4A6imaDBc6P4cOaLo7hNyS3e1h6SfwigEX9H42wkC3TWOiITakFq3tKkVwahMkdeds_uxloNoeicdGePjob6BfU8xq0IKxJh9UoeCsSX4KVtIrErHyYuoU-_ENZXYArSwfqorKgdjmQAa13NQjOiHzpgA5HngSCK1xy6zq9NvNA7hUe9O0gTqrFZDsYhRWSYEuOt5QpxJYalPGKIPXlsUJOURfR_J0iT52UxiOZIuXmpk-X8fgDhM_0fZm8GQ2GaIsf3nR49h7QnZRG9azTZV5q9Bs0bqPvP7wRL5xenByeIBsdwP6Bwaqd3n5BkFn-LE7eo6UPn_9o7Gx3g9VlN8pG7SIo_3a07L0yauJIO4ahL5aC07uBCLu3pJQW0ftlVpLdAA8gh3XPhfvOuH4XV7yU1fQhqGKich1hhHx2dFHyVr5mJPxc1VQvAyyAhxjvyQ2TTLfvSiYLOP1vFCVjUwb_RjF5tuh3ArH9IlXH9kIBVQbiSOWfgQ1PqD-go4jqDR_ie3aGc5Fm8Vd6lSh_2HW2GR0Ht0ASoCbl3C5roXCRkyTTaGl3nX6uwfg.thw4B0ug4OIZsZrDzCtJcQ","expires_in": 3600,"token_type": "Bearer"}`
+
+func TestHandleUserConsentToken(t *testing.T) {
+	setupUserConsentMockAuthClient()
+
+	authResp, err := auth.RequestUserConsentToken()
+	if err != nil {
+		t.Errorf("should not return an error")
+	}
+
+	const someValueCondition = "Some value"
+
+	if len(authResp.AccessToken.JWTToken) == 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "Access token", someValueCondition, authResp.AccessToken.JWTToken)
+	}
+
+	if len(authResp.IDToken.JWTToken) == 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "ID token", someValueCondition, authResp.IDToken.JWTToken)
+	}
+
+	if len(authResp.RefreshToken.Token) == 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "Refresh token", someValueCondition, authResp.RefreshToken.Token)
+	}
+
+	token, err := auth.CognitoUserConsentJWT.GetUserConsentToken()
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	if token != authResp.AccessToken.JWTToken {
+		t.Errorf(testhelper.TestErrorTemplate, "Access token", someValueCondition, authResp.AccessToken.JWTToken)
+	}
+}
+
+func setupUserConsentMockAuthClient() {
+	httpclient.Client = &mock.Client{
+		DoFunc: func(r *http.Request) (*http.Response, error) {
+			return &http.Response{
+				Body: ioutil.NopCloser(bytes.NewBufferString(responseUserConsentJSON)),
+			}, nil
+
+		},
+	}
+}

pkg/azurestoragecontainer/asc.go

@@ -0,0 +1,241 @@
+package azurestoragecontainer
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"fiskerinc.com/modules/logger"
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/pkg/errors"
+)
+
+// This set of modules will contain the ability to get the list of files from a container, and create a sas link to download them from
+
+var azureLogsBlobPath = "https://%s.blob.core.windows.net/%s"
+
+// For your module to use this they need a CollectionManagement. These store your connection information to
+// azure. Leave it up to the user to not recreate everytime
+type CollectionManagement struct {
+	SharedKeyCredential *azblob.SharedKeyCredential
+	BaseLink            *url.URL  // AzureAccount.blob.core.windows.net/LogsContainerName
+	cachedTokenTime     time.Time // The Time we last got the token
+	cachedToken         string    // Once a day get a new access token
+	cmci                CollectionManagementConnectionInformation
+}
+
+type CollectionManagementConnectionInformation struct {
+	AzureAccount       string
+	AzureContainerName string
+	AzureAccountKey    string
+}
+
+// Need somekind of one time sync thing. Lets worry about it later. Probably each service needs its own
+func NewCollectionConnection(cmci CollectionManagementConnectionInformation) (cm *CollectionManagement, err error) {
+	collect := CollectionManagement{}
+
+	link := makeAzureBlobLink(cmci.AzureAccount, cmci.AzureContainerName)
+	parsedURL, err := url.Parse(link)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	collect.BaseLink = parsedURL
+
+	cred, err := azblob.NewSharedKeyCredential(cmci.AzureAccount, cmci.AzureAccountKey)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	collect.SharedKeyCredential = cred
+
+	collect.cmci = cmci
+
+	return &collect, err
+}
+
+type FilePath struct {
+	Path       string `json:"Path,omitempty"`
+	File       bool
+	UnderPaths []*FilePath `json:"UnderPaths,omitempty"`
+}
+
+// This is working code to get the full set of all the different files. I am unsure why I wrote this for this ticket,
+// but could be used to grey out the calender on the ota-portal
+
+// Generate a file path for all the files
+// Prefix is the possible begging file path: e.g. vin/year will get all the files under that year folder
+// rootOnly will only get the root files instead of nested folders
+func (cm *CollectionManagement) GetFolderStruct(prefix string, rootOnly bool) (fp *FilePath, err error) {
+	fp, err = cm.startRecursiveFilePathSearch(prefix, rootOnly)
+	return
+}
+
+// If prefix is empty, start at the root
+func (cm *CollectionManagement) startRecursiveFilePathSearch(prefix string, rootOnly bool) (startPath *FilePath, err error) {
+	cred := cm.SharedKeyCredential
+	parsedURL := cm.BaseLink
+	containerURL := azblob.NewContainerURL(*parsedURL, azblob.NewPipeline(cred, azblob.PipelineOptions{}))
+
+	//marker := azblob.Marker{}
+	startPath = &FilePath{}
+	startPath.Path = prefix
+	startPath.UnderPaths = make([]*FilePath, 0)
+	if prefix != "" {
+		prefix = prefix + "/"
+	}
+
+	marker := azblob.Marker{}
+	for marker.NotDone() {
+		// This should probably be outside the for loop?
+		hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{
+			Prefix: prefix,
+		})
+		if err != nil {
+			err = errors.WithStack(err)
+			return startPath, err
+		}
+		// BlobItems are actual files
+		for _, file := range hierarchBlob.Segment.BlobItems {
+			startPath.UnderPaths = append(startPath.UnderPaths, &FilePath{Path: file.Name[len(prefix):], File: true})
+		}
+		if !rootOnly {
+			for _, path := range hierarchBlob.Segment.BlobPrefixes {
+				startPath.UnderPaths = append(startPath.UnderPaths, recursiveFilePathSearch(path.Name, containerURL))
+			}
+		}
+
+		marker = hierarchBlob.NextMarker
+	}
+	return
+}
+
+func makeAzureBlobLink(azureAccountName, azureContainerName string) string {
+	link := fmt.Sprintf(
+		azureLogsBlobPath,
+		azureAccountName,
+		azureContainerName,
+	)
+	return link //fmt.Sprintf("%s  /   %s", link, sasToken), err
+}
+
+// We start at the path and explore all those children
+// Prefix is the path
+func recursiveFilePathSearch(prefix string, containerURL azblob.ContainerURL) (fp *FilePath) {
+	fp = &FilePath{
+		UnderPaths: make([]*FilePath, 0),
+	}
+
+	paths := strings.Split(strings.TrimRight(prefix, "/"), "/")
+	fp.Path = paths[len(paths)-1]
+
+	marker := azblob.Marker{}
+	for marker.NotDone() {
+		hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{
+			Prefix: prefix,
+		})
+		if err != nil {
+			return
+		}
+		// BlobItems are actual files
+		for _, file := range hierarchBlob.Segment.BlobItems {
+			_, nextFile := filepath.Split(file.Name)
+			fp.UnderPaths = append(fp.UnderPaths, &FilePath{Path: nextFile, File: true})
+			//file.Name
+		}
+		//path.name includes the whole path
+		for _, path := range hierarchBlob.Segment.BlobPrefixes {
+			fp.UnderPaths = append(fp.UnderPaths, recursiveFilePathSearch(path.Name, containerURL))
+		}
+		marker = hierarchBlob.NextMarker
+	}
+	return
+}
+
+// Returns a list of full file paths
+func (fp *FilePath) ReturnFilePaths() []string {
+	filePaths := make([]string, 0)
+
+	for _, uFP := range fp.UnderPaths {
+		if uFP.File {
+			filePaths = append(filePaths, fp.Path+uFP.Path)
+		} else {
+			morePaths := uFP.ReturnFilePaths()
+			for x := range morePaths {
+				filePaths = append(filePaths, fp.Path+morePaths[x])
+			}
+		}
+	}
+	return filePaths
+}
+
+// If returns true, remove the file
+type FileFilter func(fileName string) (remove bool)
+
+// Assuming FilePath is not a file itself. Only its children are
+func (fp *FilePath) FilterFiles(ff FileFilter) {
+	fp.filterFilesRecursive(ff)
+}
+
+// If the child item returns true to be filtered, we prune it from our child list
+func (fp *FilePath) filterFilesRecursive(ff FileFilter) bool {
+	if fp.File {
+		return ff(fp.Path)
+	}
+
+	// The index to still keep up to
+	keepDex := 0
+	for pth := range fp.UnderPaths {
+		remove := fp.UnderPaths[pth].filterFilesRecursive(ff)
+		if !remove {
+			fp.UnderPaths[keepDex] = fp.UnderPaths[pth]
+			keepDex++
+		}
+	}
+	fp.UnderPaths = fp.UnderPaths[:keepDex]
+
+	return keepDex == 0
+}
+
+func (cm *CollectionManagement) GetAzureBlobLink(filePath string) (link string, err error) {
+	link, err = url.JoinPath(cm.BaseLink.String(), filePath)
+	if err != nil {
+		logger.Err(err).Msg("")
+		return
+	}
+	sasToken, err := cm.getSASAccessTokenOnceADay()
+	if err != nil {
+		logger.Err(err).Msg("")
+	}
+	return link + "?" + sasToken, err
+}
+
+func (cm *CollectionManagement) getSASAccessTokenOnceADay() (token string, err error) {
+	if time.Since(cm.cachedTokenTime) > time.Hour*24 {
+		cm.cachedTokenTime = time.Now()
+		cm.cachedToken, err = cm.generateSASToken()
+	}
+	return cm.cachedToken, err
+}
+
+func (cm *CollectionManagement) generateSASToken() (token string, err error) {
+
+	sasQueryParams, err := azblob.BlobSASSignatureValues{
+		Protocol:      azblob.SASProtocolHTTPS,
+		StartTime:     time.Now().UTC(),
+		ExpiryTime:    time.Now().UTC().Add(48 * time.Hour),
+		Permissions:   azblob.BlobSASPermissions{Read: true, List: true}.String(),
+		IPRange:       azblob.IPRange{},
+		ContainerName: cm.cmci.AzureContainerName,
+	}.NewSASQueryParameters(cm.SharedKeyCredential)
+
+	if err != nil {
+		logger.Error().Err(err).Msg("Failed to sas.BlobSignatureValues")
+		return
+	}
+
+	token = sasQueryParams.Encode()
+	return
+}

pkg/cache/apitokens.go

@@ -0,0 +1,77 @@
+package cache
+
+import (
+	"sync"
+	"time"
+
+	"fiskerinc.com/modules/common"
+	"github.com/pkg/errors"
+
+	"fiskerinc.com/modules/db/queries"
+	"github.com/ReneKroon/ttlcache/v2"
+)
+
+const ApiKeyHeader = "Api-Key"
+
+var (
+	ErrInvalidToken = errors.New("invalid API token")
+	ErrTokenExpired = errors.New("token is expired")
+)
+
+type APITokenCache struct {
+	APITokens queries.APITokensInterface
+	cache     *ttlcache.Cache
+	onceCache sync.Once
+}
+
+func (a *APITokenCache) Get(key string) (string, error) {
+	value, err := a.Cache().Get(key)
+	if err == nil {
+		apiToken, ok := value.(*common.APIToken)
+		if !ok {
+			return "", ErrInvalidToken
+		}
+
+		if apiToken.ExpiresAt != nil && apiToken.ExpiresAt.Before(time.Now()) {
+			return "", ErrTokenExpired
+		}
+
+		return apiToken.Roles, nil
+	} else if !errors.Is(err, ttlcache.ErrNotFound) {
+		return "", err
+	}
+
+	item, err := a.APITokens.Get(key)
+	if err != nil {
+		return "", err
+	}
+	if item.ExpiresAt != nil && item.ExpiresAt.Before(time.Now()) {
+		return "", ErrTokenExpired
+	}
+
+	err = a.Cache().Set(key, item)
+	if err != nil {
+		return "", err
+	}
+
+	return item.Roles, nil
+}
+
+func (a *APITokenCache) Cache() *ttlcache.Cache {
+	a.onceCache.Do(func() {
+		if a.cache == nil {
+			cache := ttlcache.NewCache()
+			cache.SetTTL(10 * time.Minute)
+			cache.SetCacheSizeLimit(10)
+			a.cache = cache
+		}
+	})
+
+	return a.cache
+}
+
+func (a *APITokenCache) Close() {
+	a.cache.Close()
+	a.cache = nil
+	a.APITokens = nil
+}

pkg/cache/apitokens_test.go

@@ -0,0 +1,84 @@
+package cache_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/adminroles"
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestIntegration(t *testing.T) {
+	t.Skip()
+	testKey := "YYYYYYY"
+	q := queries.APITokens{}
+	client := q.GetClient()
+	client.GetConn().AddQueryHook(db.SQLLogger{})
+	err := client.InitSchema([]interface{}{
+		(*common.APIToken)(nil),
+	})
+	if err != nil {
+		t.Error(err)
+	}
+
+	_, err = q.Insert(common.APIToken{
+		Token: testKey,
+		Roles: string(adminroles.RoleCreate),
+	})
+	if err != nil {
+		t.Error(err)
+	}
+
+	type testCase struct {
+		Name          string
+		Key           string
+		ExpectedRoles string
+		ExpectedError string
+		Setup         func(client redis.Client, db queries.APITokensInterface) error
+		Teardown      func(client redis.Client, db queries.APITokensInterface) error
+	}
+
+	tests := []testCase{
+		{
+			Name:          "Invalid token",
+			Key:           "XXXXXXXX",
+			ExpectedError: "token not found",
+		},
+		{
+			Name:          "From DB",
+			Key:           testKey,
+			ExpectedRoles: string(adminroles.RoleCreate),
+		},
+		{
+			Name:          "From Cache",
+			Key:           testKey,
+			ExpectedRoles: string(adminroles.RoleCreate),
+		},
+		{
+			Name:          "No token",
+			Key:           "",
+			ExpectedError: "token required",
+		},
+	}
+
+	apitokens := cache.APITokenCache{
+		APITokens: &q,
+	}
+	defer apitokens.Close()
+
+	for _, test := range tests {
+		value, err := apitokens.Get(test.Key)
+		if err != nil && err.Error() != test.ExpectedError {
+			t.Errorf(testhelper.TestErrorTemplate, test.Name, test.ExpectedError, err.Error())
+		}
+		if value != test.ExpectedRoles {
+			t.Errorf(testhelper.TestErrorTemplate, test.Name, test.ExpectedRoles, value)
+		}
+	}
+
+	q.Delete(testKey)
+}

pkg/cache/car_dtcs.go

@@ -0,0 +1,28 @@
+package cache
+
+import (
+	"fiskerinc.com/modules/common"
+)
+
+type CarDTCsCacheInterface interface {
+	Exists(dtc common.DTC_ECU) bool
+	Set(dtc common.DTC_ECU)
+}
+
+type CarDTCsCache struct {
+	ringMap *RingMap
+}
+
+func NewCarDTCsCache(capacity int) CarDTCsCacheInterface {
+	return &CarDTCsCache{
+		ringMap: NewRingMap(capacity),
+	}
+}
+
+func (carDtcCache *CarDTCsCache) Exists(dtc common.DTC_ECU) bool {
+	return carDtcCache.ringMap.Exists(dtc.CacheKey(), dtc.StatusByte)
+}
+
+func (carDtcCache *CarDTCsCache) Set(dtc common.DTC_ECU) {
+	carDtcCache.ringMap.Put(dtc.CacheKey(), dtc.StatusByte)
+}

pkg/cache/car_dtcs_test.go

@@ -0,0 +1,31 @@
+package cache_test
+
+import (
+	"testing"
+
+	m "fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/cache"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	dtc = m.DTC_ECU{
+		VIN: "3FAFP13P71R199432",
+		ECU: "ACU",
+		TroubleCode: 8388881,
+	}
+)
+
+func TestSetAndExists(t *testing.T) {
+	carDtcCache := cache.NewCarDTCsCache(1000)
+	exists := carDtcCache.Exists(dtc)
+
+	assert.Equal(t, exists, false)
+
+    carDtcCache.Set(dtc)
+
+	exists = carDtcCache.Exists(dtc)
+
+	assert.Equal(t, exists, true)
+}

pkg/cache/constants.go

@@ -0,0 +1,4 @@
+package cache
+
+const redisObjectExpire = 600
+const redisObjectExpireDay = 86400

pkg/cache/digital_twin.go

@@ -0,0 +1,192 @@
+package cache
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"fiskerinc.com/modules/dbc/state"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/utils/querystring"
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+const (
+	pattern = "car:*:state"
+)
+
+type DigitalTwinTimestampState struct {
+	redisClient redis.Client
+}
+
+func NewDigitalTwinTimestampState(redisClient redis.Client) *DigitalTwinTimestampState {
+	return &DigitalTwinTimestampState{
+		redisClient: redisClient,
+	}
+}
+
+// getStateKeys retrieves car state keys from Redis based on the specified pattern
+// and returns a sliced list of keys according to the provided offset and limit.
+//
+// Parameters:
+//   - offset: An integer indicating the starting index of the slice.
+//   - limit: An integer specifying the maximum number of elements in the sliced list.
+//
+// Returns:
+//   - []string: A sliced list of car state keys based on the given offset and limit.
+//   - error: An error, if any, encountered during the Redis operation or slicing process.
+func (dtts *DigitalTwinTimestampState) getStateKeys(offset, limit int) ([]string, error) {
+	keys, err := redigo.Strings(dtts.redisClient.Execute("KEYS", pattern))
+	if err != nil {
+		return nil, err
+	}
+	totalKeys := len(keys)
+	if totalKeys <= offset {
+		return nil, nil
+	}
+
+	if (offset + limit) > totalKeys {
+		limit = totalKeys - offset
+	}
+
+	sort.Strings(keys)
+
+	keys = keys[offset : offset+limit]
+	return keys, nil
+}
+
+// readCarStateByKey retrieves data from Redis based on the specified key using the HGETALL command.
+// It iterates over all keys and values returned by the command, sets them in a response map,
+// and returns the populated map along with any encountered errors.
+//
+// Parameters:
+//   - key: A string representing the key to retrieve data from in Redis.
+//
+// Returns:
+//   - map[string]interface{}: A map containing keys and values retrieved from Redis.
+//   - error: An error, if any, encountered during the Redis HGETALL operation or mapping process.
+func (dtts *DigitalTwinTimestampState) readCarStateByKey(key string) (map[string]interface{}, error) {
+
+	keyval := make(map[string]interface{})
+	batch := redis.NewRedisBatchCommands()
+
+	batch.Add("HGETALL", key)
+
+	payload, err := redigo.Values(dtts.redisClient.ExecuteBatch(batch))
+	if err != nil {
+		return keyval, err
+	}
+
+	stateValues, err := redigo.Values(payload[0], nil)
+	if err != nil {
+		return keyval, err
+	}
+
+	for i := 0; i < len(stateValues); i += 2 {
+		key, okKey := stateValues[i].([]byte)
+		value, okValue := stateValues[i+1].([]byte)
+
+		if !okKey || !okValue {
+			continue
+		}
+
+		err = dtts.parseCarState(string(key), value, keyval)
+		// log error, do not return error so we can read other properties for digital twin
+		if err != nil {
+			logger.Warn().Err(err).Send()
+			continue
+		}
+	}
+	return keyval, nil
+}
+
+// GetDigitalTwinSignals retrieves digital twin signals from Redis based on the specified offset and limit.
+// It reads all signals from Redis and returns a list of maps, where each map represents a cars signal with its properties.
+//
+// Parameters:
+//   - offset: An integer indicating the starting index of the signals to retrieve.
+//   - limit: An integer specifying the maximum number of signals to retrieve.
+//
+// Returns:
+//   - []map[string]interface{}: A list of maps representing digital twin signals.
+
+func (dtts *DigitalTwinTimestampState) GetDigitalTwinSignals(offset, limit int) (resp []map[string]interface{}) {
+
+	keys, err := dtts.getStateKeys(offset, limit)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+		return
+	}
+	for _, key := range keys {
+		keyval, err := dtts.readCarStateByKey(key)
+		if err != nil {
+			logger.Warn().Err(err).Send()
+			continue
+		}
+		if len(keyval) > 0 {
+			keySlice := strings.Split(key, ":")
+			keyval["VIN"] = keySlice[1]
+			resp = append(resp, keyval)
+		}
+	}
+	return
+}
+
+// timestampKey generates a timestamp key based on the provided key by appending ":updated" to it.
+// It formats the key in a way suitable for storing timestamps associated with the original key in data storage systems.
+//
+// Parameters:
+//   - key: A string representing the original key for which the timestamp key is generated.
+//
+// Returns:
+//   - string: A formatted string representing the timestamp key.
+func (dtts *DigitalTwinTimestampState) timestampKey(key string) string {
+	return fmt.Sprintf("%s:%s", key, "updated")
+}
+
+// timestampVal parses a byte slice containing a JSON-encoded timestamp and returns a pointer to a time.Time
+// representing the parsed timestamp. It uses the UnmarshalJSON method of the time.Time type for decoding.
+//
+// Parameters:
+//   - val: A byte slice containing the JSON-encoded timestamp to be parsed.
+//
+// Returns:
+//   - time.Time: A time representing the parsed timestamp.
+//   - error: An error, if any, encountered during the parsing process.
+func (dtts *DigitalTwinTimestampState) timestampVal(val []byte) (time.Time, error) {
+	t := &time.Time{}
+	err := t.UnmarshalJSON(val)
+	return *t, err
+}
+
+// parseCarState checks if the provided key is needed and, if so, sets the key and value in the given map.
+//
+// Parameters:
+//   - key: A string representing the key to check and potentially set in the map.
+//   - value: A byte slice containing the value associated with the key.
+//   - keyval: A map[string]interface{} where the key and valueset if the key is needed.
+//
+// Returns:
+//   - error: An error, if any, encountered during the parsing and mapping process.
+func (dtts *DigitalTwinTimestampState) parseCarState(key string, value []byte, keyval map[string]interface{}) error {
+	var err error
+	val := string(value)
+	switch key {
+	case state.BMS_PwrBattRmngCpSOC, state.BMS_RmChrgTi_FullChrg, state.BCM_PwrMod, state.PWC_ChrgSts, state.VCU_DCChrgRmngTi, state.BMS_RmChrgTi_TrgtSoC:
+		keyval[key], err = strconv.Atoi(val)
+	case state.ICC_TotMilg_ODO:
+		keyval[key], err = querystring.ConvertStringToInt(val)
+	case state.IBS_BatteryVoltage:
+		keyval[key], err = strconv.ParseFloat(val, 64)
+
+	// updated timestamps
+	case dtts.timestampKey(state.BMS_PwrBattRmngCpSOC), dtts.timestampKey(state.ICC_TotMilg_ODO), dtts.timestampKey(state.VCU_DCChrgRmngTi), dtts.timestampKey(state.BMS_RmChrgTi_TrgtSoC), dtts.timestampKey(state.IBS_BatteryVoltage),
+		dtts.timestampKey(state.BMS_RmChrgTi_FullChrg), dtts.timestampKey(state.BCM_PwrMod), dtts.timestampKey(state.PWC_ChrgSts):
+		keyval[key], err = dtts.timestampVal(value)
+	}
+	return errors.WithStack(err)
+}

pkg/cache/drivers.go

@@ -0,0 +1,141 @@
+package cache
+
+import (
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+	"github.com/pkg/errors"
+)
+
+func NewDriversCache(redisClient redis.ClientPoolInterface, cars queries.CarsInterface) *DriversCache {
+	return &DriversCache{
+		redisClientPool: redisClient,
+		cars:        cars,
+	}
+}
+
+type DriversCache struct {
+	redisClientPool redis.ClientPoolInterface
+	cars        queries.CarsInterface
+}
+
+func (dc *DriversCache) RedisClientPool() redis.ClientPoolInterface {
+	return dc.redisClientPool
+}
+
+func (dc *DriversCache) Cars() queries.CarsInterface {
+	return dc.cars
+}
+
+func (dc *DriversCache) hasCachedNoDrivers(drivers []string) bool {
+	// Redis will return []string{""} for no drivers
+	return len(drivers) == 1 && len(drivers[0]) == 0
+}
+
+func (dc *DriversCache) cacheDrivers(key string, drivers []string) error {
+	client := dc.redisClientPool.GetFromPool()
+	defer client.Close()
+	// cache driver IDs
+	if len(drivers) > 0 {
+		return client.NewSet(key, drivers, redisObjectExpire)
+	}
+
+	// Redis will not take an empty array as an arg
+	return client.NewSet(key, nil, redisObjectExpire)
+}
+
+// RetrieveDriverIDs retrieves IDs from redis or from DB and proceeds to cache both the drivers and IDs
+// redis keys:
+//
+//	car:<VIN>:drivers
+func (dc *DriversCache) RetrieveDriverIDs(vin string) ([]string, error) {
+	var driverIDs []string
+	driverIDsKey := redis.CarToAllDriversKey(vin)
+
+	// retrieve IDs from redis
+	client := dc.redisClientPool.GetFromPool()
+	err := client.GetSet(driverIDsKey, &driverIDs)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+	}
+	client.Close()
+
+	if dc.hasCachedNoDrivers(driverIDs) {
+		return []string{}, nil
+	}
+	if len(driverIDs) > 0 {
+		return driverIDs, nil
+	}
+
+	// if IDs not present in redis perform DB lookup
+	var drivers []common.CarToDriver
+	drivers, err = dc.cars.GetDrivers(vin)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, driver := range drivers {
+		driverIDs = append(driverIDs, driver.DriverID)
+	}
+
+	err = dc.cacheDrivers(driverIDsKey, driverIDs)
+	if err != nil {
+		return driverIDs, err
+	}
+
+	return driverIDs, nil
+}
+
+// RetrieveDriverIDsAsSet retrieves IDs from redis or from DB and proceeds to cache both the drivers and IDs
+// redis keys:
+//
+//	car:<VIN>:drivers
+func (dc *DriversCache) RetrieveDriverIDsAsSet(vin string) (map[string]struct{}, error) {
+	driverIDs, err := dc.RetrieveDriverIDs(vin)
+	if err != nil {
+		return nil, err
+	}
+
+	var dIDsSet = make(map[string]struct{})
+	for _, did := range driverIDs {
+		dIDsSet[did] = struct{}{}
+	}
+
+	return dIDsSet, nil
+}
+
+func (dc *DriversCache) IsDriverOfVIN(vin string, driverid string) (bool, error) {
+	ids, err := dc.RetrieveDriverIDs(vin)
+	if err != nil {
+		return false, err
+	}
+
+	for _, id := range ids {
+		if id == driverid {
+			return true, nil
+		}
+	}
+
+	return false, dc.NotDriverError(vin, driverid)
+}
+
+// Add driver to database and cache
+func (dc *DriversCache) AddDriver(car *common.Car, driver *common.Driver, role string) (*common.CarToDriver, error) {
+
+	relation, err := dc.cars.AddDriver(car, driver, role)
+	if err != nil {
+		return nil, err
+	}
+
+	driverIDsKey := redis.CarToAllDriversKey(car.VIN)
+
+	client := dc.redisClientPool.GetFromPool()
+	defer client.Close()
+	client.AddToSet(driverIDsKey, driver.ID, redisObjectExpire)
+	return relation, nil
+}
+
+func (dc DriversCache) NotDriverError(vin string, driverid string) error {
+	return errors.Errorf("id %s is not a driver for vin %v", driverid, vin)
+}

pkg/cache/drivers_test.go

@@ -0,0 +1,107 @@
+package cache_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/db/queries/mocks"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/redis/tester"
+	"fiskerinc.com/modules/testhelper"
+)
+
+var mockRedis redis.Client
+var mockDB queries.CarsInterface
+
+func setupRedisMock() {
+	redis.MockRedisConnection()
+}
+
+func setupDBMock() {
+	mockDB = &mocks.MockCars{}
+}
+
+type mockRedisCache struct {
+	redis.Connection
+}
+
+func (c *mockRedisCache) GetSet(id string, data interface{}) error {
+	drivers := []string{"valid-id-1", "valid-id-2", "valid-id-3"}
+
+	dataBytes, err := json.Marshal(drivers)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(dataBytes, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type mockRedisEmptyCache struct {
+	redis.Connection
+}
+
+func (c *mockRedisEmptyCache) GetSet(id string, data interface{}) error {
+	drivers := []string{}
+
+	dataBytes, err := json.Marshal(drivers)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(dataBytes, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *mockRedisEmptyCache) SetObjects(id []string, data []interface{}, expire int) error {
+	return nil
+}
+
+func TestRetrieveAndCacheDriverIDs(t *testing.T) {
+	setupRedisMock()
+	setupDBMock()
+	mockRedis = &mockRedisCache{}
+	redisPool := tester.NewMockClientPool(mockRedis)
+	drivers := cache.NewDriversCache(redisPool, mockDB)
+
+	_, err := drivers.RetrieveDriverIDs("FISKER123")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+
+	mockRedis = &mockRedisEmptyCache{}
+	_, err = drivers.RetrieveDriverIDs("FISKER456")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+}
+
+func TestRetrieveAndCacheDriverIDsAsSet(t *testing.T) {
+	setupRedisMock()
+	setupDBMock()
+	mockRedis = &mockRedisCache{}
+	redisPool := tester.NewMockClientPool(mockRedis)
+
+	drivers := cache.NewDriversCache(redisPool, mockDB)
+
+	_, err := drivers.RetrieveDriverIDs("FISKER123")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+
+	mockRedis = &mockRedisEmptyCache{}
+	_, err = drivers.RetrieveDriverIDsAsSet("FISKER456")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+}

pkg/cache/errors.go

@@ -0,0 +1,11 @@
+package cache
+
+import "github.com/pkg/errors"
+
+func ErrInvalidCarToDriverAssociation(vin string, driverID string) error {
+	return errors.Errorf("no relationship found between vin %s and driver %s", vin, driverID)
+}
+
+func ErrCarHasNoDrivers(vin string) error {
+	return errors.Errorf("car %s has no drivers", vin)
+}

pkg/cache/fileids.go

@@ -0,0 +1,147 @@
+package cache
+
+import (
+	"encoding/json"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+
+	r "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+func RetrieveFileEncryptionParams(client redis.Client, db queries.FileKeysInterface, fileids []string) ([]common.FileKeyResponse, error) {
+	result, err := retrieveFileEncryptionParamsRedis(client, fileids)
+	if err != nil && !errors.Is(err, redis.ErrNilObject) {
+		return nil, err
+	}
+
+	dbFileIDs := missingFileIDs(result, fileids)
+	if len(dbFileIDs) > 0 {
+		rows, err := retrieveFileEncryptionParamsDB(db, dbFileIDs)
+		if err != nil {
+			return nil, err
+		}
+		err = cacheFileEncryptionParamsRedis(client, rows, redisObjectExpireDay)
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(rows, result...)
+	}
+
+	return result, nil
+}
+
+func retrieveFileEncryptionParamsRedis(client redis.Client, fileids []string) ([]common.FileKeyResponse, error) {
+	keys := make([]string, len(fileids))
+
+	for i, fileid := range fileids {
+		keys[i] = redis.FileIDEncryptionParamsKey(fileid)
+	}
+
+	if len(keys) == 0 {
+		return []common.FileKeyResponse{}, nil
+	}
+
+	values, err := client.GetMulti(keys)
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := getFileKeyResponses(values)
+	return result, err
+}
+
+func getFileKeyResponses(values []interface{}) ([]common.FileKeyResponse, error) {
+	result := []common.FileKeyResponse{}
+
+	for _, value := range values {
+		if value == nil {
+			continue
+		}
+
+		file := common.FileKeyResponse{}
+		data, err := r.Bytes(value, nil)
+		if err != nil {
+			file.Error = err.Error()
+			return result, errors.WithStack(err)
+		}
+
+		err = json.Unmarshal(data, &file)
+		if err != nil {
+			return result, errors.WithStack(err)
+		}
+
+		result = append(result, file)
+	}
+
+	return result, nil
+}
+
+func missingFileIDs(items []common.FileKeyResponse, fileids []string) []string {
+	result := []string{}
+	hash := map[string]bool{}
+
+	for _, item := range items {
+		if item.FileID != "" {
+			hash[item.FileID] = true
+		}
+	}
+
+	for _, fileid := range fileids {
+		if !hash[fileid] {
+			result = append(result, fileid)
+		}
+	}
+
+	return result
+}
+
+func retrieveFileEncryptionParamsDB(db queries.FileKeysInterface, fileids []string) ([]common.FileKeyResponse, error) {
+	result := make([]common.FileKeyResponse, len(fileids))
+	hash := map[string]bool{}
+	data, err := db.GetMulti(fileids)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	for i, file := range data {
+		result[i].Apply(&file)
+		hash[file.FileID] = true
+	}
+
+	if len(fileids) != len(data) {
+		starting := len(data)
+		current := 0
+		for _, fileid := range fileids {
+			if !hash[fileid] {
+				logger.Warn().Msgf("file id %s not found", fileid)
+				file := &result[starting+current]
+				file.FileID = fileid
+				file.Error = "not found"
+				current++
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func cacheFileEncryptionParamsRedis(client redis.Client, files []common.FileKeyResponse, expire int) error {
+	batch := redis.NewRedisBatchCommands()
+
+	for _, file := range files {
+		serialized, err := json.Marshal(file)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		id := redis.FileIDEncryptionParamsKey(file.FileID)
+		batch.Add("SET", id, serialized, "EX", expire)
+	}
+
+	_, err := client.ExecuteBatch(batch)
+	return errors.WithStack(err)
+}

pkg/cache/fileids_test.go

@@ -0,0 +1,74 @@
+package cache_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/db/queries/mocks"
+	"fiskerinc.com/modules/redis"
+)
+
+func TestRetrieveFileEncryptionParams(t *testing.T) {
+	key := "b7f74938c9402dc2"
+	c := NewMockRedisConn()
+	q := mocks.MockFileKeys{
+		GetResponse: &common.FileKey{},
+	}
+
+	_, err := cache.RetrieveFileEncryptionParams(c, &q, []string{key})
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkRetrieveFileEncryptionParams(b *testing.B) {
+	c := redis.NewClient()
+	q := queries.FileKeys{}
+
+	for n := 0; n < b.N; n++ {
+		_, err := cache.RetrieveFileEncryptionParams(c, &q, []string{"b7f74938c9402dc2"})
+		if err != nil {
+			b.Error(err)
+		}
+	}
+}
+
+func NewMockRedisConn() *MockRedisConn {
+	mock := &MockRedisConn{}
+	mock.SetConn(redis.GetMockPool().Get())
+	return mock
+}
+
+type MockRedisConn struct {
+	redis.Connection
+}
+
+func (m *MockRedisConn) GetCache(id string, dest interface{}, expire int) error {
+	return nil
+}
+
+func (m *MockRedisConn) SetCache(id string, data interface{}, expire int) error {
+	return nil
+}
+
+func (m *MockRedisConn) GetValuesMulti(ids []string, data interface{}) error {
+	return nil
+}
+
+func (m *MockRedisConn) SetMulti(ids []string, data []interface{}) error {
+	return nil
+}
+func TestRetrieveFileEncryptionParamsIntegration(t *testing.T) {
+	t.Skip()
+	c := redis.NewClient()
+	q := queries.FileKeys{}
+	q.GetClient().GetConn().AddQueryHook(db.SQLLogger{})
+
+	_, err := cache.RetrieveFileEncryptionParams(c, &q, []string{"bd2c7a6cc94042cb", "83165a80c940e8b3"})
+	if err != nil {
+		t.Error(err)
+	}
+}

pkg/cache/filters.go

@@ -0,0 +1,22 @@
+package cache
+
+import (
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/redis"
+)
+
+func FillCarFilterOnline(redisCLI redis.Client, filter *common.CarSearch) error {
+	if filter.Online == nil {
+		return nil
+	}
+
+	var onlineVehicles []string
+	err := redisCLI.GetSet(redis.CarSessionsKey(), &onlineVehicles)
+	if err != nil {
+		return err
+	}
+
+	filter.Online.VINsOnline = onlineVehicles
+
+	return nil
+}

pkg/cache/filters_test.go

@@ -0,0 +1,67 @@
+package cache_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	m "fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/redis/tester"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+var someErr = errors.New("some error")
+
+func TestFillCarFilterOnline(t *testing.T) {
+	val_false := false
+	tests := map[string]struct {
+		redisCLI  *tester.MockRedis
+		filter    m.CarSearch
+		expErr    error
+		expFilter m.CarSearch
+	}{
+		"correct_common": {
+			redisCLI: &tester.MockRedis{
+				GetSetResults: `["FISKERVIN1","FISKERVIN2"]`,
+			},
+			filter: m.CarSearch{
+				Online: &m.CarOnlineFilter{
+					Online: &val_false,
+				},
+			},
+			expFilter: m.CarSearch{
+				Online: &m.CarOnlineFilter{
+					Online:     &val_false,
+					VINsOnline: []string{"FISKERVIN1", "FISKERVIN2"},
+				},
+			},
+		},
+		"filter_nil": {
+			filter:    m.CarSearch{},
+			expFilter: m.CarSearch{},
+		},
+		"redis_err": {
+			redisCLI: &tester.MockRedis{
+				Error: someErr,
+			},
+			filter: m.CarSearch{
+				Online: &m.CarOnlineFilter{
+					Online: &val_false,
+				},
+			},
+			expErr: someErr,
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			err := cache.FillCarFilterOnline(tt.redisCLI, &tt.filter)
+			if err != nil && tt.expErr != nil {
+				assert.Equal(t, tt.expErr.Error(), err.Error())
+				return
+			}
+
+			assert.Equal(t, tt.expErr, err)
+			assert.Equal(t, tt.filter, tt.expFilter)
+		})
+	}
+}

pkg/cache/ringmap.go

@@ -0,0 +1,161 @@
+package cache
+
+import (
+	"sync"
+
+	"github.com/elliotchance/orderedmap/v2"
+)
+
+type RingMapInterface interface {
+	Exists(key string, value interface{}) bool
+}
+
+// Copied from https://github.com/prgsmall/ringmap to use orderedmap v2
+
+type RingMap struct {
+	orderedMap *orderedmap.OrderedMap[string, interface{}]
+	capacity   int
+	writeLock  sync.RWMutex
+}
+
+func NewRingMap(capacity int) *RingMap {
+	return &RingMap{
+		orderedMap: orderedmap.NewOrderedMap[string, interface{}](),
+		capacity:   capacity,
+	}
+}
+
+// Convenience function to check if key and value already exists
+// If key and value does not exists, it is added
+// If key exists with different value, it is replaced with new value
+func (m *RingMap) Exists(key string, value interface{}) bool {
+	m.writeLock.RLock()
+	el := m.orderedMap.GetElement(key)
+	m.writeLock.RUnlock()
+	exists := el != nil
+
+	if exists && el.Value != value {
+		m.Delete(key)
+		exists = false
+	} else {
+		m.clearLast()
+	}
+	m.writeLock.Lock()
+	defer m.writeLock.Unlock()
+	m.orderedMap.Set(key, value)
+
+	return exists
+}
+
+// Get returns the value for a key. If the key does not exist, the second return
+// parameter will be false and the value will be nil.
+func (m *RingMap) Get(key string) (interface{}, bool) {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.orderedMap.Get(key)
+}
+
+// Set will set (or replace) a value for a key. If the key was new, then true
+// will be returned. The returned value will be false if the value was replaced
+// (even if the value was the same).  If a new key is being added and the map is
+// full, then the front element will be deleted to make room for the new element.
+func (m *RingMap) Set(key string, value interface{}) bool {
+	_, didExist := m.Get(key)
+
+	if !didExist {
+		m.clearLast()
+	}
+	m.writeLock.Lock()
+	defer m.writeLock.Unlock()
+	m.orderedMap.Set(key, value)
+
+	return !didExist
+}
+
+// Put will set a value for a key. If the key already exists, it will be deleted
+// from and a recreated at the end of the list.  If the key was new, then true
+// will be returned. The returned value will be false if the value was replaced
+// (even if the value was the same).  If a new key is being added and the map is
+// full, then the front element will be deleted to make room for the new element.
+func (m *RingMap) Put(key string, value interface{}) bool {
+	_, didExist := m.Get(key)
+
+	if didExist {
+		m.Delete(key)
+	} else {
+		m.clearLast()
+	}
+	m.writeLock.Lock()
+	defer m.writeLock.Unlock()
+	m.orderedMap.Set(key, value)
+
+	return !didExist
+}
+
+// GetOrDefault returns the value for a key. If the key does not exist, returns
+// the default value instead.
+func (m *RingMap) GetOrDefault(key string, defaultValue interface{}) interface{} {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.orderedMap.GetOrDefault(key, defaultValue)
+}
+
+// Len returns the number of elements in the map.
+func (m *RingMap) Len() int {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.orderedMap.Len()
+}
+
+// Capacity returns the capacity of the map
+func (m *RingMap) Capacity() int {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.capacity
+}
+
+// IsFull returns true if the number of elements in the map is Capacity()
+func (m *RingMap) IsFull() bool {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.orderedMap.Len() == m.capacity
+}
+
+// Keys returns all of the keys in the order they were inserted. If a key was
+// replaced it will retain the same position. To ensure most recently set keys
+// are always at the end you must always Delete before Set.
+func (m *RingMap) Keys() (keys []string) {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.orderedMap.Keys()
+}
+
+// Delete will remove a key from the map. It will return true if the key was
+// removed (the key did exist).
+func (m *RingMap) Delete(key string) (didDelete bool) {
+	m.writeLock.Lock()
+	defer m.writeLock.Unlock()
+	return m.orderedMap.Delete(key)
+}
+
+// Front will return the element that is the first (oldest Set element). If
+// there are no elements this will return nil.
+func (m *RingMap) Front() *orderedmap.Element[string, interface{}] {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.orderedMap.Front()
+}
+
+// Back will return the element that is the last (most recent Set element). If
+// there are no elements this will return nil.
+func (m *RingMap) Back() *orderedmap.Element[string, interface{}] {
+	m.writeLock.RLock()
+	defer m.writeLock.RUnlock()
+	return m.orderedMap.Back()
+}
+
+func (m *RingMap) clearLast() {
+	if m.IsFull() {
+		m.Delete(m.Front().Key)
+	}
+}

pkg/cache/ringmap_test.go

@@ -0,0 +1,913 @@
+package cache_test
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var ringMapCapacity = 777
+
+func TestObjectCreation(t *testing.T) {
+
+	t.Run("TestNewRingMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		assert.IsType(t, &cache.RingMap{}, m)
+		assert.Equal(t, ringMapCapacity, m.Capacity())
+		assert.EqualValues(t, false, m.IsFull())
+	})
+}
+
+func TestGet(t *testing.T) {
+	t.Run("ReturnsNotOKIfStringKeyDoesntExist", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		_, ok := m.Get("foo")
+		assert.False(t, ok)
+	})
+
+	t.Run("ReturnsNotOKIfNonStringKeyDoesntExist", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		_, ok := m.Get("123")
+		assert.False(t, ok)
+	})
+
+	t.Run("ReturnsOKIfKeyExists", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", "bar")
+		_, ok := m.Get("foo")
+		assert.True(t, ok)
+	})
+
+	t.Run("ReturnsValueForKey", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", "bar")
+		value, _ := m.Get("foo")
+		assert.Equal(t, "bar", value)
+	})
+
+	t.Run("ReturnsDynamicValueForKey", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", "baz")
+		value, _ := m.Get("foo")
+		assert.Equal(t, "baz", value)
+	})
+
+	t.Run("KeyDoesntExistOnNonEmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", "baz")
+		_, ok := m.Get("bar")
+		assert.False(t, ok)
+	})
+
+	t.Run("ValueForKeyDoesntExistOnNonEmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", "baz")
+		value, _ := m.Get("bar")
+		assert.Nil(t, value)
+	})
+}
+
+func TestPut(t *testing.T) {
+	t.Run("ReturnsTrueIfStringKeyIsNew", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		ok := m.Put("foo", "bar")
+		assert.True(t, ok)
+	})
+
+	t.Run("ReturnsTrueIfNonStringKeyIsNew", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		ok := m.Put("123", "bar")
+		assert.True(t, ok)
+	})
+
+	t.Run("ValueCanBeNonString", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		ok := m.Put("123", true)
+		assert.True(t, ok)
+	})
+
+	t.Run("ReturnsFalseIfKeyIsNotNew", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Put("foo", "bar")
+		ok := m.Put("foo", "bar")
+		assert.False(t, ok)
+	})
+
+	t.Run("PutMultipleKeys", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Put("foo", "bar")
+		m.Put("baz", "qux")
+		m.Put("mik", "qux")
+		ok := m.Put("quux", "corge")
+		assert.True(t, ok)
+	})
+
+	t.Run("PutMultipleDifferentKeysWithReplace", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Put("foo", "bar")
+		m.Put("baz", "baz")
+		m.Put("mik", "mik")
+		ok := m.Put("foo", "corge")
+		assert.False(t, ok)
+		assert.Equal(t, "baz", m.Front().Key)
+		assert.Equal(t, "foo", m.Back().Key)
+	})
+
+	t.Run("PutMultipleDifferentKeysWithReplace", func(t *testing.T) {
+		m := cache.NewRingMap(3)
+		m.Put("ace", "bev")
+		m.Put("foo", "bar")
+		m.Put("baz", "baz")
+		m.Put("mik", "mik")
+		ok := m.Put("foo", "corge")
+		assert.False(t, ok)
+		assert.Equal(t, "baz", m.Front().Key)
+		assert.Equal(t, "foo", m.Back().Key)
+	})
+}
+
+func TestSet(t *testing.T) {
+	t.Run("ReturnsTrueIfStringKeyIsNew", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		ok := m.Set("foo", "bar")
+		assert.True(t, ok)
+	})
+
+	t.Run("ReturnsTrueIfNonStringKeyIsNew", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		ok := m.Set("123", "bar")
+		assert.True(t, ok)
+	})
+
+	t.Run("ValueCanBeNonString", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		ok := m.Set("123", true)
+		assert.True(t, ok)
+	})
+
+	t.Run("ReturnsFalseIfKeyIsNotNew", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", "bar")
+		ok := m.Set("foo", "bar")
+		assert.False(t, ok)
+	})
+
+	t.Run("SetThreeDifferentKeys", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", "bar")
+		m.Set("baz", "qux")
+		ok := m.Set("quux", "corge")
+		assert.True(t, ok)
+	})
+}
+
+func TestLen(t *testing.T) {
+	t.Run("EmptyMapIsZeroLen", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		assert.Equal(t, 0, m.Len())
+	})
+
+	t.Run("SingleElementIsLenOne", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("123", true)
+		assert.Equal(t, 1, m.Len())
+	})
+
+	t.Run("ThreeElements", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("1", true)
+		m.Set("2", true)
+		m.Set("3", true)
+		assert.Equal(t, 3, m.Len())
+	})
+
+	t.Run("ThreeElementsWithMax", func(t *testing.T) {
+		m := cache.NewRingMap(3)
+		assert.Equal(t, false, m.IsFull())
+		m.Set("1", true)
+		assert.Equal(t, false, m.IsFull())
+		m.Set("2", true)
+		assert.Equal(t, false, m.IsFull())
+		m.Set("3", true)
+		assert.Equal(t, 3, m.Len())
+		assert.Equal(t, true, m.IsFull())
+		assert.Equal(t, m.Front().Key, "1")
+
+		m.Set("4", true)
+		assert.Equal(t, 3, m.Len())
+		assert.Equal(t, true, m.IsFull())
+		assert.Equal(t, m.Front().Key, "2")
+		assert.Equal(t, m.Back().Key, "4")
+	})
+}
+
+func TestKeys(t *testing.T) {
+	t.Run("EmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		assert.Empty(t, m.Keys())
+	})
+
+	t.Run("OneElement", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("1", true)
+		assert.Equal(t, []string{"1"}, m.Keys())
+	})
+
+	t.Run("RetainsOrder", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		for i := 1; i < 10; i++ {
+			m.Set(strconv.Itoa(i), true)
+		}
+		assert.Equal(t,
+			[]string{"1", "2", "3", "4", "5", "6", "7", "8", "9"},
+			m.Keys())
+	})
+
+	t.Run("ReplacingKeyDoesntChangeOrder", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", true)
+		m.Set("bar", true)
+		m.Set("foo", false)
+		assert.Equal(t,
+			[]string{"foo", "bar"},
+			m.Keys())
+	})
+
+	t.Run("KeysAfterDelete", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", true)
+		m.Set("bar", true)
+		m.Delete("foo")
+		assert.Equal(t, []string{"bar"}, m.Keys())
+	})
+}
+
+func TestDelete(t *testing.T) {
+	t.Run("KeyDoesntExistReturnsFalse", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		assert.False(t, m.Delete("foo"))
+	})
+
+	t.Run("KeyDoesExist", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", nil)
+		assert.True(t, m.Delete("foo"))
+	})
+
+	t.Run("KeyNoLongerExists", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", nil)
+		m.Delete("foo")
+		_, exists := m.Get("foo")
+		assert.False(t, exists)
+	})
+
+	t.Run("KeyDeleteIsIsolated", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("foo", nil)
+		m.Set("bar", nil)
+		m.Delete("foo")
+		_, exists := m.Get("bar")
+		assert.True(t, exists)
+	})
+}
+
+func TestRingMap_Front(t *testing.T) {
+	t.Run("NilOnEmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		assert.Nil(t, m.Front())
+	})
+
+	t.Run("NilOnEmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("1", true)
+		assert.NotNil(t, m.Front())
+	})
+}
+
+func TestRingMap_Back(t *testing.T) {
+	t.Run("NilOnEmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		assert.Nil(t, m.Back())
+	})
+
+	t.Run("NilOnEmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		m.Set("1", true)
+		assert.NotNil(t, m.Back())
+	})
+}
+func TestRingMap_Concurrency(t *testing.T) {
+	t.Run("NilOnEmptyMap", func(t *testing.T) {
+		m := cache.NewRingMap(ringMapCapacity)
+		assert.Nil(t, m.Back())
+		for i := 0; i < 1000000; i++ {
+			go func() {
+				m.Set("foo", nil)
+				m.Exists("foo", nil)
+				m.Set("bar", nil)
+				m.Delete("foo")
+				m.Get("bar")
+				m.Delete("bar")
+			}()
+		}
+
+	})
+
+}
+
+func benchmarkMap_Set(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := make(map[int]bool)
+		for i := 0; i < b.N*multiplier; i++ {
+			m[i] = true
+		}
+	}
+}
+
+func BenchmarkMap_Set(b *testing.B) {
+	benchmarkMap_Set(1)(b)
+}
+
+func benchmarkRingMap_Set(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := cache.NewRingMap(ringMapCapacity)
+		for i := 0; i < b.N*multiplier; i++ {
+			m.Set(strconv.Itoa(i), true)
+		}
+	}
+}
+
+func BenchmarkRingMap_Set(b *testing.B) {
+	benchmarkRingMap_Set(1)(b)
+}
+
+func benchmarkMap_Get(multiplier int) func(b *testing.B) {
+	m := make(map[int]bool)
+	for i := 0; i < 1000*multiplier; i++ {
+		m[i] = true
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_ = m[i%1000*multiplier]
+		}
+	}
+}
+
+func BenchmarkMap_Get(b *testing.B) {
+	benchmarkMap_Get(1)(b)
+}
+
+func benchmarkRingMap_Get(multiplier int) func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	for i := 0; i < 1000*multiplier; i++ {
+		m.Set(strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			m.Get(strconv.Itoa(i % 1000 * multiplier))
+		}
+	}
+}
+
+func BenchmarkRingMap_Get(b *testing.B) {
+	benchmarkRingMap_Get(1)(b)
+}
+
+// prevent compiler from optimising Len away.
+var tempInt int
+
+func benchmarkRingMap_Len(multiplier int) func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	for i := 0; i < 1000*multiplier; i++ {
+		m.Set(strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		var temp int
+		for i := 0; i < b.N; i++ {
+			temp = m.Len()
+		}
+
+		// prevent compiler from optimising Len away.
+		tempInt = temp
+	}
+}
+
+func BenchmarkRingMap_Len(b *testing.B) {
+	benchmarkRingMap_Len(1)(b)
+}
+
+func benchmarkMap_Delete(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := make(map[int]bool)
+		for i := 0; i < b.N*multiplier; i++ {
+			m[i] = true
+		}
+
+		for i := 0; i < b.N; i++ {
+			delete(m, i)
+		}
+	}
+}
+
+func BenchmarkMap_Delete(b *testing.B) {
+	benchmarkMap_Delete(1)(b)
+}
+
+func benchmarkRingMap_Delete(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := cache.NewRingMap(ringMapCapacity)
+		for i := 0; i < b.N*multiplier; i++ {
+			m.Set(strconv.Itoa(i), true)
+		}
+
+		for i := 0; i < b.N; i++ {
+			m.Delete(strconv.Itoa(i))
+		}
+	}
+}
+
+func BenchmarkRingMap_Delete(b *testing.B) {
+	benchmarkRingMap_Delete(1)(b)
+}
+
+func benchmarkMap_Iterate(multiplier int) func(b *testing.B) {
+	m := make(map[int]bool)
+	for i := 0; i < 1000*multiplier; i++ {
+		m[i] = true
+	}
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, v := range m {
+				nothing(v)
+			}
+		}
+	}
+}
+func BenchmarkMap_Iterate(b *testing.B) {
+	benchmarkMap_Iterate(1)(b)
+}
+
+func benchmarkRingMap_Iterate(multiplier int) func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	for i := 0; i < 1000*multiplier; i++ {
+		m.Set(strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, key := range m.Keys() {
+				_, v := m.Get(key)
+				nothing(v)
+			}
+		}
+	}
+}
+
+func BenchmarkRingMap_Iterate(b *testing.B) {
+	benchmarkRingMap_Iterate(1)(b)
+}
+
+func benchmarkRingMap_Keys(multiplier int) func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	for i := 0; i < 1000*multiplier; i++ {
+		m.Set(strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			m.Keys()
+		}
+	}
+}
+
+func benchmarkMapString_Set(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := make(map[string]bool)
+		a := "12345678"
+		for i := 0; i < b.N*multiplier; i++ {
+			m[a+strconv.Itoa(i)] = true
+		}
+	}
+}
+
+func BenchmarkMapString_Set(b *testing.B) {
+	benchmarkMapString_Set(1)(b)
+}
+
+func benchmarkRingMapString_Set(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := cache.NewRingMap(ringMapCapacity)
+		a := "12345678"
+		for i := 0; i < b.N*multiplier; i++ {
+			m.Set(a+strconv.Itoa(i), true)
+		}
+	}
+}
+
+func BenchmarkRingMapString_Set(b *testing.B) {
+	benchmarkRingMapString_Set(1)(b)
+}
+
+func benchmarkMapString_Get(multiplier int) func(b *testing.B) {
+	m := make(map[string]bool)
+	a := "12345678"
+	for i := 0; i < 1000*multiplier; i++ {
+		m[a+strconv.Itoa(i)] = true
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_ = m[a+strconv.Itoa(i%1000*multiplier)]
+		}
+	}
+}
+
+func BenchmarkMapString_Get(b *testing.B) {
+	benchmarkMapString_Get(1)(b)
+}
+
+func benchmarkRingMapString_Get(multiplier int) func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	a := "12345678"
+	for i := 0; i < 1000*multiplier; i++ {
+		m.Set(a+strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			m.Get(a + strconv.Itoa(i%1000*multiplier))
+		}
+	}
+}
+
+func BenchmarkRingMapString_Get(b *testing.B) {
+	benchmarkRingMapString_Get(1)(b)
+}
+
+func benchmarkMapString_Delete(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := make(map[string]bool)
+		a := "12345678"
+		for i := 0; i < b.N*multiplier; i++ {
+			m[a+strconv.Itoa(i)] = true
+		}
+
+		for i := 0; i < b.N; i++ {
+			delete(m, a+strconv.Itoa(i))
+		}
+	}
+}
+
+func BenchmarkMapString_Delete(b *testing.B) {
+	benchmarkMapString_Delete(1)(b)
+}
+
+func benchmarkRingMapString_Delete(multiplier int) func(b *testing.B) {
+	return func(b *testing.B) {
+		m := cache.NewRingMap(ringMapCapacity)
+		a := "12345678"
+		for i := 0; i < b.N*multiplier; i++ {
+			m.Set(a+strconv.Itoa(i), true)
+		}
+
+		for i := 0; i < b.N; i++ {
+			m.Delete(a + strconv.Itoa(i))
+		}
+	}
+}
+
+func BenchmarkRingMapString_Delete(b *testing.B) {
+	benchmarkRingMapString_Delete(1)(b)
+}
+
+func benchmarkMapString_Iterate(multiplier int) func(b *testing.B) {
+	m := make(map[string]bool)
+	a := "12345678"
+	for i := 0; i < 1000*multiplier; i++ {
+		m[a+strconv.Itoa(i)] = true
+	}
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, v := range m {
+				nothing(v)
+			}
+		}
+	}
+}
+func BenchmarkMapString_Iterate(b *testing.B) {
+	benchmarkMapString_Iterate(1)(b)
+}
+
+func benchmarkRingMapString_Iterate(multiplier int) func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	a := "12345678"
+	for i := 0; i < 1000*multiplier; i++ {
+		m.Set(a+strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, key := range m.Keys() {
+				_, v := m.Get(key)
+				nothing(v)
+			}
+		}
+	}
+}
+
+func BenchmarkRingMapString_Iterate(b *testing.B) {
+	benchmarkRingMapString_Iterate(1)(b)
+}
+
+func BenchmarkRingMap_Keys(b *testing.B) {
+	benchmarkRingMap_Keys(1)(b)
+}
+
+func ExampleNewRingMap() {
+	m := cache.NewRingMap(ringMapCapacity)
+
+	m.Set("foo", "bar")
+	m.Set("qux", 1.23)
+	m.Set("123", true)
+
+	m.Delete("qux")
+
+	for _, key := range m.Keys() {
+		value, _ := m.Get(key)
+		fmt.Println(key, value)
+	}
+}
+
+func ExampleRingMap_Front() {
+	m := cache.NewRingMap(ringMapCapacity)
+	m.Set("1", true)
+	m.Set("2", true)
+
+	for el := m.Front(); el != nil; el = el.Next() {
+		fmt.Println(el)
+	}
+}
+
+func nothing(v interface{}) {
+	v = false
+}
+
+func benchmarkBigMap_Set() func(b *testing.B) {
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			m := make(map[int]bool)
+			for i := 0; i < 10000000; i++ {
+				m[i] = true
+			}
+		}
+	}
+}
+
+func BenchmarkBigMap_Set(b *testing.B) {
+	benchmarkBigMap_Set()(b)
+}
+
+func benchmarkBigRingMap_Set() func(b *testing.B) {
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			m := cache.NewRingMap(ringMapCapacity)
+			for i := 0; i < 10000000; i++ {
+				m.Set(strconv.Itoa(i), true)
+			}
+		}
+	}
+}
+
+func BenchmarkBigRingMap_Set(b *testing.B) {
+	benchmarkBigRingMap_Set()(b)
+}
+
+func benchmarkBigMap_Get() func(b *testing.B) {
+	m := make(map[int]bool)
+	for i := 0; i < 10000000; i++ {
+		m[i] = true
+	}
+
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			for i := 0; i < 10000000; i++ {
+				_ = m[i]
+			}
+		}
+	}
+}
+
+func BenchmarkBigMap_Get(b *testing.B) {
+	benchmarkBigMap_Get()(b)
+}
+
+func benchmarkBigRingMap_Get() func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	for i := 0; i < 10000000; i++ {
+		m.Set(strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			for i := 0; i < 10000000; i++ {
+				m.Get(strconv.Itoa(i))
+			}
+		}
+	}
+}
+
+func BenchmarkBigRingMap_Get(b *testing.B) {
+	benchmarkBigRingMap_Get()(b)
+}
+
+func benchmarkBigMap_Iterate() func(b *testing.B) {
+	m := make(map[int]bool)
+	for i := 0; i < 10000000; i++ {
+		m[i] = true
+	}
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, v := range m {
+				nothing(v)
+			}
+		}
+	}
+}
+func BenchmarkBigMap_Iterate(b *testing.B) {
+	benchmarkBigMap_Iterate()(b)
+}
+
+func benchmarkBigRingMap_Iterate() func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	for i := 0; i < 10000000; i++ {
+		m.Set(strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, key := range m.Keys() {
+				_, v := m.Get(key)
+				nothing(v)
+			}
+		}
+	}
+}
+
+func BenchmarkBigRingMap_Iterate(b *testing.B) {
+	benchmarkBigRingMap_Iterate()(b)
+}
+
+func benchmarkBigMapString_Set() func(b *testing.B) {
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			m := make(map[string]bool)
+			a := "1234567"
+			for i := 0; i < 10000000; i++ {
+				m[a+strconv.Itoa(i)] = true
+			}
+		}
+	}
+}
+
+func BenchmarkBigMapString_Set(b *testing.B) {
+	benchmarkBigMapString_Set()(b)
+}
+
+func benchmarkBigRingMapString_Set() func(b *testing.B) {
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			m := cache.NewRingMap(ringMapCapacity)
+			a := "1234567"
+			for i := 0; i < 10000000; i++ {
+				m.Set(a+strconv.Itoa(i), true)
+			}
+		}
+	}
+}
+
+func BenchmarkBigRingMapString_Set(b *testing.B) {
+	benchmarkBigRingMapString_Set()(b)
+}
+
+func benchmarkBigMapString_Get() func(b *testing.B) {
+	m := make(map[string]bool)
+	a := "1234567"
+	for i := 0; i < 10000000; i++ {
+		m[a+strconv.Itoa(i)] = true
+	}
+
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			for i := 0; i < 10000000; i++ {
+				_ = m[a+strconv.Itoa(i)]
+			}
+		}
+	}
+}
+
+func BenchmarkBigMapString_Get(b *testing.B) {
+	benchmarkBigMapString_Get()(b)
+}
+
+func benchmarkBigRingMapString_Get() func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	a := "1234567"
+	for i := 0; i < 10000000; i++ {
+		m.Set(a+strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for j := 0; j < b.N; j++ {
+			for i := 0; i < 10000000; i++ {
+				m.Get(a + strconv.Itoa(i))
+			}
+		}
+	}
+}
+
+func BenchmarkBigRingMapString_Get(b *testing.B) {
+	benchmarkBigRingMapString_Get()(b)
+}
+
+func benchmarkBigMapString_Iterate() func(b *testing.B) {
+	m := make(map[string]bool)
+	a := "12345678"
+	for i := 0; i < 10000000; i++ {
+		m[a+strconv.Itoa(i)] = true
+	}
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, v := range m {
+				nothing(v)
+			}
+		}
+	}
+}
+func BenchmarkBigMapString_Iterate(b *testing.B) {
+	benchmarkBigMapString_Iterate()(b)
+}
+
+func benchmarkBigRingMapString_Iterate() func(b *testing.B) {
+	m := cache.NewRingMap(ringMapCapacity)
+	a := "12345678"
+	for i := 0; i < 10000000; i++ {
+		m.Set(a+strconv.Itoa(i), true)
+	}
+
+	return func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			for _, key := range m.Keys() {
+				_, v := m.Get(key)
+				nothing(v)
+			}
+		}
+	}
+}
+
+func BenchmarkBigRingMapString_Iterate(b *testing.B) {
+	benchmarkBigRingMapString_Iterate()(b)
+}
+
+func BenchmarkAll(b *testing.B) {
+	b.Run("BenchmarkRingMap_Keys", BenchmarkRingMap_Keys)
+
+	b.Run("BenchmarkRingMap_Set", BenchmarkRingMap_Set)
+	b.Run("BenchmarkMap_Set", BenchmarkMap_Set)
+	b.Run("BenchmarkRingMap_Get", BenchmarkRingMap_Get)
+	b.Run("BenchmarkMap_Get", BenchmarkMap_Get)
+	b.Run("BenchmarkRingMap_Delete", BenchmarkRingMap_Delete)
+	b.Run("BenchmarkMap_Delete", BenchmarkMap_Delete)
+	b.Run("BenchmarkRingMap_Iterate", BenchmarkRingMap_Iterate)
+	b.Run("BenchmarkMap_Iterate", BenchmarkMap_Iterate)
+
+	b.Run("BenchmarkBigMap_Set", BenchmarkBigMap_Set)
+	b.Run("BenchmarkBigRingMap_Set", BenchmarkBigRingMap_Set)
+	b.Run("BenchmarkBigMap_Get", BenchmarkBigMap_Get)
+	b.Run("BenchmarkBigRingMap_Get", BenchmarkBigRingMap_Get)
+	b.Run("BenchmarkBigRingMap_Iterate", BenchmarkBigRingMap_Iterate)
+	b.Run("BenchmarkBigMap_Iterate", BenchmarkBigMap_Iterate)
+
+	b.Run("BenchmarkRingMapString_Set", BenchmarkRingMapString_Set)
+	b.Run("BenchmarkMapString_Set", BenchmarkMapString_Set)
+	b.Run("BenchmarkRingMapString_Get", BenchmarkRingMapString_Get)
+	b.Run("BenchmarkMapString_Get", BenchmarkMapString_Get)
+	b.Run("BenchmarkRingMapString_Delete", BenchmarkRingMapString_Delete)
+	b.Run("BenchmarkMapString_Delete", BenchmarkMapString_Delete)
+	b.Run("BenchmarkRingMapString_Iterate", BenchmarkRingMapString_Iterate)
+	b.Run("BenchmarkMapString_Iterate", BenchmarkMapString_Iterate)
+
+	b.Run("BenchmarkBigMapString_Set", BenchmarkBigMapString_Set)
+	b.Run("BenchmarkBigRingMapString_Set", BenchmarkBigRingMapString_Set)
+	b.Run("BenchmarkBigMapString_Get", BenchmarkBigMapString_Get)
+	b.Run("BenchmarkBigRingMapString_Get", BenchmarkBigRingMapString_Get)
+	b.Run("BenchmarkBigRingMapString_Iterate", BenchmarkBigRingMapString_Iterate)
+	b.Run("BenchmarkBigMapString_Iterate", BenchmarkBigMapString_Iterate)
+}

pkg/cache/subscription_types.go

@@ -0,0 +1,103 @@
+package cache
+
+import (
+	"encoding/json"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/duration"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+func RetrieveSubscriptionTypesList(client redis.Client, db queries.SubscriptionTypesInterface) ([]common.SubscriptionType, error) {
+	data, err := retrieveSubscriptionTypesRedis(client, uuid.Nil)
+	if err != nil && !errors.Is(err, redis.ErrInvalidResults) {
+		return nil, err
+	}
+	if data != nil {
+		return convertSubTypesList(data)
+	}
+
+	items, err := db.Select(&common.SubscriptionType{})
+	if err != nil {
+		return nil, err
+	}
+	if len(items) == 0 {
+		logger.Warn().Msg("no subscription types")
+		return []common.SubscriptionType{}, nil
+	}
+
+	for i := range items {
+		item := &items[i]
+		item.ClearDates()
+	}
+
+	err = storeSubscriptionTypesRedis(client, uuid.Nil, items)
+	if err != nil {
+		return nil, err
+	}
+
+	return items, nil
+}
+
+func RetrieveSubscriptionType(client redis.Client, db queries.SubscriptionTypesInterface, subtypeID uuid.UUID) (*common.SubscriptionType, error) {
+	data, err := retrieveSubscriptionTypesRedis(client, subtypeID)
+	if err != nil && !errors.Is(err, redis.ErrInvalidResults) {
+		return nil, err
+	}
+	if data != nil {
+		return convertSubType(data)
+	}
+
+	items, err := db.Select(&common.SubscriptionType{ID: subtypeID})
+	if err != nil {
+		return nil, err
+	}
+	if len(items) == 0 {
+		return nil, errors.Errorf("subscription type %v not found", subtypeID)
+	}
+
+	err = storeSubscriptionTypesRedis(client, subtypeID, items[0])
+	if err != nil {
+		return nil, err
+	}
+
+	return &items[0], nil
+}
+
+func storeSubscriptionTypesRedis(client redis.Client, subtypeID uuid.UUID, data interface{}) error {
+	return client.SetCache(redis.SubscriptionTypeListKey(subtypeID), data, duration.Hour)
+}
+
+func retrieveSubscriptionTypesRedis(client redis.Client, subtypeID uuid.UUID) ([]byte, error) {
+	key := redis.SubscriptionTypeListKey(subtypeID)
+	values, err := client.Get(key)
+	if err != nil {
+		return nil, err
+	}
+	if values == nil {
+		return nil, redis.ErrInvalidResults
+	}
+
+	data, ok := values.([]byte)
+	if !ok {
+		return nil, errors.New("unable to convert to []byte")
+	}
+
+	return data, nil
+}
+
+func convertSubTypesList(data []byte) ([]common.SubscriptionType, error) {
+	result := []common.SubscriptionType{}
+	err := json.Unmarshal(data, &result)
+	return result, err
+}
+
+func convertSubType(data []byte) (*common.SubscriptionType, error) {
+	result := common.SubscriptionType{}
+	err := json.Unmarshal(data, &result)
+	return &result, err
+}

pkg/cache/subscription_types_test.go

@@ -0,0 +1,88 @@
+package cache_test
+
+import (
+	"fmt"
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/testhelper"
+	"github.com/google/uuid"
+)
+
+func TestRetrieveSubscriptionTypes(t *testing.T) {
+	t.Skip()
+	query := queries.SubscriptionTypes{}
+	client := redis.NewClient()
+	subtype, err := setupTestSubscritionType(&query)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer cleanupTestSubscriptionType(client, &query, subtype)
+
+	list, err := cache.RetrieveSubscriptionTypesList(client, &query)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(list) == 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "From DB", "1 or more", len(list))
+	}
+
+	list, err = cache.RetrieveSubscriptionTypesList(client, &query)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(list) == 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "From Redis", "1 or more", len(list))
+	}
+
+	item, err := cache.RetrieveSubscriptionType(client, &query, subtype.ID)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if item.Name != subtype.Name {
+		t.Errorf(testhelper.TestErrorTemplate, "RetrieveSubscriptionType From DB", subtype.Name, item.Name)
+	}
+
+	item, err = cache.RetrieveSubscriptionType(client, &query, subtype.ID)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if item.Name != subtype.Name {
+		t.Errorf(testhelper.TestErrorTemplate, "RetrieveSubscriptionType From Redis", subtype.Name, item.Name)
+	}
+
+	item, err = cache.RetrieveSubscriptionType(client, &query, uuid.New())
+	if err == nil {
+		t.Errorf(testhelper.TestErrorTemplate, "Invalid subscription type id", "found out", err)
+	}
+	if item != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "Invalid subscription type item", nil, item)
+	}
+}
+
+func setupTestSubscritionType(query *queries.SubscriptionTypes) (*common.SubscriptionType, error) {
+	subtype := common.SubscriptionType{
+		Name:          fmt.Sprintf("Test type %s", uuid.New().String()),
+		Destination:   "ICC",
+		Description:   "test",
+		Currency:      "USD",
+		Price:         10000, // $100 USD
+		DurationValue: 1,
+		DurationUnit:  "Hours",
+	}
+	_, err := query.Insert(&subtype)
+
+	return &subtype, err
+}
+
+func cleanupTestSubscriptionType(client redis.Client, query *queries.SubscriptionTypes, subtype *common.SubscriptionType) {
+	client.Delete(redis.SubscriptionTypeListKey(subtype.ID))
+	client.Delete(redis.SubscriptionTypeListKey(uuid.Nil))
+	query.Delete(subtype)
+}

pkg/cache/vehicle_config.go

@@ -0,0 +1,197 @@
+package cache
+
+import (
+	"encoding/json"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/mongo"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/utils/envtool"
+
+	"fiskerinc.com/modules/utils/elptr"
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+const (
+	ENABLE_DBG_MASK_EV_NAME     = "ENABLE_DEBUGMASK"
+	ENABLE_DBG_MASK_VAL_FALSE   = "0"
+	ENABLE_DBG_MASK_VAL_TRUE    = "1"
+	ENABLE_DBG_MASK_VAL_DEFAULT = ENABLE_DBG_MASK_VAL_FALSE
+)
+
+// This flag is to decide whether retrieved value of DebugMask is to be passed to TrexCfg or not.
+// When the flag is true, the retrieved value is passed; else no value is passed.
+// The value of flag is fetched from the specific environmental variable. If that environmental
+// variable is not present / not defined, we assume the flag itself to be FALSE. That is the
+// default value (FALSE) of the environmental variable. When user/developer has set this evironmental
+// variable correctly, the flag can become TRUE in which case the value is passed to TrexCfg.
+var ENABLE_DEBUG_MASK = DbgMaskEnabled()
+
+// method introduced so as unit testing is easier otherwise not necessary since environment variables
+// can't be changed so easily subsequent to a process start (meaning revaluation at runtime of no much use).
+func DbgMaskEnabled() bool {
+	return envtool.GetEnv(ENABLE_DBG_MASK_EV_NAME, ENABLE_DBG_MASK_VAL_DEFAULT) == ENABLE_DBG_MASK_VAL_TRUE
+}
+
+func RetrieveVehicleConfig(r redis.Client, m mongo.Client, id string) (*common.TRexConfigResponse, error) {
+	config := &common.TRexConfigResponse{}
+
+	reply, err := checkCacheForVehicleConfig(r, id)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if reply != nil {
+		err = json.Unmarshal(reply, config)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		if config.CANBus.DTCEnabled == nil {
+			config.CANBus.DTCEnabled = elptr.ElPtr(false)
+		}
+		return config, nil
+	}
+
+	config.LogLevel = common.Critical
+	config.Log = &common.LogConfig{
+		Matches: []common.LogConfigChannel{
+			{
+				Channel: common.ChannelCMD,
+				Level:   common.Trace,
+			},
+		},
+	}
+
+	config.CANBus.Enabled = true
+	config.CANBus.DataLogger = true
+
+	filters := make(FiltersMap)
+
+	f, err := checkFleetsDBForVehicleConfig(m, id)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+	}
+	if f != nil {
+		config.CANBus = f.CANBus
+		config.LogLevel = f.LogLevel
+		filters.AppendFilters(f.CANBus.Filters)
+	}
+
+	v, err := checkVehiclesDBForVehicleConfig(m, id)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+	}
+	if v != nil {
+		config.CANBus = v.CANBus
+		config.LogLevel = v.LogLevel
+		config.DLTEnabled = v.DLTEnabled
+		config.DLTLevel = v.DLTLevel
+		// we should evaluate at run-time, not just at start-up time
+		if ENABLE_DEBUG_MASK {
+			config.DebugMask = v.DebugMask
+		}
+		config.IDPSEnabled = v.IDPSEnabled
+
+		filters.AppendFilters(v.CANBus.Filters)
+	}
+
+	config.CANBus.Filters = filters.ToSlice()
+	if config.CANBus.DTCEnabled == nil {
+		config.CANBus.DTCEnabled = elptr.ElPtr(false)
+	}
+	err = setCacheForVehicleConfig(r, id, config)
+	return config, err
+}
+
+func checkCacheForVehicleConfig(r redis.Client, id string) ([]byte, error) {
+	key := redis.CarConfigKey(id)
+
+	reply, err := redigo.Bytes(r.Execute("GET", key))
+	if err != nil {
+		if errors.Is(err, redigo.ErrNil) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return reply, nil
+}
+
+func checkVehiclesDBForVehicleConfig(m mongo.Client, id string) (*mongo.Vehicle, error) {
+	return m.GetVehicles().FindVehicle(&mongo.Vehicle{VIN: id})
+}
+
+func checkFleetsDBForVehicleConfig(m mongo.Client, id string) (*mongo.Fleet, error) {
+	return m.GetFleets().GetCANBusForVehicle(id)
+}
+
+func setCacheForVehicleConfig(r redis.Client, id string, config *common.TRexConfigResponse) error {
+	key := redis.CarConfigKey(id)
+
+	data, err := json.Marshal(config)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	batch := redis.NewRedisBatchCommands()
+	batch.Add("SET", key, data)
+	batch.Add("EXPIRE", key, redisObjectExpire)
+
+	_, err = r.ExecuteBatch(batch)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+func RemoveCacheConfigForVehicles(r redis.Client, vins []string) error {
+	batch := redis.NewRedisBatchCommands()
+	for _, vin := range vins {
+		batch.Add("DEL", redis.CarConfigKey(vin))
+	}
+
+	_, err := r.ExecuteBatch(batch)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+type IntervalEdgeMask struct {
+	Interval *int
+	EdgeMask *common.BinaryHex
+}
+
+type FiltersMap map[string]IntervalEdgeMask
+
+func (f FiltersMap) AppendFilters(filters []common.CANFilter) {
+	for _, filter := range filters {
+		if filter.EdgeMask != nil && filter.EdgeMask.String() != "" {
+			f[filter.CANID] = IntervalEdgeMask{
+				EdgeMask: filter.EdgeMask,
+			}
+		} else if filter.Interval != nil {
+			f[filter.CANID] = IntervalEdgeMask{
+				Interval: filter.Interval,
+			}
+		}
+	}
+}
+
+func (f FiltersMap) ToSlice() []common.CANFilter {
+	filters := make([]common.CANFilter, 0, len(f))
+
+	for k, v := range f {
+		filters = append(filters, common.CANFilter{
+			CANID:    k,
+			Interval: v.Interval,
+			EdgeMask: v.EdgeMask,
+		})
+	}
+
+	return filters
+}

pkg/cache/vehicle_config_test.go

@@ -0,0 +1,203 @@
+package cache_test
+
+import (
+	"encoding/json"
+	"sort"
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/mongo"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/testhelper"
+	"fiskerinc.com/modules/utils/elptr"
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRetrieveVehicleConfig(t *testing.T) {
+	setupRedisMock()
+	id := "TESTVIN1234567"
+
+	mockRedis = &mockRedisVehicleConfig{}
+	config, err := cache.RetrieveVehicleConfig(mockRedis, mongo.NewMockClient(), id)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	data, err := json.Marshal(&config)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	assert.Equal(t, `{"canbus":{"enabled":false,"data_logger_enabled":false,"dtc_enabled":false},"log_level":"trace"}`, string(data))
+
+	mockRedis = &mockRedisNoVehicleConfig{}
+	config, err = cache.RetrieveVehicleConfig(mockRedis, mongo.NewMockClient(), id)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	data, err = json.Marshal(&config)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	assert.Equal(t, `{"canbus":{"enabled":false,"data_logger_enabled":false,"dtc_enabled":false},"log_level":"trace","log":{"matches":[{"channel":"cmd","level":"trace"}]}}`, string(data))
+}
+
+func TestRetrieveVehicleConfigDbgMask(t *testing.T) {
+	setupRedisMock()
+	id := "TESTVIN1234567"
+	mockVehicle := mongo.Vehicle{VIN: id}
+	mockRedis = &mockRedisNoVehicleConfig{}
+
+	// validate that by default, retrieved debug value IS NOT passed to trxCfg
+	trxCfg, err := cache.RetrieveVehicleConfig(mockRedis, mongo.NewMockClient(), id)
+	existingValue := trxCfg.DebugMask
+	assert.Nil(t, err)
+	assert.NotNil(t, trxCfg)
+	// assert that trxCfg value is unchanged
+	assert.Equal(t, trxCfg.DebugMask, existingValue)
+
+	// let us try to enable
+	// the mock for redis is with no data so that code will fall through to the DB part
+	// we ensure that what we get from DB has speific debug mask which should be
+	// passed to Trex when the flag is true
+	t.Setenv(cache.ENABLE_DBG_MASK_EV_NAME, cache.ENABLE_DBG_MASK_VAL_TRUE)
+	cache.ENABLE_DEBUG_MASK = cache.DbgMaskEnabled()
+	mmc := mongo.NewMockMongoClient()
+	mockVehicle.DebugMask = "test"
+	mmc.GetVehicles().AddVehicle(&mockVehicle)
+	trxCfg, _ = cache.RetrieveVehicleConfig(mockRedis, mmc, id)
+	// now validate that Trex config got the value as set in the mocked vehicle
+	// (presumed as retrieved)
+	assert.Equal(t, trxCfg.DebugMask, mockVehicle.DebugMask)
+
+	// now set back the env variable so new values don't flow to trex
+	t.Setenv(cache.ENABLE_DBG_MASK_EV_NAME, cache.ENABLE_DBG_MASK_VAL_FALSE)
+	cache.ENABLE_DEBUG_MASK = cache.DbgMaskEnabled()
+	oldMask := mockVehicle.DebugMask
+	mockVehicle.DebugMask = "new-value"
+	// skipping adding to the cache/DB as we still had the valid reference
+	trxCfg, _ = cache.RetrieveVehicleConfig(mockRedis, mmc, id)
+	// assert that trex does not have new value
+	assert.NotEqual(t, trxCfg.DebugMask, oldMask)
+}
+
+func TestFiltersMap(t *testing.T) {
+	filters := make(cache.FiltersMap)
+
+	if len(filters) != 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 0, len(filters))
+		return
+	}
+
+	emptyHex := common.NewBinaryHex([]byte{})
+	bhex := common.BinaryHex("123")
+	filters.AppendFilters(
+		[]common.CANFilter{
+			{CANID: "123", Interval: elptr.ElPtr(123)},
+			{CANID: "456", Interval: elptr.ElPtr(456)},
+			{CANID: "789", EdgeMask: &emptyHex},
+			{CANID: "901", EdgeMask: &bhex},
+			{CANID: "222", Interval: elptr.ElPtr(123), EdgeMask: &bhex},
+			{CANID: "333", Interval: elptr.ElPtr(0)},
+		},
+	)
+	if len(filters) != 5 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 5, len(filters))
+		return
+	}
+
+	interval, ok := filters["123"]
+	if !ok || *interval.Interval != 123 && interval.EdgeMask != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 123, "error")
+		return
+	}
+
+	interval, ok = filters["456"]
+
+	if !ok || *interval.Interval != 456 && interval.EdgeMask != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 456, "error")
+		return
+	}
+
+	interval, ok = filters["789"]
+	if ok || interval.EdgeMask != nil || interval.Interval != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", emptyHex, "error")
+		return
+	}
+
+	interval, ok = filters["901"]
+	if !ok || interval.EdgeMask.String() != bhex.String() && interval.Interval != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", bhex, "error")
+		return
+	}
+
+	interval, ok = filters["222"]
+	if !ok || interval.EdgeMask.String() != bhex.String() && interval.Interval != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", bhex, "error")
+		return
+	}
+
+	interval, ok = filters["333"]
+	if !ok || interval.EdgeMask != nil && *interval.Interval != 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", nil, "error")
+		return
+	}
+
+	slice := filters.ToSlice()
+	if len(slice) != 5 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 5, len(slice))
+		return
+	}
+
+	sort.Slice(slice, func(i, j int) bool {
+		return slice[i].CANID < slice[j].CANID
+	})
+
+	if slice[0].CANID != "123" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "123", slice[0].CANID)
+		return
+	}
+
+	if slice[1].CANID != "222" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "222", slice[1].CANID)
+		return
+	}
+
+	if slice[2].CANID != "333" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "333", slice[2].CANID)
+		return
+	}
+
+	if slice[3].CANID != "456" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "456", slice[0].CANID)
+		return
+	}
+
+	if slice[4].CANID != "901" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "901", slice[0].CANID)
+		return
+	}
+
+}
+
+type mockRedisVehicleConfig struct {
+	redis.Connection
+}
+
+func (c *mockRedisVehicleConfig) Execute(command ...interface{}) (interface{}, error) {
+	config := common.TRexConfigResponse{}
+	data, _ := json.Marshal(config)
+	return data, nil
+}
+
+type mockRedisNoVehicleConfig struct {
+	redis.Connection
+}
+
+func (c *mockRedisNoVehicleConfig) Execute(command ...interface{}) (interface{}, error) {
+	return nil, redigo.ErrNil
+}
+
+func (c *mockRedisNoVehicleConfig) ExecuteBatch(batch *redis.RedisBatchCommands) (interface{}, error) {
+	return nil, nil
+}

pkg/cache/vehicle_state.go

@@ -0,0 +1,583 @@
+package cache
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"fiskerinc.com/modules/common"
+	dt "fiskerinc.com/modules/dbc/state"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/utils/querystring"
+
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+const UPDATED_TIME_FORMAT = "2006-01-02T15:04:05Z"
+
+type stateParser func(state *common.CarState, key string, value []byte) error
+
+func NewVehicleState(client redis.ClientPoolInterface) *VehicleState {
+	return &VehicleState{client: client}
+}
+
+type VehicleState struct {
+	client redis.ClientPoolInterface
+}
+
+func (v *VehicleState) Get(vin string) (common.CarState, error) {
+	var state common.CarState
+
+	values, err := v.queryVehicleState(vin)
+	if err != nil {
+		return state, err
+	}
+
+	state, err = v.ParsePayloadForVehicleState(values)
+	if err != nil {
+		return state, err
+	}
+
+	return state, nil
+}
+
+func (v *VehicleState) queryVehicleState(vin string) ([]interface{}, error) {
+	var payload []interface{}
+	client := v.client.GetFromPool()
+	defer client.Close()
+
+	batch := redis.NewRedisBatchCommands()
+	batch.Add("SISMEMBER", redis.CarSessionsKey(), vin)
+	batch.Add("SISMEMBER", redis.HMISessionsKey(), vin)
+	batch.Add("HGETALL", redis.CarStateHashKey(vin))
+
+	payload, err := redigo.Values(client.ExecuteBatch(batch))
+	if err != nil {
+		return payload, errors.WithStack(err)
+	}
+
+	return payload, nil
+}
+
+func (v *VehicleState) ParsePayloadForVehicleState(payload []interface{}) (common.CarState, error) {
+	var state common.CarState
+
+	if len(payload) != 3 {
+		return state, redis.ErrInvalidResults
+	}
+
+	online, err := redigo.Bool(payload[0], nil)
+	if err != nil {
+		return state, err
+	} else {
+		state.Online = online
+	}
+
+	online, err = redigo.Bool(payload[1], nil)
+	if err != nil {
+		return state, errors.WithStack(err)
+	} else {
+		state.OnlineHMI = online
+	}
+
+	err = v.parseCarStatePayload(&state, payload[2])
+
+	return state, err
+}
+
+func (v *VehicleState) parseCarStatePayload(state *common.CarState, payload interface{}) error {
+	stateValues, err := redigo.Values(payload, nil)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	if len(stateValues)%2 != 0 {
+		return errors.New("object does not contain equal number of key value pairs")
+	}
+
+	err = v.parseStateValues(state, stateValues, parseCarState)
+
+	return err
+}
+
+func (v *VehicleState) parseStateValues(state *common.CarState, stateValues []interface{}, parser stateParser) error {
+	for i := 0; i < len(stateValues); i += 2 {
+		key, okKey := stateValues[i].([]byte)
+		value, okValue := stateValues[i+1].([]byte)
+
+		if !okKey || !okValue {
+			return errors.New("cannot parse object into car state")
+		}
+
+		err := parser(state, string(key), value)
+		// log error, do not return error so we can read other properties for digital twin
+		if err != nil {
+			logger.Err(err).Send()
+		}
+	}
+
+	return nil
+}
+
+func parseCarState(state *common.CarState, key string, value []byte) error {
+	var err error
+	val := string(value)
+
+	switch key {
+	case dt.VCU_VehChrgDchgMod:
+		state.GetVCU0x260().ChargeType = string(value)
+	case dt.BMS_Bat_SoC_usable:
+		state.GetStateOfCharge().Usable, err = strconv.Atoi(val)
+	case dt.BMS_Bat_SOH:
+		state.GetStateOfCharge().Health, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_LeFrntWinPosnInfo:
+		state.GetWindows().LeftFront, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_RiFrntWinPosnInfo:
+		state.GetWindows().RightFront, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_LeReWinPosnInfo:
+		state.GetWindows().LeftRear, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_RiReWinPosnInfo:
+		state.GetWindows().RightRear, err = strconv.Atoi(val)
+	case dt.BMS_PwrBattRmngCpSOC:
+		state.GetBattery().Percent, err = strconv.Atoi(val)
+	case dt.BCM_ReDefrstHeatgCmd:
+		state.GetRearDefrost().On, err = strconv.ParseBool(val)
+	case dt.BCM_PasFrntDoorSts:
+		state.GetDoors().RightFront, err = strconv.ParseBool(val)
+	case dt.BCM_DrFrntDoorSts:
+		state.GetDoors().LeftFront, err = strconv.ParseBool(val)
+	case dt.BCM_FrntDrDoorLockSts:
+		state.GetLocks().Driver, err = notValue(strconv.ParseBool(val))
+	case dt.BCM_CenLockSwtSts:
+		state.GetLocks().All = (val == "2")
+	case dt.BCM_RiReDoorSts:
+		state.GetDoors().RightRear, err = strconv.ParseBool(val)
+	case dt.BCM_LeReDoorSts:
+		state.GetDoors().LeftRear, err = strconv.ParseBool(val)
+	case dt.BCM_FrntHoodLidSts:
+		state.GetDoors().Hood, err = strconv.ParseBool(val)
+	case dt.PLGM_TrSts:
+		state.GetDoors().Trunk, err = strconv.ParseBool(val)
+	case dt.BCM_SunroofPosnInfo:
+		state.GetSunroof().Sunroof, err = strconv.Atoi(val)
+	case dt.BCM_AP_TL_LeReWinPosnInfo:
+		state.GetMiscWindows().LeftRearQuarter, err = strconv.Atoi(val)
+	case dt.BCM_AP_TL_RiReWinPosnInfo:
+		state.GetMiscWindows().RightRearQuarter, err = strconv.Atoi(val)
+	case dt.BCM_AP_RW_WinPosnInfo:
+		state.GetMiscWindows().RearWindshield, err = strconv.Atoi(val)
+	case dt.BMS_BattAvrgT:
+		state.GetCellTemperature().AvgBatteryTemp, err = strconv.Atoi(val)
+	case dt.ECC_OutdT:
+		state.GetAmbientTemperature().Temperature, err = strconv.Atoi(val)
+	case dt.BCM_HeatedSteerWhlSt:
+		state.GetSteeringWheelHeat().On, err = strconv.ParseBool(val)
+	case dt.ESP_VehSpd:
+		state.GetVehicleSpeed().Speed, err = strconv.ParseFloat(val, 64)
+	case dt.VCU_DrvgMilg:
+		state.GetMaxRange().MaxMiles, err = strconv.Atoi(val)
+	case dt.PSM_PassSeatHeatgSts:
+		state.GetPassengerSeatHeat().Level, err = strconv.Atoi(val)
+	case dt.DSMC_DrvrSeatHeatgSts:
+		state.GetDriverSeatHeat().Level, err = strconv.Atoi(val)
+	case dt.ICC_TotMilg_ODO:
+		state.GetBattery().TotalMileageOdometer, err = querystring.ConvertStringToInt(val)
+	case dt.VCU_DCChrgRmngTi, dt.BMS_RmChrgTi_TrgtSoC:
+		state.GetChargingMetrics().RemainingChargingTime, err = strconv.Atoi(val)
+	case dt.IBS_BatteryVoltage:
+		state.GetBattery().BatteryVoltage, err = strconv.ParseFloat(val, 64)
+		state.GetBattery12V().IBS_BatteryVoltage = ref(state.GetBattery().BatteryVoltage)
+	case dt.VCU_GearSig:
+		var gear int
+		gear, err = strconv.Atoi(val)
+		state.GetGear().InPark = (gear <= 2)
+	case dt.BMS_RmChrgTi_FullChrg:
+		state.GetChargingMetrics().RemainingChargingTimeFull, err = strconv.Atoi(val)
+	case dt.ECC_InsdT:
+		state.GetCabinClimate().InternalTemperature, err = strconv.Atoi(val)
+	case dt.ECC_RemTSetSts:
+		state.GetCabinClimate().CabinTemperature, err = strconv.Atoi(val)
+	case dt.TBOX_GPSHei:
+		state.GetLocation().Altitude, err = strconv.ParseFloat(val, 64)
+	case dt.TBOX_GPSLongi:
+		state.GetLocation().Longitude, err = strconv.ParseFloat(val, 64)
+	case dt.TBOX_GPSLati:
+		state.GetLocation().Latitude, err = strconv.ParseFloat(val, 64)
+	case dt.DBC_VERSION:
+		state.DBCVersion = val
+	case dt.TREX_VERSION:
+		state.TRexVersion = val
+	case dt.TREX_IP:
+		state.IP = val
+	case dt.UPDATED_AT:
+		var t time.Time
+		t, err = time.Parse(UPDATED_TIME_FORMAT, strings.Trim(val, "\""))
+		if !t.IsZero() {
+			state.UpdatedAt = ref(t)
+		}
+	case dt.VCU_VehSt:
+		state.GetSafeState().VehicleSafeState = val == dt.VCU_VehSt_Safestate
+	case dt.VCU_VcuState:
+		state.GetSafeState().VCUSafeState = val == dt.VCU_VcuState_Safestate
+	case dt.MCU_F_ActSafeSt:
+		state.GetSafeState().MCUFrontSafeState = val == dt.MCU_F_ActSafeSt_AS0 || val == dt.MCU_F_ActSafeSt_ASC || val == dt.MCU_F_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_ActSafeSt:
+		state.GetSafeState().MCURearSafeState = val == dt.MCU_R_ActSafeSt_AS0 || val == dt.MCU_R_ActSafeSt_ASC || val == dt.MCU_R_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_Decoup_State:
+		state.GetSafeState().MCURearDecoupState = val == dt.MCU_R_Decoup_State_Connected
+	case dt.MCU_F_CrtMod:
+		state.GetSafeState().MCUFrontInverterError = val == dt.MCU_F_CrtMod_Internal_inverter_error || val == dt.MCU_F_CrtMod_Invalid
+	case dt.MCU_R_CrtMod:
+		state.GetSafeState().MCURearInverterError = val == dt.MCU_R_CrtMod_Internal_inverter_error || val == dt.MCU_R_CrtMod_Invalid
+	case dt.ACU_Drvr_Occpt_St:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.DriverOccupySeatState = ref(vi)
+	case dt.BCM_PwrMod:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.PowerMode = ref(vi)
+	case dt.PWC_ChrgSts:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.ChargingStatus = ref(vi)
+	case dt.VCU_RdyLamp:
+		state.GetVehicleReadyState().IsVehicleReady, err = strconv.ParseBool(val)
+	// New untested signals
+	// case dt.IBS_SOCUpperTolerance:
+	// 	var vi float64
+	// 	vi, err = strconv.ParseFloat(val, 64)
+	// 	state.GetExpandedSignals().IBS_SOCUpperTolerance = ref(vi)
+	// case dt.IBS_SOCLowerTolerance:
+	// 	var vi float64
+	// 	vi, err = strconv.ParseFloat(val, 64)
+	// 	state.GetExpandedSignals().IBS_SOCLowerTolerance = ref(vi)
+	case dt.IBS_StateOfCharge:
+		var vi float64
+		vi, err = strconv.ParseFloat(val, 64)
+		state.GetBattery12V().IBS_StateOfCharge = ref(vi)
+	case dt.IBS_StateOfHealth:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetBattery12V().IBS_StateOfHealth = ref(vi)
+	case dt.IBS_NominalCapacity:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().IBS_NominalCapacity = ref(vi)
+	case dt.IBS_AvailableCapacity:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().IBS_AvailableCapacity = ref(vi)
+	case dt.BCM_TotMilg_ODO:
+		var vi float64
+		vi, err = strconv.ParseFloat(val, 64)
+		state.GetExpandedSignals().BCM_TotMilg_ODO = ref(vi)
+	case dt.BMS_SwVersS:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVersS = ref(vi)
+	case dt.BMS_SwVersM:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVersM = ref(vi)
+	case dt.BMS_SwVers:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVers = ref(vi)
+	case dt.BMS_AccueDchaTotAh:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_AccueDchaTotAh = ref(vi)
+	case dt.BMS_AccueChrgTotAh:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_AccueChrgTotAh = ref(vi)
+	case dt.TBOX_Heading:
+		state.GetLocation().Heading, err = strconv.ParseFloat(val, 64)
+	case dt.PKC_KeyStsMod:
+		state.GetGear().Immobilizer = val
+	}
+
+	return errors.WithStack(err)
+}
+
+func ParseCarState(state *common.CarState, key string, value interface{}) (err error) {
+	found := false
+	ok := false
+	switch key {
+	case dt.VCU_VehChrgDchgMod:
+		found = true
+		state.GetVCU0x260().ChargeType = value.(string)
+	case dt.BMS_Bat_SoC_usable:
+		found = true
+		state.GetStateOfCharge().Usable, ok = value.(int)
+	case dt.BMS_Bat_SOH:
+		found = true
+		state.GetStateOfCharge().Health, ok = value.(int)
+	case dt.BCM_AP_FL_LeFrntWinPosnInfo:
+		found = true
+		state.GetWindows().LeftFront, ok = value.(int)
+	case dt.BCM_AP_FL_RiFrntWinPosnInfo:
+		found = true
+		state.GetWindows().RightFront, ok = value.(int)
+	case dt.BCM_AP_FL_LeReWinPosnInfo:
+		found = true
+		state.GetWindows().LeftRear, ok = value.(int)
+	case dt.BCM_AP_FL_RiReWinPosnInfo:
+		found = true
+		state.GetWindows().RightRear, ok = value.(int)
+	case dt.BMS_PwrBattRmngCpSOC:
+		found = true
+		state.GetBattery().Percent, ok = value.(int)
+	case dt.BCM_ReDefrstHeatgCmd:
+		found = true
+		state.GetRearDefrost().On, ok = value.(bool)
+	case dt.BCM_PasFrntDoorSts:
+		found = true
+		state.GetDoors().RightFront, ok = value.(bool)
+	case dt.BCM_DrFrntDoorSts:
+		found = true
+		state.GetDoors().LeftFront, ok = value.(bool)
+	case dt.BCM_FrntDrDoorLockSts:
+		found = true
+		var vv bool
+		vv, ok = value.(bool)
+		state.GetLocks().Driver = !vv
+	case dt.BCM_CenLockSwtSts:
+		found = true
+		state.GetLocks().All = strconv.Itoa(value.(int)) == "2"
+	case dt.BCM_RiReDoorSts:
+		found = true
+		state.GetDoors().RightRear, ok = value.(bool)
+	case dt.BCM_LeReDoorSts:
+		found = true
+		state.GetDoors().LeftRear, ok = value.(bool)
+	case dt.BCM_FrntHoodLidSts:
+		found = true
+		state.GetDoors().Hood, ok = value.(bool)
+	case dt.PLGM_TrSts:
+		found = true
+		state.GetDoors().Trunk, ok = value.(bool)
+	case dt.BCM_SunroofPosnInfo:
+		found = true
+		state.GetSunroof().Sunroof, ok = value.(int)
+	case dt.BCM_AP_TL_LeReWinPosnInfo:
+		found = true
+		state.GetMiscWindows().LeftRearQuarter, ok = value.(int)
+	case dt.BCM_AP_TL_RiReWinPosnInfo:
+		found = true
+		state.GetMiscWindows().RightRearQuarter, ok = value.(int)
+	case dt.BCM_AP_RW_WinPosnInfo:
+		found = true
+		state.GetMiscWindows().RearWindshield, ok = value.(int)
+	case dt.BMS_BattAvrgT:
+		found = true
+		state.GetCellTemperature().AvgBatteryTemp, ok = value.(int)
+	case dt.ECC_OutdT:
+		found = true
+		state.GetAmbientTemperature().Temperature, ok = value.(int)
+	case dt.BCM_HeatedSteerWhlSt:
+		found = true
+		state.GetSteeringWheelHeat().On, ok = value.(bool)
+	case dt.ESP_VehSpd:
+		found = true
+		state.GetVehicleSpeed().Speed, ok = value.(float64)
+	case dt.VCU_DrvgMilg:
+		found = true
+		state.GetMaxRange().MaxMiles, ok = value.(int)
+	case dt.PSM_PassSeatHeatgSts:
+		found = true
+		state.GetPassengerSeatHeat().Level, ok = value.(int)
+	case dt.DSMC_DrvrSeatHeatgSts:
+		found = true
+		state.GetDriverSeatHeat().Level, ok = value.(int)
+	case dt.ICC_TotMilg_ODO:
+		found = true
+		// Seems wierd its sometimes an int, sometimes a float
+		state.GetBattery().TotalMileageOdometer, ok = value.(int)
+	case dt.VCU_DCChrgRmngTi, dt.BMS_RmChrgTi_TrgtSoC:
+		found = true
+		state.GetChargingMetrics().RemainingChargingTime, ok = value.(int)
+	case dt.IBS_BatteryVoltage:
+		found = true
+		state.GetBattery().BatteryVoltage, ok = value.(float64)
+		state.GetBattery12V().IBS_BatteryVoltage = ref(state.GetBattery().BatteryVoltage)
+	case dt.VCU_GearSig:
+		found = true
+		var gear int
+		gear, ok = value.(int)
+		state.GetGear().InPark = (gear <= 2)
+	case dt.BMS_RmChrgTi_FullChrg:
+		found = true
+		state.GetChargingMetrics().RemainingChargingTimeFull, ok = value.(int)
+	case dt.ECC_InsdT:
+		found = true
+		state.GetCabinClimate().InternalTemperature, ok = value.(int)
+	case dt.ECC_RemTSetSts:
+		found = true
+		state.GetCabinClimate().CabinTemperature, ok = value.(int)
+	case dt.TBOX_GPSHei:
+		found = true
+		state.GetLocation().Altitude, ok = value.(float64)
+	case dt.TBOX_GPSLongi:
+		found = true
+		state.GetLocation().Longitude, ok = value.(float64)
+	case dt.TBOX_GPSLati:
+		found = true
+		state.GetLocation().Latitude, ok = value.(float64)
+	case dt.DBC_VERSION:
+		found = true
+		state.DBCVersion = value.(string)
+	case dt.TREX_VERSION:
+		found = true
+		state.TRexVersion = value.(string)
+	case dt.TREX_IP:
+		found = true
+		state.IP = value.(string)
+	case dt.UPDATED_AT:
+		var t time.Time
+		t, err = time.Parse(UPDATED_TIME_FORMAT, strings.Trim(value.(string), "\""))
+		if !t.IsZero() {
+			state.UpdatedAt = ref(t)
+		}
+	case dt.VCU_VehSt:
+		found = true
+		state.GetSafeState().VehicleSafeState = strconv.Itoa(value.(int)) == dt.VCU_VehSt_Safestate
+	case dt.VCU_VcuState:
+		found = true
+		state.GetSafeState().VCUSafeState = strconv.Itoa(value.(int)) == dt.VCU_VcuState_Safestate
+	case dt.MCU_F_ActSafeSt:
+		found = true
+		state.GetSafeState().MCUFrontSafeState = strconv.Itoa(value.(int)) == dt.MCU_F_ActSafeSt_AS0 || strconv.Itoa(value.(int)) == dt.MCU_F_ActSafeSt_ASC || strconv.Itoa(value.(int)) == dt.MCU_F_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_ActSafeSt:
+		found = true
+		state.GetSafeState().MCURearSafeState = strconv.Itoa(value.(int)) == dt.MCU_R_ActSafeSt_AS0 || strconv.Itoa(value.(int)) == dt.MCU_R_ActSafeSt_ASC || strconv.Itoa(value.(int)) == dt.MCU_R_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_Decoup_State:
+		found = true
+		state.GetSafeState().MCURearDecoupState = strconv.Itoa(value.(int)) == dt.MCU_R_Decoup_State_Connected
+	case dt.MCU_F_CrtMod:
+		found = true
+		state.GetSafeState().MCUFrontInverterError = strconv.Itoa(value.(int)) == dt.MCU_F_CrtMod_Internal_inverter_error || strconv.Itoa(value.(int)) == dt.MCU_F_CrtMod_Invalid
+	case dt.MCU_R_CrtMod:
+		found = true
+		state.GetSafeState().MCURearInverterError = strconv.Itoa(value.(int)) == dt.MCU_R_CrtMod_Internal_inverter_error || strconv.Itoa(value.(int)) == dt.MCU_R_CrtMod_Invalid
+	case dt.ACU_Drvr_Occpt_St:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.DriverOccupySeatState = ref(vi)
+	case dt.BCM_PwrMod:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.PowerMode = ref(vi)
+	case dt.PWC_ChrgSts:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.ChargingStatus = ref(vi)
+	case dt.VCU_RdyLamp:
+		found = true
+		state.GetVehicleReadyState().IsVehicleReady, ok = value.(bool)
+	case "online":
+		found = true
+		state.Online, ok = value.(bool)
+	case "online_hmi":
+		found = true
+		state.OnlineHMI, ok = value.(bool)
+		// New untested signals
+	// case dt.IBS_SOCUpperTolerance:
+	// 	found = true
+	// 	var vi float64
+	// 	vi, ok = value.(float64)
+	// 	state.GetExpandedSignals().IBS_SOCUpperTolerance = ref(vi)
+	// case dt.IBS_SOCLowerTolerance:
+	// 	found = true
+	// 	var vi float64
+	// 	vi, ok = value.(float64)
+	// 	state.GetExpandedSignals().IBS_SOCLowerTolerance = ref(vi)
+	case dt.IBS_StateOfCharge:
+		found = true
+		var vi float64
+		vi, ok = value.(float64)
+		state.GetBattery12V().IBS_StateOfCharge = ref(vi)
+	case dt.IBS_StateOfHealth:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetBattery12V().IBS_StateOfHealth = ref(vi)
+	case dt.IBS_NominalCapacity:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetExpandedSignals().IBS_NominalCapacity = ref(vi)
+	case dt.IBS_AvailableCapacity:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetExpandedSignals().IBS_AvailableCapacity = ref(vi)
+	case dt.BCM_TotMilg_ODO:
+		found = true
+		var vi float64
+		vi, ok = value.(float64)
+		state.GetExpandedSignals().BCM_TotMilg_ODO = ref(vi)
+	case dt.BMS_SwVersS:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetExpandedSignals().BMS_SwVersS = ref(vi)
+	case dt.BMS_SwVersM:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetExpandedSignals().BMS_SwVersM = ref(vi)
+	case dt.BMS_SwVers:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetExpandedSignals().BMS_SwVers = ref(vi)
+	case dt.BMS_AccueDchaTotAh:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetExpandedSignals().BMS_AccueDchaTotAh = ref(vi)
+	case dt.BMS_AccueChrgTotAh:
+		found = true
+		var vi int
+		vi, ok = value.(int)
+		state.GetExpandedSignals().BMS_AccueChrgTotAh = ref(vi)
+	case dt.TBOX_Heading:
+		found = true
+		state.GetLocation().Heading, ok = value.(float64)
+	case dt.PKC_KeyStsMod:
+		found = true
+		state.GetGear().Immobilizer = value.(string)
+	}
+	if found {
+		if !ok {
+			err = fmt.Errorf("failed on key %s value %v", key, value)
+		}
+	} else {
+		logger.Info().Str("key", key).Interface("value", value).Msgf("did not have parsing mode for key")
+	}
+
+	return errors.WithStack(err)
+}
+
+func ref[T any](v T) *T {
+	return &v
+}
+
+func IsCarOnline(clientPool redis.ClientPoolInterface, vin string) (bool, error) {
+	client := clientPool.GetFromPool()
+	defer client.Close()
+	return redigo.Bool(
+		client.Execute("SISMEMBER", redis.CarSessionsKey(), vin),
+	)
+}
+
+func notValue(value bool, err error) (bool, error) {
+	return !value, err
+}

pkg/cache/vehicle_state_multi.go

@@ -0,0 +1,99 @@
+package cache
+
+import (
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+
+func GetVINListDigitalTwin(vins []string, clientPool redis.ClientPoolInterface) (digitalTwins map[string]common.CarState, errorList []error) {
+	digitalTwins = make(map[string]common.CarState)
+	client := clientPool.GetFromPool()
+	defer client.Close()
+
+	batch := redis.NewRedisBatchCommands()
+	for _, vin := range vins {
+		batch.Add("SISMEMBER", redis.CarSessionsKey(), vin)
+		batch.Add("SISMEMBER", redis.HMISessionsKey(), vin)
+		batch.Add("HGETALL", redis.CarStateHashKey(vin))
+
+	}
+
+	payload, err := redigo.Values(client.ExecuteBatch(batch))
+	if err != nil {
+		errorList = append(errorList, err)
+		return
+	}
+	
+	for index, vin := range vins {
+		startPoint := index * 3
+		tempTwin, err := ParsePayloadForVehicleState(payload[startPoint:startPoint+3])
+		if err != nil {
+			err = errors.WithMessage(err, vin)
+			errorList = append(errorList, err)
+			continue
+		}
+		digitalTwins[vin] = tempTwin
+	}
+
+	return
+}
+
+func ParsePayloadForVehicleState(payload []interface{}) (common.CarState, error) {
+	var state common.CarState
+
+	online, err := redigo.Bool(payload[0], nil)
+	if err != nil {
+		return state, err
+	} else {
+		state.Online = online
+	}
+
+	online, err = redigo.Bool(payload[1], nil)
+	if err != nil {
+		return state, errors.WithStack(err)
+	} else {
+		state.OnlineHMI = online
+	}
+
+	err = parseCarStatePayload(&state, payload[2])
+
+	return state, err
+}
+
+func parseCarStatePayload(state *common.CarState, payload interface{}) error {
+	stateValues, err := redigo.Values(payload, nil)
+	if err != nil {
+		return err
+	}
+	if len(stateValues)%2 != 0 {
+		return errors.New("object does not contain equal number of key value pairs")
+	}
+
+	err = parseStateValues(state, stateValues, parseCarState)
+
+	return err
+}
+
+func parseStateValues(state *common.CarState, stateValues []interface{}, parser stateParser) error {
+	for i := 0; i < len(stateValues); i += 2 {
+		key, okKey := stateValues[i].([]byte)
+		value, okValue := stateValues[i+1].([]byte)
+
+		if !okKey || !okValue {
+			return errors.New("cannot parse object into car state")
+		}
+
+		err := parser(state, string(key), value)
+		// log error, do not return error so we can read other properties for digital twin
+		if err != nil {
+			logger.Err(err).Send()
+		}
+	}
+
+	return nil
+}

pkg/cache/vehicle_state_test.go

@@ -0,0 +1,193 @@
+package cache_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/redis/tester"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConnGetVehicleState(t *testing.T) {
+	var updateTime = time.Date(2020, time.October, 3, 12, 10, 0, 0, time.UTC)
+	vin := "TESTVIN123"
+	redisMock := tester.NewRedisMock()
+	redisPool := tester.NewMockClientPool(redisMock)
+
+	testCases := map[string]struct {
+		sismemberResults map[string]map[string]interface{}
+		hgetallResults   map[string][]interface{}
+		expResp          common.CarState
+		expErr           error
+	}{
+		"correct": {
+			sismemberResults: map[string]map[string]interface{}{
+				redis.CarSessionsKey(): {
+					vin: int64(1),
+				},
+				redis.HMISessionsKey(): {
+					vin: int64(1),
+				},
+			},
+			hgetallResults: map[string][]interface{}{
+				fmt.Sprintf("car:%s:state", vin): {
+					[]byte("DSMC_DrvrSeatHeatgSts"), []byte("2"),
+					[]byte("ESP_VehSpd"), []byte("123.4"),
+					[]byte("BMS_RmChrgTi_TrgtSoC"), []byte("5000"),
+					[]byte("BMS_RmChrgTi_FullChrg"), []byte("6000"),
+					[]byte("VCU_VehChrgDchgMod"), []byte("DC_charging"),
+					[]byte("BCM_AP_FL_LeReWinPosnInfo"), []byte("30"),
+					[]byte("BCM_ReDefrstHeatgCmd"), []byte("1"),
+					[]byte("BCM_FrntHoodLidSts"), []byte("1"),
+					[]byte("BMS_Bat_SOH"), []byte("20"),
+					[]byte("ICC_TotMilg_ODO"), []byte("2345"),
+					[]byte("IBS_BatteryVoltage"), []byte("12.3"),
+					[]byte("TBOX_GPSHei"), []byte("16"),
+					[]byte("ECC_OutdT"), []byte("30"),
+					[]byte("PSM_PassSeatHeatgSts"), []byte("4"),
+					[]byte("TBOX_GPSLati"), []byte("35.831"),
+					[]byte("BCM_PasFrntDoorSts"), []byte("0"),
+					[]byte("BCM_CenLockSwtSts"), []byte("3"),
+					[]byte("BCM_RiReDoorSts"), []byte("1"),
+					[]byte("BCM_LeReDoorSts"), []byte("1"),
+					[]byte("VCU_DrvgMilg"), []byte("1234"),
+					[]byte("TBOX_GPSLongi"), []byte("-120.398"),
+					[]byte("BCM_AP_FL_RiReWinPosnInfo"), []byte("40"),
+					[]byte("BCM_FrntDrDoorLockSts"), []byte("1"),
+					[]byte("BCM_DrFrntDoorSts"), []byte("0"),
+					[]byte("BCM_AP_TL_LeReWinPosnInfo"), []byte("60"),
+					[]byte("ECC_RemTSetSts"), []byte("120"),
+					[]byte("BCM_AP_FL_RiFrntWinPosnInfo"), []byte("20"),
+					[]byte("BMS_PwrBattRmngCpSOC"), []byte("50"),
+					[]byte("BCM_AP_TL_RiReWinPosnInfo"), []byte("70"),
+					[]byte("BCM_HeatedSteerWhlSt"), []byte("1"),
+					[]byte("BCM_AP_RW_WinPosnInfo"), []byte("80"),
+					[]byte("ECC_InsdT"), []byte("30"),
+					[]byte("updated"), []byte(`"2020-10-03T12:10:00Z"`),
+					[]byte("BMS_Bat_SoC_usable"), []byte("10"),
+					[]byte("BCM_AP_FL_LeFrntWinPosnInfo"), []byte("10"),
+					[]byte("BCM_SunroofPosnInfo"), []byte("50"),
+					[]byte("BMS_BattAvrgT"), []byte("90"),
+					[]byte("dbc_version"), []byte("hash"),
+					[]byte("VCU_VehSt"), []byte("12"),
+					[]byte("VCU_VcuState"), []byte("18"),
+					[]byte("MCU_F_ActSafeSt"), []byte("4"),
+					[]byte("MCU_R_ActSafeSt"), []byte("2"),
+					[]byte("MCU_R_Decoup_State"), []byte("3"),
+					[]byte("MCU_F_CrtMod"), []byte("7"),
+					[]byte("MCU_R_CrtMod"), []byte("8"),
+					[]byte("VCU_RdyLamp"), []byte("1"),
+				},
+			},
+			expResp: common.CarState{
+				Online:    true,
+				OnlineHMI: true,
+				VehicleSpeed: &common.VehicleSpeed{
+					Speed: 123.4,
+				},
+				Battery: &common.Battery{
+					Percent:              50,
+					TotalMileageOdometer: 2345,
+					BatteryVoltage:       12.3,
+				},
+				MaxRange: &common.MaxRange{
+					MaxMiles: 1234,
+				},
+				Doors: &common.Doors{
+					Hood:       true,
+					LeftFront:  false,
+					LeftRear:   true,
+					RightFront: false,
+					RightRear:  true,
+				},
+				Location: &common.Location{
+					Altitude:  16,
+					Longitude: -120.398,
+					Latitude:  35.831,
+				},
+				Locks: &common.Locks{
+					Driver: false,
+					All:    false,
+				},
+				Windows: &common.Windows{
+					LeftFront:  10,
+					LeftRear:   30,
+					RightFront: 20,
+					RightRear:  40,
+				},
+				MiscWindows: &common.MiscWindows{
+					LeftRearQuarter:  60,
+					RightRearQuarter: 70,
+					RearWindshield:   80,
+				},
+				Sunroof: &common.Sunroof{
+					Sunroof: 50,
+				},
+				CabinClimate: &common.CabinClimate{
+					CabinTemperature:    120,
+					InternalTemperature: 30,
+				},
+				RearDefrost: &common.RearDefrost{
+					On: true,
+				},
+				DriverSeatHeat: &common.DriverSeatHeat{
+					Level: 2,
+				},
+				PassengerSeatHeat: &common.PassengerSeatHeat{
+					Level: 4,
+				},
+				CellTemperature: &common.CellTemperature{
+					AvgBatteryTemp: 90,
+				},
+				ChargingMetrics: &common.VCUChargingMetrics{
+					RemainingChargingTime:     5000,
+					RemainingChargingTimeFull: 6000,
+				},
+				SteeringWheelHeat: &common.SteeringWheelHeat{
+					On: true,
+				},
+				AmbientTemperature: &common.AmbientTemperature{
+					Temperature: 30,
+				},
+				VCU0x260: &common.VCU0x260Descriptor{
+					ChargeType: "DC_charging",
+				},
+				StateOfCharge: &common.StateOfCharge{
+					Usable: 10,
+					Health: 20,
+				},
+				DBCVersion: "hash",
+				UpdatedAt:  &updateTime,
+				SafeState: &common.SafeState{
+					VehicleSafeState:      false,
+					VCUSafeState:          true,
+					MCUFrontSafeState:     false,
+					MCURearSafeState:      true,
+					MCURearDecoupState:    false,
+					MCUFrontInverterError: true,
+					MCURearInverterError:  false,
+				},
+				VehicleReadyState: &common.VehicleReadyState{
+					IsVehicleReady: true,
+				},
+			},
+			expErr: nil,
+		},
+	}
+
+	parser := cache.NewVehicleState(redisPool)
+
+	for tName, tt := range testCases {
+		t.Run(tName, func(t *testing.T) {
+			redisMock.SISMEMBEResults = tt.sismemberResults
+			redisMock.HGETALLResults = tt.hgetallResults
+			state, err := parser.Get(vin)
+			assert.Equal(t, tt.expErr, err)
+			assert.Equal(t, tt.expResp, state)
+		})
+	}
+}

pkg/cache/vehicles.go

@@ -0,0 +1,124 @@
+package cache
+
+import (
+	"encoding/json"
+	"fiskerinc.com/modules/common"
+	orm "fiskerinc.com/modules/db/queries"
+	"fmt"
+	"github.com/ReneKroon/ttlcache/v2"
+	"github.com/pkg/errors"
+	"sync"
+	"time"
+)
+
+var (
+	ErrCacheNotInitialized = errors.New("cache is not initialized")
+)
+
+type VehicleCacher interface {
+	Set(key VehiclesTTLParams, value *VehiclesTTLResult) error
+	Get(key VehiclesTTLParams) (*VehiclesTTLResult, error)
+}
+
+type VehiclesCache struct {
+	duration  time.Duration
+	limit     int
+	cache     *ttlcache.Cache
+	onceCache *sync.Once
+}
+
+func (c *VehiclesCache) Duration() time.Duration {
+	return c.duration
+}
+
+func (c *VehiclesCache) Limit() int {
+	return c.limit
+}
+
+func (c *VehiclesCache) Once() *sync.Once {
+	return c.onceCache
+}
+
+func (c *VehiclesCache) SetCache(cache *ttlcache.Cache) {
+	c.cache = cache
+}
+
+func (c *VehiclesCache) Cache() *ttlcache.Cache {
+	return c.cache
+}
+
+type VehiclesTTLParams struct {
+	Options         orm.PageQueryOptions `json:"options"`
+	CarOnlineFilter *common.CarOnlineFilter   `json:"car_online_filter"`
+	Search          string               `json:"search"`
+}
+
+type VehiclesTTLResult struct {
+	Data  []common.Car `json:"data"`
+	Total int     `json:"total"`
+}
+
+func NewVehiclesCache(duration time.Duration, limit int) (*VehiclesCache, error) {
+	c := &VehiclesCache{
+		duration:  duration,
+		limit:     limit,
+		onceCache: &sync.Once{},
+	}
+
+	if cache := logCache(c); cache == nil {
+		return nil, ErrCacheNotInitialized
+	}
+
+	return c, nil
+}
+func (c *VehiclesCache) Set(key VehiclesTTLParams, value *VehiclesTTLResult) error {
+	if cache := logCache(c); cache == nil {
+		return ErrCacheNotInitialized
+	}
+
+	keyBts, err := json.Marshal(key)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to marshal key")
+	}
+
+	return c.cache.Set(string(keyBts), value)
+}
+
+func (c *VehiclesCache) Get(key VehiclesTTLParams) (*VehiclesTTLResult, error) {
+	if cache := logCache(c); cache == nil {
+		return nil, ErrCacheNotInitialized
+	}
+
+	keyBts, err := json.Marshal(key)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to marshal key")
+	}
+
+	value, err := c.cache.Get(string(keyBts))
+	if err != nil {
+		return nil, fmt.Errorf("failed to get value from cache: %w", err)
+	}
+
+	return value.(*VehiclesTTLResult), nil
+}
+
+type Cacher interface {
+	Duration() time.Duration
+	Limit() int
+	Once() *sync.Once
+	Cache() *ttlcache.Cache
+	SetCache(*ttlcache.Cache)
+}
+
+func logCache(cacher Cacher) *ttlcache.Cache {
+	cacher.Once().Do(func() {
+		if cacher.Cache() == nil {
+			cache := ttlcache.NewCache()
+			cache.SetTTL(cacher.Duration())
+			cache.SetCacheSizeLimit(cacher.Limit())
+			cacher.SetCache(cache)
+		}
+	})
+
+	return cacher.Cache()
+}

pkg/cache/verify.go

@@ -0,0 +1,58 @@
+package cache
+
+import (
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+// VerifyCarToDriver checks cache and DB for car to driver relationship.
+// If relationship exists and not in cache, will cache value.
+//
+//	car:<VIN>:driver:<DRIVER_ID>
+func VerifyCarToDriver(clientPool redis.ClientPoolInterface, db queries.CarsInterface, vin string, driverID string) (bool, error) {
+	key := redis.CarToDriverKey(vin, driverID)
+
+	ok, err := redisCheckGet(clientPool, key)
+	if err != nil {
+		return ok, err
+	}
+
+	if ok {
+		return ok, err
+	}
+
+	carToDrivers, err := db.SelectCarToDriver(&common.CarToDriver{VIN: vin, DriverID: driverID})
+	if err != nil {
+		return false, err
+	}
+	verified := len(carToDrivers) == 1
+	redisPlaceDriverCache(clientPool, key, verified)
+
+	return verified, err
+}
+
+func redisCheckGet(clientPool redis.ClientPoolInterface, key string) (bool, error) {
+	client := clientPool.GetFromPool()
+	defer client.Close()
+	ok, err := redigo.Bool(client.Execute("GET", key))
+	if err != nil && !errors.Is(err, redigo.ErrNil) {
+		logger.Warn().Err(err).Send()
+		return ok, err
+	}
+	return ok, nil
+}
+
+func redisPlaceDriverCache(clientPool redis.ClientPoolInterface, key string, verified bool) (err error) {
+	client := clientPool.GetFromPool()
+	defer client.Close()
+	batch := redis.NewRedisBatchCommands()
+	batch.Add("SET", key, verified)
+	batch.Add("EXPIRE", key, redisObjectExpire)
+	_, err = client.ExecuteBatch(batch)
+	return
+}

pkg/cache/verify_test.go

@@ -0,0 +1,55 @@
+package cache_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries/mocks"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/redis/tester"
+	"fiskerinc.com/modules/testhelper"
+
+	redigo "github.com/gomodule/redigo/redis"
+)
+
+type mockRedisCacheDriverToCars struct {
+	redis.Connection
+}
+
+func (c *mockRedisCacheDriverToCars) Execute(command ...interface{}) (interface{}, error) {
+	return []byte("1"), nil
+}
+
+type mockRedisEmptyCacheDriverToCars struct {
+	redis.Connection
+}
+
+func (c *mockRedisEmptyCacheDriverToCars) Execute(command ...interface{}) (interface{}, error) {
+	return nil, redigo.ErrNil
+}
+
+func (c *mockRedisEmptyCacheDriverToCars) ExecuteBatch(batch *redis.RedisBatchCommands) (interface{}, error) {
+	return nil, nil
+}
+
+func TestVerifyCarToDriver(t *testing.T) {
+	setupRedisMock()
+	mockDB := &mocks.MockCars{
+		SelectCarsForDrivers: []common.CarToDriver{{}},
+	}
+
+	mockRedis = &mockRedisCacheDriverToCars{}
+	redisPool := tester.NewMockClientPool(mockRedis)
+	_, err := cache.VerifyCarToDriver(redisPool, mockDB, "VALID_VIN", "VALID_ID")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", nil, err)
+	}
+
+	mockRedis = &mockRedisEmptyCacheDriverToCars{}
+	redisPool = tester.NewMockClientPool(mockRedis)
+	_, err = cache.VerifyCarToDriver(redisPool, mockDB, "VALID_VIN", "VALID_ID")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", nil, err)
+	}
+}

pkg/cache/vins.go

@@ -0,0 +1,60 @@
+package cache
+
+import (
+	"errors"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+)
+
+// RetrieveVINs retrieves VINs from redis or from DB based on driver ID and proceeds to cache VINs
+// redis keys:
+//
+//	driver:<ID>:cars
+func RetrieveVINs(client redis.Client, db queries.CarsInterface, id string) ([]string, error) {
+	var vins []string
+	driverVINsKey := redis.DriverToVINsKey(id)
+
+	// retrieve VINs from redis
+	err := client.GetCache(driverVINsKey, &vins, 0)
+	if err != nil && !errors.Is(err, redis.ErrNilObject) {
+		logger.Warn().Err(err).Send()
+	} else if len(vins) > 0 {
+		return vins, nil
+	}
+
+	// if VINs not present in redis perform DB lookup
+	var vehicles []common.CarToDriver
+	vehicles, err = db.GetCarsForDriver(id)
+	if err != nil {
+		return vins, err
+	}
+
+	for _, vehicle := range vehicles {
+		vins = append(vins, vehicle.VIN)
+	}
+
+	// cache drivers vehicles
+	err = client.SetCache(driverVINsKey, vins, redisObjectExpire)
+	if err != nil {
+		return vins, err
+	}
+
+	return vins, nil
+}
+
+func RetrieveVINsAsSet(client redis.Client, db queries.CarsInterface, id string) (map[string]struct{}, error) {
+	vins, err := RetrieveVINs(client, db, id)
+	if err != nil {
+		return nil, err
+	}
+
+	var vinsSet = make(map[string]struct{})
+	for _, vin := range vins {
+		vinsSet[vin] = struct{}{}
+	}
+
+	return vinsSet, nil
+}

pkg/cache/vins_test.go

@@ -0,0 +1,71 @@
+package cache_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/testhelper"
+)
+
+type mockRedisCacheVINs struct {
+	redis.Connection
+}
+
+func (c *mockRedisCacheVINs) GetCache(id string, data interface{}, expire int) error {
+	vins := []string{"TESTVIN123", "TESTVIN456"}
+
+	dataBytes, err := json.Marshal(vins)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(dataBytes, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type mockRedisEmptyCacheVINs struct {
+	redis.Connection
+}
+
+func (c *mockRedisEmptyCacheVINs) GetCache(id string, data interface{}, expire int) error {
+	vins := []string{}
+
+	dataBytes, err := json.Marshal(vins)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(dataBytes, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *mockRedisEmptyCacheVINs) SetCache(id string, data interface{}, expire int) error {
+	return nil
+}
+
+func TestRetrieveAndCacheVINs(t *testing.T) {
+	setupRedisMock()
+	setupDBMock()
+
+	mockRedis = &mockRedisCacheVINs{}
+	_, err := cache.RetrieveVINs(mockRedis, mockDB, "VALID_ID")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", nil, err)
+	}
+
+	mockRedis = &mockRedisEmptyCacheVINs{}
+	_, err = cache.RetrieveVINs(mockRedis, mockDB, "VALID_ID")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", nil, err)
+	}
+}

pkg/cachev2/constants.go

@@ -0,0 +1,6 @@
+package cachev2
+
+import "time"
+
+const redisObjectExpire = time.Hour
+const redisObjectExpireDay = 24 * time.Hour

pkg/cachev2/digital_twin.go

@@ -0,0 +1,192 @@
+package cachev2
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"fiskerinc.com/modules/dbc/state"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/utils/querystring"
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+const (
+	pattern = "car:*:state"
+)
+
+type DigitalTwinTimestampState struct {
+	redisClient redis.Client
+}
+
+func NewDigitalTwinTimestampState(redisClient redis.Client) *DigitalTwinTimestampState {
+	return &DigitalTwinTimestampState{
+		redisClient: redisClient,
+	}
+}
+
+// getStateKeys retrieves car state keys from Redis based on the specified pattern
+// and returns a sliced list of keys according to the provided offset and limit.
+//
+// Parameters:
+//   - offset: An integer indicating the starting index of the slice.
+//   - limit: An integer specifying the maximum number of elements in the sliced list.
+//
+// Returns:
+//   - []string: A sliced list of car state keys based on the given offset and limit.
+//   - error: An error, if any, encountered during the Redis operation or slicing process.
+func (dtts *DigitalTwinTimestampState) getStateKeys(offset, limit int) ([]string, error) {
+	keys, err := redigo.Strings(dtts.redisClient.Execute("KEYS", pattern))
+	if err != nil {
+		return nil, err
+	}
+	totalKeys := len(keys)
+	if totalKeys <= offset {
+		return nil, nil
+	}
+
+	if (offset + limit) > totalKeys {
+		limit = totalKeys - offset
+	}
+
+	sort.Strings(keys)
+
+	keys = keys[offset : offset+limit]
+	return keys, nil
+}
+
+// readCarStateByKey retrieves data from Redis based on the specified key using the HGETALL command.
+// It iterates over all keys and values returned by the command, sets them in a response map,
+// and returns the populated map along with any encountered errors.
+//
+// Parameters:
+//   - key: A string representing the key to retrieve data from in Redis.
+//
+// Returns:
+//   - map[string]interface{}: A map containing keys and values retrieved from Redis.
+//   - error: An error, if any, encountered during the Redis HGETALL operation or mapping process.
+func (dtts *DigitalTwinTimestampState) readCarStateByKey(key string) (map[string]interface{}, error) {
+
+	keyval := make(map[string]interface{})
+	batch := redis.NewRedisBatchCommands()
+
+	batch.Add("HGETALL", key)
+
+	payload, err := redigo.Values(dtts.redisClient.ExecuteBatch(batch))
+	if err != nil {
+		return keyval, err
+	}
+
+	stateValues, err := redigo.Values(payload[0], nil)
+	if err != nil {
+		return keyval, err
+	}
+
+	for i := 0; i < len(stateValues); i += 2 {
+		key, okKey := stateValues[i].([]byte)
+		value, okValue := stateValues[i+1].([]byte)
+
+		if !okKey || !okValue {
+			continue
+		}
+
+		err = dtts.parseCarState(string(key), value, keyval)
+		// log error, do not return error so we can read other properties for digital twin
+		if err != nil {
+			logger.Warn().Err(err).Send()
+			continue
+		}
+	}
+	return keyval, nil
+}
+
+// GetDigitalTwinSignals retrieves digital twin signals from Redis based on the specified offset and limit.
+// It reads all signals from Redis and returns a list of maps, where each map represents a cars signal with its properties.
+//
+// Parameters:
+//   - offset: An integer indicating the starting index of the signals to retrieve.
+//   - limit: An integer specifying the maximum number of signals to retrieve.
+//
+// Returns:
+//   - []map[string]interface{}: A list of maps representing digital twin signals.
+
+func (dtts *DigitalTwinTimestampState) GetDigitalTwinSignals(offset, limit int) (resp []map[string]interface{}) {
+
+	keys, err := dtts.getStateKeys(offset, limit)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+		return
+	}
+	for _, key := range keys {
+		keyval, err := dtts.readCarStateByKey(key)
+		if err != nil {
+			logger.Warn().Err(err).Send()
+			continue
+		}
+		if len(keyval) > 0 {
+			keySlice := strings.Split(key, ":")
+			keyval["VIN"] = keySlice[1]
+			resp = append(resp, keyval)
+		}
+	}
+	return
+}
+
+// timestampKey generates a timestamp key based on the provided key by appending ":updated" to it.
+// It formats the key in a way suitable for storing timestamps associated with the original key in data storage systems.
+//
+// Parameters:
+//   - key: A string representing the original key for which the timestamp key is generated.
+//
+// Returns:
+//   - string: A formatted string representing the timestamp key.
+func (dtts *DigitalTwinTimestampState) timestampKey(key string) string {
+	return fmt.Sprintf("%s:%s", key, "updated")
+}
+
+// timestampVal parses a byte slice containing a JSON-encoded timestamp and returns a pointer to a time.Time
+// representing the parsed timestamp. It uses the UnmarshalJSON method of the time.Time type for decoding.
+//
+// Parameters:
+//   - val: A byte slice containing the JSON-encoded timestamp to be parsed.
+//
+// Returns:
+//   - time.Time: A time representing the parsed timestamp.
+//   - error: An error, if any, encountered during the parsing process.
+func (dtts *DigitalTwinTimestampState) timestampVal(val []byte) (time.Time, error) {
+	t := &time.Time{}
+	err := t.UnmarshalJSON(val)
+	return *t, err
+}
+
+// parseCarState checks if the provided key is needed and, if so, sets the key and value in the given map.
+//
+// Parameters:
+//   - key: A string representing the key to check and potentially set in the map.
+//   - value: A byte slice containing the value associated with the key.
+//   - keyval: A map[string]interface{} where the key and valueset if the key is needed.
+//
+// Returns:
+//   - error: An error, if any, encountered during the parsing and mapping process.
+func (dtts *DigitalTwinTimestampState) parseCarState(key string, value []byte, keyval map[string]interface{}) error {
+	var err error
+	val := string(value)
+	switch key {
+	case state.BMS_PwrBattRmngCpSOC, state.BMS_RmChrgTi_FullChrg, state.BCM_PwrMod, state.PWC_ChrgSts, state.VCU_DCChrgRmngTi, state.BMS_RmChrgTi_TrgtSoC:
+		keyval[key], err = strconv.Atoi(val)
+	case state.ICC_TotMilg_ODO:
+		keyval[key], err = querystring.ConvertStringToInt(val)
+	case state.IBS_BatteryVoltage:
+		keyval[key], err = strconv.ParseFloat(val, 64)
+
+	// updated timestamps
+	case dtts.timestampKey(state.BMS_PwrBattRmngCpSOC), dtts.timestampKey(state.ICC_TotMilg_ODO), dtts.timestampKey(state.VCU_DCChrgRmngTi), dtts.timestampKey(state.BMS_RmChrgTi_TrgtSoC), dtts.timestampKey(state.IBS_BatteryVoltage),
+		dtts.timestampKey(state.BMS_RmChrgTi_FullChrg), dtts.timestampKey(state.BCM_PwrMod), dtts.timestampKey(state.PWC_ChrgSts):
+		keyval[key], err = dtts.timestampVal(value)
+	}
+	return errors.WithStack(err)
+}

pkg/cachev2/drivers.go

@@ -0,0 +1,136 @@
+package cachev2
+
+import (
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/logger"
+	redis "fiskerinc.com/modules/redisv2"
+	"github.com/pkg/errors"
+)
+
+func NewDriversCache(redisClient redis.ClientInterface, cars queries.CarsInterface) *DriversCache {
+	return &DriversCache{
+		redisClient: redisClient,
+		cars:        cars,
+	}
+}
+
+type DriversCache struct {
+	redisClient redis.ClientInterface
+	cars        queries.CarsInterface
+}
+
+func (dc *DriversCache) RedisClientPool() redis.ClientInterface {
+	return dc.redisClient
+}
+
+func (dc *DriversCache) Cars() queries.CarsInterface {
+	return dc.cars
+}
+
+func (dc *DriversCache) hasCachedNoDrivers(drivers []string) bool {
+	// Redis will return []string{""} for no drivers
+	return len(drivers) == 1 && len(drivers[0]) == 0
+}
+
+func (dc *DriversCache) cacheDrivers(key string, drivers []string) error {
+	// cache driver IDs
+	if len(drivers) > 0 {
+		return dc.redisClient.NewSet(key, drivers, redisObjectExpire)
+	}
+
+	// Redis will not take an empty array as an arg
+	return nil
+}
+
+// RetrieveDriverIDs retrieves IDs from redis or from DB and proceeds to cache both the drivers and IDs
+// redis keys:
+//
+//	car:<VIN>:drivers
+func (dc *DriversCache) RetrieveDriverIDs(vin string) ([]string, error) {
+	var driverIDs []string
+	driverIDsKey := redis.CarToAllDriversKey(vin)
+
+	// retrieve IDs from redis
+	err := dc.redisClient.GetSet(driverIDsKey, &driverIDs)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+		return []string{}, err
+	}
+
+	if dc.hasCachedNoDrivers(driverIDs) {
+		return []string{}, nil
+	}
+	if len(driverIDs) > 0 {
+		return driverIDs, nil
+	}
+
+	// if IDs not present in redis perform DB lookup
+	var drivers []common.CarToDriver
+	drivers, err = dc.cars.GetDrivers(vin)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, driver := range drivers {
+		driverIDs = append(driverIDs, driver.DriverID)
+	}
+
+	err = dc.cacheDrivers(driverIDsKey, driverIDs)
+	if err != nil {
+		return driverIDs, err
+	}
+
+	return driverIDs, nil
+}
+
+// RetrieveDriverIDsAsSet retrieves IDs from redis or from DB and proceeds to cache both the drivers and IDs
+// redis keys:
+//
+//	car:<VIN>:drivers
+func (dc *DriversCache) RetrieveDriverIDsAsSet(vin string) (map[string]struct{}, error) {
+	driverIDs, err := dc.RetrieveDriverIDs(vin)
+	if err != nil {
+		return nil, err
+	}
+
+	var dIDsSet = make(map[string]struct{})
+	for _, did := range driverIDs {
+		dIDsSet[did] = struct{}{}
+	}
+
+	return dIDsSet, nil
+}
+
+func (dc *DriversCache) IsDriverOfVIN(vin string, driverid string) (bool, error) {
+	ids, err := dc.RetrieveDriverIDs(vin)
+	if err != nil {
+		return false, err
+	}
+
+	for _, id := range ids {
+		if id == driverid {
+			return true, nil
+		}
+	}
+
+	return false, dc.NotDriverError(vin, driverid)
+}
+
+// Add driver to database and cache
+func (dc *DriversCache) AddDriver(car *common.Car, driver *common.Driver, role string) (*common.CarToDriver, error) {
+
+	relation, err := dc.cars.AddDriver(car, driver, role)
+	if err != nil {
+		return nil, err
+	}
+
+	driverIDsKey := redis.CarToAllDriversKey(car.VIN)
+
+	dc.redisClient.AddToSet(driverIDsKey, driver.ID, redisObjectExpire)
+	return relation, nil
+}
+
+func (dc DriversCache) NotDriverError(vin string, driverid string) error {
+	return errors.Errorf("id %s is not a driver for vin %v", driverid, vin)
+}

pkg/cachev2/drivers_test.go

@@ -0,0 +1,107 @@
+package cachev2_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	cache "fiskerinc.com/modules/cachev2"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/db/queries/mocks"
+	"fiskerinc.com/modules/redis/tester"
+	redis "fiskerinc.com/modules/redisv2"
+	"fiskerinc.com/modules/testhelper"
+)
+
+var mockRedis redis.Client
+var mockDB queries.CarsInterface
+
+func setupRedisMock() {
+	redis.MockRedisConnection()
+}
+
+func setupDBMock() {
+	mockDB = &mocks.MockCars{}
+}
+
+type mockRedisCache struct {
+	redis.Connection
+}
+
+func (c *mockRedisCache) GetSet(id string, data interface{}) error {
+	drivers := []string{"valid-id-1", "valid-id-2", "valid-id-3"}
+
+	dataBytes, err := json.Marshal(drivers)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(dataBytes, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type mockRedisEmptyCache struct {
+	redis.Connection
+}
+
+func (c *mockRedisEmptyCache) GetSet(id string, data interface{}) error {
+	drivers := []string{}
+
+	dataBytes, err := json.Marshal(drivers)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(dataBytes, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *mockRedisEmptyCache) SetObjects(id []string, data []interface{}, expire int) error {
+	return nil
+}
+
+func TestRetrieveAndCacheDriverIDs(t *testing.T) {
+	setupRedisMock()
+	setupDBMock()
+	mockRedis = &mockRedisCache{}
+	redisPool := tester.NewMockClientPool(mockRedis)
+	drivers := cache.NewDriversCache(redisPool, mockDB)
+
+	_, err := drivers.RetrieveDriverIDs("FISKER123")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+
+	mockRedis = &mockRedisEmptyCache{}
+	_, err = drivers.RetrieveDriverIDs("FISKER456")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+}
+
+func TestRetrieveAndCacheDriverIDsAsSet(t *testing.T) {
+	setupRedisMock()
+	setupDBMock()
+	mockRedis = &mockRedisCache{}
+	redisPool := tester.NewMockClientPool(mockRedis)
+
+	drivers := cache.NewDriversCache(redisPool, mockDB)
+
+	_, err := drivers.RetrieveDriverIDs("FISKER123")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+
+	mockRedis = &mockRedisEmptyCache{}
+	_, err = drivers.RetrieveDriverIDsAsSet("FISKER456")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", "no error", err)
+	}
+}

pkg/cachev2/errors.go

@@ -0,0 +1,11 @@
+package cachev2
+
+import "github.com/pkg/errors"
+
+func ErrInvalidCarToDriverAssociation(vin string, driverID string) error {
+	return errors.Errorf("no relationship found between vin %s and driver %s", vin, driverID)
+}
+
+func ErrCarHasNoDrivers(vin string) error {
+	return errors.Errorf("car %s has no drivers", vin)
+}

pkg/cachev2/vehicle_config.go

@@ -0,0 +1,197 @@
+package cachev2
+
+import (
+	"encoding/json"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/mongo"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/utils/envtool"
+
+	"fiskerinc.com/modules/utils/elptr"
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+const (
+	ENABLE_DBG_MASK_EV_NAME     = "ENABLE_DEBUGMASK"
+	ENABLE_DBG_MASK_VAL_FALSE   = "0"
+	ENABLE_DBG_MASK_VAL_TRUE    = "1"
+	ENABLE_DBG_MASK_VAL_DEFAULT = ENABLE_DBG_MASK_VAL_FALSE
+)
+
+// This flag is to decide whether retrieved value of DebugMask is to be passed to TrexCfg or not.
+// When the flag is true, the retrieved value is passed; else no value is passed.
+// The value of flag is fetched from the specific environmental variable. If that environmental
+// variable is not present / not defined, we assume the flag itself to be FALSE. That is the
+// default value (FALSE) of the environmental variable. When user/developer has set this evironmental
+// variable correctly, the flag can become TRUE in which case the value is passed to TrexCfg.
+var ENABLE_DEBUG_MASK = DbgMaskEnabled()
+
+// method introduced so as unit testing is easier otherwise not necessary since environment variables
+// can't be changed so easily subsequent to a process start (meaning revaluation at runtime of no much use).
+func DbgMaskEnabled() bool {
+	return envtool.GetEnv(ENABLE_DBG_MASK_EV_NAME, ENABLE_DBG_MASK_VAL_DEFAULT) == ENABLE_DBG_MASK_VAL_TRUE
+}
+
+func RetrieveVehicleConfig(r redis.Client, m mongo.Client, id string) (*common.TRexConfigResponse, error) {
+	config := &common.TRexConfigResponse{}
+
+	reply, err := checkCacheForVehicleConfig(r, id)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if reply != nil {
+		err = json.Unmarshal(reply, config)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		if config.CANBus.DTCEnabled == nil {
+			config.CANBus.DTCEnabled = elptr.ElPtr(false)
+		}
+		return config, nil
+	}
+
+	config.LogLevel = common.Critical
+	// config.Log = &common.LogConfig{
+	// 	Matches: []common.LogConfigChannel{
+	// 		{
+	// 			Channel: common.ChannelCMD,
+	// 			Level:   common.Trace,
+	// 		},
+	// 	},
+	// }
+
+	config.CANBus.Enabled = true
+	config.CANBus.DataLogger = true
+
+	filters := make(FiltersMap)
+
+	f, err := checkFleetsDBForVehicleConfig(m, id)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+	}
+	if f != nil {
+		config.CANBus = f.CANBus
+		config.LogLevel = f.LogLevel
+		filters.AppendFilters(f.CANBus.Filters)
+	}
+
+	v, err := checkVehiclesDBForVehicleConfig(m, id)
+	if err != nil {
+		logger.Warn().Err(err).Send()
+	}
+	if v != nil {
+		config.CANBus = v.CANBus
+		config.LogLevel = v.LogLevel
+		config.DLTEnabled = v.DLTEnabled
+		config.DLTLevel = v.DLTLevel
+		// we should evaluate at run-time, not just at start-up time
+		if ENABLE_DEBUG_MASK {
+			config.DebugMask = v.DebugMask
+		}
+		config.IDPSEnabled = v.IDPSEnabled
+
+		filters.AppendFilters(v.CANBus.Filters)
+	}
+
+	config.CANBus.Filters = filters.ToSlice()
+	if config.CANBus.DTCEnabled == nil {
+		config.CANBus.DTCEnabled = elptr.ElPtr(false)
+	}
+	err = setCacheForVehicleConfig(r, id, config)
+	return config, err
+}
+
+func checkCacheForVehicleConfig(r redis.Client, id string) ([]byte, error) {
+	key := redis.CarConfigKey(id)
+
+	reply, err := redigo.Bytes(r.Execute("GET", key))
+	if err != nil {
+		if errors.Is(err, redigo.ErrNil) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return reply, nil
+}
+
+func checkVehiclesDBForVehicleConfig(m mongo.Client, id string) (*mongo.Vehicle, error) {
+	return m.GetVehicles().FindVehicle(&mongo.Vehicle{VIN: id})
+}
+
+func checkFleetsDBForVehicleConfig(m mongo.Client, id string) (*mongo.Fleet, error) {
+	return m.GetFleets().GetCANBusForVehicle(id)
+}
+
+func setCacheForVehicleConfig(r redis.Client, id string, config *common.TRexConfigResponse) error {
+	key := redis.CarConfigKey(id)
+
+	data, err := json.Marshal(config)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	batch := redis.NewRedisBatchCommands()
+	batch.Add("SET", key, data)
+	batch.Add("EXPIRE", key, redisObjectExpire.Seconds())
+
+	_, err = r.ExecuteBatch(batch)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+func RemoveCacheConfigForVehicles(r redis.Client, vins []string) error {
+	batch := redis.NewRedisBatchCommands()
+	for _, vin := range vins {
+		batch.Add("DEL", redis.CarConfigKey(vin))
+	}
+
+	_, err := r.ExecuteBatch(batch)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+type IntervalEdgeMask struct {
+	Interval *int
+	EdgeMask *common.BinaryHex
+}
+
+type FiltersMap map[string]IntervalEdgeMask
+
+func (f FiltersMap) AppendFilters(filters []common.CANFilter) {
+	for _, filter := range filters {
+		if filter.EdgeMask != nil && filter.EdgeMask.String() != "" {
+			f[filter.CANID] = IntervalEdgeMask{
+				EdgeMask: filter.EdgeMask,
+			}
+		} else if filter.Interval != nil {
+			f[filter.CANID] = IntervalEdgeMask{
+				Interval: filter.Interval,
+			}
+		}
+	}
+}
+
+func (f FiltersMap) ToSlice() []common.CANFilter {
+	filters := make([]common.CANFilter, 0, len(f))
+
+	for k, v := range f {
+		filters = append(filters, common.CANFilter{
+			CANID:    k,
+			Interval: v.Interval,
+			EdgeMask: v.EdgeMask,
+		})
+	}
+
+	return filters
+}

pkg/cachev2/vehicle_config_test.go

@@ -0,0 +1,203 @@
+package cachev2_test
+
+import (
+	"encoding/json"
+	"sort"
+	"testing"
+
+	cache "fiskerinc.com/modules/cachev2"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/mongo"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/testhelper"
+	"fiskerinc.com/modules/utils/elptr"
+	redigo "github.com/gomodule/redigo/redis"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRetrieveVehicleConfig(t *testing.T) {
+	setupRedisMock()
+	id := "TESTVIN1234567"
+
+	mockRedis = &mockRedisVehicleConfig{}
+	config, err := cache.RetrieveVehicleConfig(mockRedis, mongo.NewMockClient(), id)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	data, err := json.Marshal(&config)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	assert.Equal(t, `{"canbus":{"enabled":false,"data_logger_enabled":false,"dtc_enabled":false},"log_level":"trace"}`, string(data))
+
+	mockRedis = &mockRedisNoVehicleConfig{}
+	config, err = cache.RetrieveVehicleConfig(mockRedis, mongo.NewMockClient(), id)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	data, err = json.Marshal(&config)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveVehicleConfig", nil, err)
+	}
+	assert.Equal(t, `{"canbus":{"enabled":false,"data_logger_enabled":false,"dtc_enabled":false},"log_level":"trace","log":{"matches":[{"channel":"cmd","level":"trace"}]}}`, string(data))
+}
+
+func TestRetrieveVehicleConfigDbgMask(t *testing.T) {
+	setupRedisMock()
+	id := "TESTVIN1234567"
+	mockVehicle := mongo.Vehicle{VIN: id}
+	mockRedis = &mockRedisNoVehicleConfig{}
+
+	// validate that by default, retrieved debug value IS NOT passed to trxCfg
+	trxCfg, err := cache.RetrieveVehicleConfig(mockRedis, mongo.NewMockClient(), id)
+	existingValue := trxCfg.DebugMask
+	assert.Nil(t, err)
+	assert.NotNil(t, trxCfg)
+	// assert that trxCfg value is unchanged
+	assert.Equal(t, trxCfg.DebugMask, existingValue)
+
+	// let us try to enable
+	// the mock for redis is with no data so that code will fall through to the DB part
+	// we ensure that what we get from DB has speific debug mask which should be
+	// passed to Trex when the flag is true
+	t.Setenv(cache.ENABLE_DBG_MASK_EV_NAME, cache.ENABLE_DBG_MASK_VAL_TRUE)
+	cache.ENABLE_DEBUG_MASK = cache.DbgMaskEnabled()
+	mmc := mongo.NewMockMongoClient()
+	mockVehicle.DebugMask = "test"
+	mmc.GetVehicles().AddVehicle(&mockVehicle)
+	trxCfg, _ = cache.RetrieveVehicleConfig(mockRedis, mmc, id)
+	// now validate that Trex config got the value as set in the mocked vehicle
+	// (presumed as retrieved)
+	assert.Equal(t, trxCfg.DebugMask, mockVehicle.DebugMask)
+
+	// now set back the env variable so new values don't flow to trex
+	t.Setenv(cache.ENABLE_DBG_MASK_EV_NAME, cache.ENABLE_DBG_MASK_VAL_FALSE)
+	cache.ENABLE_DEBUG_MASK = cache.DbgMaskEnabled()
+	oldMask := mockVehicle.DebugMask
+	mockVehicle.DebugMask = "new-value"
+	// skipping adding to the cache/DB as we still had the valid reference
+	trxCfg, _ = cache.RetrieveVehicleConfig(mockRedis, mmc, id)
+	// assert that trex does not have new value
+	assert.NotEqual(t, trxCfg.DebugMask, oldMask)
+}
+
+func TestFiltersMap(t *testing.T) {
+	filters := make(cache.FiltersMap)
+
+	if len(filters) != 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 0, len(filters))
+		return
+	}
+
+	emptyHex := common.NewBinaryHex([]byte{})
+	bhex := common.BinaryHex("123")
+	filters.AppendFilters(
+		[]common.CANFilter{
+			{CANID: "123", Interval: elptr.ElPtr(123)},
+			{CANID: "456", Interval: elptr.ElPtr(456)},
+			{CANID: "789", EdgeMask: &emptyHex},
+			{CANID: "901", EdgeMask: &bhex},
+			{CANID: "222", Interval: elptr.ElPtr(123), EdgeMask: &bhex},
+			{CANID: "333", Interval: elptr.ElPtr(0)},
+		},
+	)
+	if len(filters) != 5 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 5, len(filters))
+		return
+	}
+
+	interval, ok := filters["123"]
+	if !ok || *interval.Interval != 123 && interval.EdgeMask != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 123, "error")
+		return
+	}
+
+	interval, ok = filters["456"]
+
+	if !ok || *interval.Interval != 456 && interval.EdgeMask != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 456, "error")
+		return
+	}
+
+	interval, ok = filters["789"]
+	if ok || interval.EdgeMask != nil || interval.Interval != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", emptyHex, "error")
+		return
+	}
+
+	interval, ok = filters["901"]
+	if !ok || interval.EdgeMask.String() != bhex.String() && interval.Interval != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", bhex, "error")
+		return
+	}
+
+	interval, ok = filters["222"]
+	if !ok || interval.EdgeMask.String() != bhex.String() && interval.Interval != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", bhex, "error")
+		return
+	}
+
+	interval, ok = filters["333"]
+	if !ok || interval.EdgeMask != nil && *interval.Interval != 0 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", nil, "error")
+		return
+	}
+
+	slice := filters.ToSlice()
+	if len(slice) != 5 {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", 5, len(slice))
+		return
+	}
+
+	sort.Slice(slice, func(i, j int) bool {
+		return slice[i].CANID < slice[j].CANID
+	})
+
+	if slice[0].CANID != "123" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "123", slice[0].CANID)
+		return
+	}
+
+	if slice[1].CANID != "222" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "222", slice[1].CANID)
+		return
+	}
+
+	if slice[2].CANID != "333" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "333", slice[2].CANID)
+		return
+	}
+
+	if slice[3].CANID != "456" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "456", slice[0].CANID)
+		return
+	}
+
+	if slice[4].CANID != "901" {
+		t.Errorf(testhelper.TestErrorTemplate, "TestFiltersMap", "901", slice[0].CANID)
+		return
+	}
+
+}
+
+type mockRedisVehicleConfig struct {
+	redis.Connection
+}
+
+func (c *mockRedisVehicleConfig) Execute(command ...interface{}) (interface{}, error) {
+	config := common.TRexConfigResponse{}
+	data, _ := json.Marshal(config)
+	return data, nil
+}
+
+type mockRedisNoVehicleConfig struct {
+	redis.Connection
+}
+
+func (c *mockRedisNoVehicleConfig) Execute(command ...interface{}) (interface{}, error) {
+	return nil, redigo.ErrNil
+}
+
+func (c *mockRedisNoVehicleConfig) ExecuteBatch(batch *redis.RedisBatchCommands) (interface{}, error) {
+	return nil, nil
+}

pkg/cachev2/vehicle_state.go

@@ -0,0 +1,591 @@
+package cachev2
+
+import (
+	"context"
+	"strconv"
+	"strings"
+	"time"
+
+	"fiskerinc.com/modules/common"
+	dt "fiskerinc.com/modules/dbc/state"
+	"fiskerinc.com/modules/logger"
+	redis "fiskerinc.com/modules/redisv2"
+	"fiskerinc.com/modules/utils/querystring"
+	redispkg "github.com/redis/go-redis/v9"
+
+	"github.com/pkg/errors"
+)
+
+const UPDATED_TIME_FORMAT = "2006-01-02T15:04:05Z"
+
+type stateParser func(state *common.CarState, key string, value []byte) (found bool, err error)
+
+func NewVehicleState(client redis.ClientInterface) *VehicleState {
+	return &VehicleState{redisClient: client}
+}
+
+type VehicleState struct {
+	redisClient redis.ClientInterface
+}
+
+func (v *VehicleState) Get(vin string) (common.CarState, error) {
+	var state common.CarState
+
+	values, err := v.queryVehicleState(vin)
+	if err != nil {
+		return state, err
+	}
+
+	state, err = v.ParsePayloadForVehicleState(values)
+	if err != nil {
+		return state, err
+	}
+
+	return state, nil
+}
+
+type QueryVehicleStateResponse struct {
+	CarSessionExists bool
+	HMISessionExists bool
+	CarState         map[string]string
+}
+
+type QueryVehicleStatePreResponse struct {
+	CarSessionExists *redispkg.BoolCmd
+	HMISessionExists *redispkg.BoolCmd
+	CarState         *redispkg.MapStringStringCmd
+}
+
+func (qvspr *QueryVehicleStatePreResponse) Resolve() (qvsr *QueryVehicleStateResponse, errR error) {
+	var err error
+	qvsr = &QueryVehicleStateResponse{}
+	qvsr.CarSessionExists, err = qvspr.CarSessionExists.Result()
+	if err != nil {
+		errR = errors.Wrap(errR, err.Error())
+	}
+	qvsr.HMISessionExists, err = qvspr.HMISessionExists.Result()
+	if err != nil {
+		errR = errors.Wrap(errR, err.Error())
+	}
+	qvsr.CarState, err = qvspr.CarState.Result()
+	if err != nil {
+		errR = errors.Wrap(errR, err.Error())
+	}
+	return
+}
+
+func (v *VehicleState) queryVehicleState(vin string) (QueryVehicleStateResponse, error) {
+	payload := QueryVehicleStateResponse{}
+
+	pipe := v.redisClient.GetClient().TxPipeline()
+
+	carSessionKey := pipe.SIsMember(context.Background(), redis.CarSessionsKey(), vin)
+	hmiSessionKey := pipe.SIsMember(context.Background(), redis.HMISessionsKey(), vin)
+	carStateHash := pipe.HGetAll(context.Background(), redis.CarStateHashKey(vin))
+
+	_, err := pipe.Exec(context.Background())
+	if err != nil {
+		return payload, errors.WithStack(err)
+	}
+	payload.CarSessionExists = carSessionKey.Val()
+	payload.HMISessionExists = hmiSessionKey.Val()
+	payload.CarState = carStateHash.Val()
+
+	return payload, nil
+}
+
+func (v *VehicleState) ParsePayloadForVehicleState(payload QueryVehicleStateResponse) (common.CarState, error) {
+	var state common.CarState
+	state.Online = payload.CarSessionExists
+	state.OnlineHMI = payload.HMISessionExists
+
+	var err error
+	err = v.parseStateValues(&state, payload.CarState, v.parseCarState)
+
+	return state, err
+}
+
+func ParsePayloadForVehicleState(payload *QueryVehicleStateResponse) (state *common.CarState, err error) {
+	state = &common.CarState{}
+	state.Online = payload.CarSessionExists
+	state.OnlineHMI = payload.HMISessionExists
+	err = parseStateValues(state, payload.CarState, ParseCarState)
+	return
+}
+
+func ParsePayloadForALVehicleState(payload *QueryVehicleStateResponse) (alState *common.CarStateAL, err error) {
+	alState.CarState = &common.CarState{}
+	alState.Online = payload.CarSessionExists
+	alState.OnlineHMI = payload.HMISessionExists
+	err = parseStateValues(alState.CarState, payload.CarState, ParseCarState)
+	return
+}
+
+func (v *VehicleState) parseStateValues(state *common.CarState, stateValues map[string]string, parser stateParser) error {
+	for key, value := range stateValues {
+
+		_, err := parser(state, string(key), []byte(value))
+		// log error, do not return error so we can read other properties for digital twin
+		if err != nil {
+			// strconv.Atoi: parsing "127.5": invalid syntax, track down. Add better info
+			logger.Err(err).Send()
+		}
+	}
+
+	return nil
+}
+
+func parseStateValues(state *common.CarState, stateValues map[string]string, parser stateParser) error {
+	for key, value := range stateValues {
+
+		_, err := parser(state, string(key), []byte(value))
+		// log error, do not return error so we can read other properties for digital twin
+		if err != nil {
+			logger.Err(err).Send()
+		}
+	}
+
+	return nil
+}
+
+func (v *VehicleState) parseCarState(state *common.CarState, key string, value []byte) (bool, error) {
+	var err error
+	val := string(value)
+
+	switch key {
+	case dt.VCU_VehChrgDchgMod:
+		state.GetVCU0x260().ChargeType = val
+	case dt.BMS_Bat_SoC_usable:
+		state.GetStateOfCharge().Usable, err = strconv.Atoi(val)
+	case dt.BMS_Bat_SOH:
+		state.GetStateOfCharge().Health, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_LeFrntWinPosnInfo:
+		state.GetWindows().LeftFront, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_RiFrntWinPosnInfo:
+		state.GetWindows().RightFront, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_LeReWinPosnInfo:
+		state.GetWindows().LeftRear, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_RiReWinPosnInfo:
+		state.GetWindows().RightRear, err = strconv.Atoi(val)
+	case dt.BMS_PwrBattRmngCpSOC:
+		state.GetBattery().Percent, err = strconv.Atoi(val)
+	case dt.BCM_ReDefrstHeatgCmd:
+		state.GetRearDefrost().On, err = strconv.ParseBool(val)
+	case dt.BCM_PasFrntDoorSts:
+		state.GetDoors().RightFront, err = strconv.ParseBool(val)
+	case dt.BCM_DrFrntDoorSts:
+		state.GetDoors().LeftFront, err = strconv.ParseBool(val)
+	case dt.BCM_FrntDrDoorLockSts:
+		state.GetLocks().Driver, err = notValue(strconv.ParseBool(val))
+	case dt.BCM_CenLockSwtSts:
+		state.GetLocks().All = (val == "2")
+	case dt.BCM_RiReDoorSts:
+		state.GetDoors().RightRear, err = strconv.ParseBool(val)
+	case dt.BCM_LeReDoorSts:
+		state.GetDoors().LeftRear, err = strconv.ParseBool(val)
+	case dt.BCM_FrntHoodLidSts:
+		state.GetDoors().Hood, err = strconv.ParseBool(val)
+	case dt.PLGM_TrSts:
+		state.GetDoors().Trunk, err = strconv.ParseBool(val)
+	case dt.BCM_SunroofPosnInfo:
+		state.GetSunroof().Sunroof, err = strconv.Atoi(val)
+	case dt.BCM_AP_TL_LeReWinPosnInfo:
+		state.GetMiscWindows().LeftRearQuarter, err = strconv.Atoi(val)
+	case dt.BCM_AP_TL_RiReWinPosnInfo:
+		state.GetMiscWindows().RightRearQuarter, err = strconv.Atoi(val)
+	case dt.BCM_AP_RW_WinPosnInfo:
+		state.GetMiscWindows().RearWindshield, err = strconv.Atoi(val)
+	case dt.BMS_BattAvrgT:
+		state.GetCellTemperature().AvgBatteryTemp, err = strconv.Atoi(val)
+	case dt.ECC_OutdT:
+		state.GetAmbientTemperature().Temperature, err = strconv.Atoi(val)
+	case dt.BCM_HeatedSteerWhlSt:
+		state.GetSteeringWheelHeat().On, err = strconv.ParseBool(val)
+	case dt.ESP_VehSpd:
+		state.GetVehicleSpeed().Speed, err = strconv.ParseFloat(val, 64)
+	case dt.VCU_DrvgMilg:
+		state.GetMaxRange().MaxMiles, err = strconv.Atoi(val)
+	case dt.PSM_PassSeatHeatgSts:
+		state.GetPassengerSeatHeat().Level, err = strconv.Atoi(val)
+	case dt.DSMC_DrvrSeatHeatgSts:
+		state.GetDriverSeatHeat().Level, err = strconv.Atoi(val)
+	case dt.ICC_TotMilg_ODO:
+		state.GetBattery().TotalMileageOdometer, err = querystring.ConvertStringToInt(val)
+	case dt.VCU_DCChrgRmngTi, dt.BMS_RmChrgTi_TrgtSoC:
+		state.GetChargingMetrics().RemainingChargingTime, err = strconv.Atoi(val)
+	case dt.IBS_BatteryVoltage:
+		state.GetBattery().BatteryVoltage, err = strconv.ParseFloat(val, 64)
+		state.GetBattery12V().IBS_BatteryVoltage = ref(state.GetBattery().BatteryVoltage)
+	case dt.VCU_GearSig:
+		var gear int
+		gear, err = strconv.Atoi(val)
+		state.GetGear().InPark = (gear <= 2)
+	case dt.BMS_RmChrgTi_FullChrg:
+		state.GetChargingMetrics().RemainingChargingTimeFull, err = strconv.Atoi(val)
+	case dt.ECC_InsdT:
+		state.GetCabinClimate().InternalTemperature, err = strconv.Atoi(val)
+	case dt.ECC_RemTSetSts:
+		state.GetCabinClimate().CabinTemperature, err = strconv.Atoi(val)
+	case dt.TBOX_GPSHei:
+		state.GetLocation().Altitude, err = strconv.ParseFloat(val, 64)
+	case dt.TBOX_GPSLongi:
+		state.GetLocation().Longitude, err = strconv.ParseFloat(val, 64)
+	case dt.TBOX_GPSLati:
+		state.GetLocation().Latitude, err = strconv.ParseFloat(val, 64)
+	case dt.DBC_VERSION:
+		state.DBCVersion = val
+	case dt.TREX_VERSION:
+		state.TRexVersion = val
+	case dt.TREX_IP:
+		state.IP = val
+	case dt.UPDATED_AT:
+		var t time.Time
+		t, err = time.Parse(UPDATED_TIME_FORMAT, strings.Trim(val, "\""))
+		if !t.IsZero() {
+			state.UpdatedAt = ref(t)
+		}
+	case dt.VCU_VehSt:
+		state.GetSafeState().VehicleSafeState = val == dt.VCU_VehSt_Safestate
+	case dt.VCU_VcuState:
+		state.GetSafeState().VCUSafeState = val == dt.VCU_VcuState_Safestate
+	case dt.MCU_F_ActSafeSt:
+		state.GetSafeState().MCUFrontSafeState = val == dt.MCU_F_ActSafeSt_AS0 || val == dt.MCU_F_ActSafeSt_ASC || val == dt.MCU_F_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_ActSafeSt:
+		state.GetSafeState().MCURearSafeState = val == dt.MCU_R_ActSafeSt_AS0 || val == dt.MCU_R_ActSafeSt_ASC || val == dt.MCU_R_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_Decoup_State:
+		state.GetSafeState().MCURearDecoupState = val == dt.MCU_R_Decoup_State_Connected
+	case dt.MCU_F_CrtMod:
+		state.GetSafeState().MCUFrontInverterError = val == dt.MCU_F_CrtMod_Internal_inverter_error || val == dt.MCU_F_CrtMod_Invalid
+	case dt.MCU_R_CrtMod:
+		state.GetSafeState().MCURearInverterError = val == dt.MCU_R_CrtMod_Internal_inverter_error || val == dt.MCU_R_CrtMod_Invalid
+	case dt.ACU_Drvr_Occpt_St:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.DriverOccupySeatState = ref(vi)
+	case dt.BCM_PwrMod:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.PowerMode = ref(vi)
+	case dt.PWC_ChrgSts:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.ChargingStatus = ref(vi)
+	case dt.VCU_RdyLamp:
+		state.GetVehicleReadyState().IsVehicleReady, err = strconv.ParseBool(val)
+		// New untested signals
+		// case dt.IBS_SOCUpperTolerance:
+		// 	var vi float64
+		// 	vi, err = strconv.ParseFloat(val, 64)
+		// 	state.GetExpandedSignals().IBS_SOCUpperTolerance = ref(vi)
+		// case dt.IBS_SOCLowerTolerance:
+		// 	var vi float64
+		// 	vi, err = strconv.ParseFloat(val, 64)
+		// 	state.GetExpandedSignals().IBS_SOCLowerTolerance = ref(vi)
+	case dt.IBS_StateOfCharge:
+		var vi float64
+		vi, err = strconv.ParseFloat(val, 64)
+		state.GetBattery12V().IBS_StateOfCharge = ref(vi)
+	case dt.IBS_StateOfHealth:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetBattery12V().IBS_StateOfHealth = ref(vi)
+	case dt.IBS_NominalCapacity:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().IBS_NominalCapacity = ref(vi)
+	case dt.IBS_AvailableCapacity:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().IBS_AvailableCapacity = ref(vi)
+	case dt.BCM_TotMilg_ODO:
+		var vi float64
+		vi, err = strconv.ParseFloat(val, 64)
+		state.GetExpandedSignals().BCM_TotMilg_ODO = ref(vi)
+	case dt.BMS_SwVersS:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVersS = ref(vi)
+	case dt.BMS_SwVersM:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVersM = ref(vi)
+	case dt.BMS_SwVers:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVers = ref(vi)
+	case dt.BMS_AccueDchaTotAh:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_AccueDchaTotAh = ref(vi)
+	case dt.BMS_AccueChrgTotAh:
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_AccueChrgTotAh = ref(vi)
+	case dt.TBOX_Heading:
+		state.GetLocation().Heading, err = strconv.ParseFloat(val, 64)
+	case dt.PKC_KeyStsMod:
+		state.GetGear().Immobilizer = val
+	}
+
+	return true, errors.WithStack(err)
+}
+
+func ParseCarState(state *common.CarState, key string, value []byte) (found bool, err error) {
+	val := string(value)
+	switch key {
+	case dt.VCU_VehChrgDchgMod:
+		found = true
+		state.GetVCU0x260().ChargeType = val
+	case dt.BMS_Bat_SoC_usable:
+		found = true
+		state.GetStateOfCharge().Usable, err = strconv.Atoi(val)
+	case dt.BMS_Bat_SOH:
+		found = true
+		state.GetStateOfCharge().Health, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_LeFrntWinPosnInfo:
+		found = true
+		state.GetWindows().LeftFront, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_RiFrntWinPosnInfo:
+		found = true
+		state.GetWindows().RightFront, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_LeReWinPosnInfo:
+		found = true
+		state.GetWindows().LeftRear, err = strconv.Atoi(val)
+	case dt.BCM_AP_FL_RiReWinPosnInfo:
+		found = true
+		state.GetWindows().RightRear, err = strconv.Atoi(val)
+	case dt.BMS_PwrBattRmngCpSOC:
+		found = true
+		state.GetBattery().Percent, err = strconv.Atoi(val)
+	case dt.BCM_ReDefrstHeatgCmd:
+		found = true
+		state.GetRearDefrost().On, err = strconv.ParseBool(val)
+	case dt.BCM_PasFrntDoorSts:
+		found = true
+		state.GetDoors().RightFront, err = strconv.ParseBool(val)
+	case dt.BCM_DrFrntDoorSts:
+		found = true
+		state.GetDoors().LeftFront, err = strconv.ParseBool(val)
+	case dt.BCM_FrntDrDoorLockSts:
+		found = true
+		state.GetLocks().Driver, err = notValue(strconv.ParseBool(val))
+	case dt.BCM_CenLockSwtSts:
+		found = true
+		state.GetLocks().All = (val == "2")
+	case dt.BCM_RiReDoorSts:
+		found = true
+		state.GetDoors().RightRear, err = strconv.ParseBool(val)
+	case dt.BCM_LeReDoorSts:
+		found = true
+		state.GetDoors().LeftRear, err = strconv.ParseBool(val)
+	case dt.BCM_FrntHoodLidSts:
+		found = true
+		state.GetDoors().Hood, err = strconv.ParseBool(val)
+	case dt.PLGM_TrSts:
+		found = true
+		state.GetDoors().Trunk, err = strconv.ParseBool(val)
+	case dt.BCM_SunroofPosnInfo:
+		found = true
+		state.GetSunroof().Sunroof, err = strconv.Atoi(val)
+	case dt.BCM_AP_TL_LeReWinPosnInfo:
+		found = true
+		state.GetMiscWindows().LeftRearQuarter, err = strconv.Atoi(val)
+	case dt.BCM_AP_TL_RiReWinPosnInfo:
+		found = true
+		state.GetMiscWindows().RightRearQuarter, err = strconv.Atoi(val)
+	case dt.BCM_AP_RW_WinPosnInfo:
+		found = true
+		state.GetMiscWindows().RearWindshield, err = strconv.Atoi(val)
+	case dt.BMS_BattAvrgT:
+		found = true
+		state.GetCellTemperature().AvgBatteryTemp, err = strconv.Atoi(val)
+	case dt.ECC_OutdT:
+		found = true
+		state.GetAmbientTemperature().Temperature, err = strconv.Atoi(val)
+	case dt.BCM_HeatedSteerWhlSt:
+		found = true
+		state.GetSteeringWheelHeat().On, err = strconv.ParseBool(val)
+	case dt.ESP_VehSpd:
+		found = true
+		state.GetVehicleSpeed().Speed, err = strconv.ParseFloat(val, 64)
+	case dt.VCU_DrvgMilg:
+		found = true
+		state.GetMaxRange().MaxMiles, err = strconv.Atoi(val)
+	case dt.PSM_PassSeatHeatgSts:
+		found = true
+		state.GetPassengerSeatHeat().Level, err = strconv.Atoi(val)
+	case dt.DSMC_DrvrSeatHeatgSts:
+		found = true
+		state.GetDriverSeatHeat().Level, err = strconv.Atoi(val)
+	case dt.ICC_TotMilg_ODO:
+		found = true
+		state.GetBattery().TotalMileageOdometer, err = querystring.ConvertStringToInt(val)
+	case dt.VCU_DCChrgRmngTi, dt.BMS_RmChrgTi_TrgtSoC:
+		found = true
+		state.GetChargingMetrics().RemainingChargingTime, err = strconv.Atoi(val)
+	case dt.IBS_BatteryVoltage:
+		found = true
+		state.GetBattery().BatteryVoltage, err = strconv.ParseFloat(val, 64)
+		state.GetBattery12V().IBS_BatteryVoltage = ref(state.GetBattery().BatteryVoltage)
+	case dt.VCU_GearSig:
+		found = true
+		var gear int
+		gear, err = strconv.Atoi(val)
+		state.GetGear().InPark = (gear <= 2)
+	case dt.BMS_RmChrgTi_FullChrg:
+		found = true
+		state.GetChargingMetrics().RemainingChargingTimeFull, err = strconv.Atoi(val)
+	case dt.ECC_InsdT:
+		found = true
+		state.GetCabinClimate().InternalTemperature, err = strconv.Atoi(val)
+	case dt.ECC_RemTSetSts:
+		found = true
+		state.GetCabinClimate().CabinTemperature, err = strconv.Atoi(val)
+	case dt.TBOX_GPSHei:
+		found = true
+		state.GetLocation().Altitude, err = strconv.ParseFloat(val, 64)
+	case dt.TBOX_GPSLongi:
+		found = true
+		state.GetLocation().Longitude, err = strconv.ParseFloat(val, 64)
+	case dt.TBOX_GPSLati:
+		found = true
+		state.GetLocation().Latitude, err = strconv.ParseFloat(val, 64)
+	case dt.DBC_VERSION:
+		found = true
+		state.DBCVersion = val
+	case dt.TREX_VERSION:
+		found = true
+		state.TRexVersion = val
+	case dt.TREX_IP:
+		found = true
+		state.IP = val
+	case dt.UPDATED_AT:
+		found = true
+		var t time.Time
+		t, err = time.Parse(UPDATED_TIME_FORMAT, strings.Trim(val, "\""))
+		if !t.IsZero() {
+			state.UpdatedAt = ref(t)
+		}
+	case dt.VCU_VehSt:
+		found = true
+		state.GetSafeState().VehicleSafeState = val == dt.VCU_VehSt_Safestate
+	case dt.VCU_VcuState:
+		found = true
+		state.GetSafeState().VCUSafeState = val == dt.VCU_VcuState_Safestate
+	case dt.MCU_F_ActSafeSt:
+		found = true
+		state.GetSafeState().MCUFrontSafeState = val == dt.MCU_F_ActSafeSt_AS0 || val == dt.MCU_F_ActSafeSt_ASC || val == dt.MCU_F_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_ActSafeSt:
+		found = true
+		state.GetSafeState().MCURearSafeState = val == dt.MCU_R_ActSafeSt_AS0 || val == dt.MCU_R_ActSafeSt_ASC || val == dt.MCU_R_ActSafeSt_ASC_Emergency
+	case dt.MCU_R_Decoup_State:
+		found = true
+		state.GetSafeState().MCURearDecoupState = val == dt.MCU_R_Decoup_State_Connected
+	case dt.MCU_F_CrtMod:
+		found = true
+		state.GetSafeState().MCUFrontInverterError = val == dt.MCU_F_CrtMod_Internal_inverter_error || val == dt.MCU_F_CrtMod_Invalid
+	case dt.MCU_R_CrtMod:
+		found = true
+		state.GetSafeState().MCURearInverterError = val == dt.MCU_R_CrtMod_Internal_inverter_error || val == dt.MCU_R_CrtMod_Invalid
+	case dt.ACU_Drvr_Occpt_St:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.DriverOccupySeatState = ref(vi)
+	case dt.BCM_PwrMod:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.PowerMode = ref(vi)
+	case dt.PWC_ChrgSts:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.ChargingStatus = ref(vi)
+	case dt.VCU_RdyLamp:
+		found = true
+		state.GetVehicleReadyState().IsVehicleReady, err = strconv.ParseBool(val)
+		// New untested signals
+		// case dt.IBS_SOCUpperTolerance:
+		found = true
+		// 	var vi float64
+		// 	vi, err = strconv.ParseFloat(val, 64)
+		// 	state.GetExpandedSignals().IBS_SOCUpperTolerance = ref(vi)
+		// case dt.IBS_SOCLowerTolerance:
+		found = true
+	// 	var vi float64
+	// 	vi, err = strconv.ParseFloat(val, 64)
+	// 	state.GetExpandedSignals().IBS_SOCLowerTolerance = ref(vi)
+	case dt.IBS_StateOfCharge:
+		found = true
+		var vi float64
+		vi, err = strconv.ParseFloat(val, 64)
+		state.GetBattery12V().IBS_StateOfCharge = ref(vi)
+	case dt.IBS_StateOfHealth:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetBattery12V().IBS_StateOfHealth = ref(vi)
+	case dt.IBS_NominalCapacity:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().IBS_NominalCapacity = ref(vi)
+	case dt.IBS_AvailableCapacity:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().IBS_AvailableCapacity = ref(vi)
+	case dt.BCM_TotMilg_ODO:
+		found = true
+		var vi float64
+		vi, err = strconv.ParseFloat(val, 64)
+		state.GetExpandedSignals().BCM_TotMilg_ODO = ref(vi)
+	case dt.BMS_SwVersS:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVersS = ref(vi)
+	case dt.BMS_SwVersM:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVersM = ref(vi)
+	case dt.BMS_SwVers:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_SwVers = ref(vi)
+	case dt.BMS_AccueDchaTotAh:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_AccueDchaTotAh = ref(vi)
+	case dt.BMS_AccueChrgTotAh:
+		found = true
+		var vi int
+		vi, err = strconv.Atoi(val)
+		state.GetExpandedSignals().BMS_AccueChrgTotAh = ref(vi)
+	case dt.TBOX_Heading:
+		found = true
+		state.GetLocation().Heading, err = strconv.ParseFloat(val, 64)
+	case dt.PKC_KeyStsMod:
+		found = true
+		state.GetGear().Immobilizer = val
+	}
+
+	return found, errors.WithStack(err)
+}
+
+func ref[T any](v T) *T {
+	return &v
+}
+
+func IsCarOnline(client redis.ClientInterface, vin string) (bool, error) {
+	res := client.GetClient().SIsMember(context.Background(), redis.CarSessionsKey(), vin)
+	return res.Result()
+}
+
+func notValue(value bool, err error) (bool, error) {
+	return !value, err
+}

pkg/cachev2/vehicle_state_multi.go

@@ -0,0 +1,101 @@
+package cachev2
+
+import (
+	"context"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/logger"
+	redis "fiskerinc.com/modules/redisv2"
+	"github.com/pkg/errors"
+
+	"fiskerinc.com/modules/db/queries"
+)
+
+func GetVINListDigitalTwin(vins []string, redisClient *redis.Connection) (digitalTwins map[string]common.CarState, errorList []error) {
+	digitalTwins = make(map[string]common.CarState)
+
+	pipe := redisClient.TxPipeline()
+	responses := make([]QueryVehicleStatePreResponse, 0, len(vins))
+	for _, vin := range vins {
+		rr := QueryVehicleStatePreResponse{}
+		rr.CarSessionExists = pipe.SIsMember(context.Background(), redis.CarSessionsKey(), vin)
+		rr.HMISessionExists = pipe.SIsMember(context.Background(), redis.HMISessionsKey(), vin)
+		rr.CarState = pipe.HGetAll(context.Background(), redis.CarStateHashKey(vin))
+		responses = append(responses, rr)
+	}
+
+	_, err := pipe.Exec(context.Background())
+	if err != nil {
+		errorList = append(errorList, err)
+		return
+	}
+
+	for index, resStruct := range responses {
+		bb, err := resStruct.Resolve()
+		if err != nil {
+			err = errors.Wrapf(err, "VIN: %s", vins[index])
+			errorList = append(errorList, err)
+			continue
+		}
+		state, err := ParsePayloadForVehicleState(bb)
+		if err != nil {
+			err = errors.Wrapf(err, "VIN: %s", vins[index])
+			errorList = append(errorList, err)
+		}
+		if state != nil {
+			digitalTwins[vins[index]] = *state
+		}
+	}
+
+	return
+}
+
+type getALDigitalTwinDBFieldsResults struct {
+	results []common.CarPKCOSVersion
+	err     error
+}
+
+func GetVINListALDigitalTwin(vins []string, redisClient *redis.Connection, carsDB queries.CarsInterface) (digitalTwinsAL map[string]common.CarStateAL, errorList []error) {
+	digitalTwinsAL = make(map[string]common.CarStateAL)
+	dbResultsChan := make(chan getALDigitalTwinDBFieldsResults)
+	// While the redis is fetching its stored info, we should go out to the database and fetch the database knowledge we need
+	// May want to put this information into a cache
+	go getALDigitalTwinDBFields(vins, carsDB, dbResultsChan)
+
+	digitalTwins, errorList := GetVINListDigitalTwin(vins, redisClient)
+	dbResults := <-dbResultsChan
+	if dbResults.err != nil {
+		errorList = append(errorList, dbResults.err)
+		return
+	}
+
+	for _, dbRes := range dbResults.results {
+		temp := common.CarStateAL{}
+		dt, ok := digitalTwins[dbRes.Vin]
+		if !ok {
+			logger.Warn().Str("VIN", dbRes.Vin).Msg("AL Digital Twin Missing Redis")
+			// Think I need to initiate it so we don't null memory maybe?
+			dt = common.CarState{}
+		}
+		temp.CarState = &dt
+		temp.OSVersion = dbRes.OSVersion
+		temp.PKCVersion = dbRes.PKCVersion
+		temp.SumsVersion = dbRes.SumsVersion
+		digitalTwinsAL[dbRes.Vin] = temp
+	}
+
+	return
+}
+
+// IDK how I feel about having this database functionality inside /cache
+func getALDigitalTwinDBFields(vins []string, carsDB queries.CarsInterface, out chan getALDigitalTwinDBFieldsResults) {
+	results, err := carsDB.GetSoftwareAndPKCVersions(vins)
+	out <- getALDigitalTwinDBFieldsResults{
+		results: results,
+		err:     err,
+	}
+}
+
+type ALDB interface {
+	GetCars() queries.CarsInterface
+}

pkg/cachev2/vehicle_state_test.go

@@ -0,0 +1,199 @@
+package cachev2_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	cache "fiskerinc.com/modules/cachev2"
+	"fiskerinc.com/modules/common"
+	redis "fiskerinc.com/modules/redisv2"
+	"github.com/go-redis/redismock/v9"
+	"github.com/stretchr/testify/assert"
+)
+
+//HERE
+
+func TestConnGetVehicleState(t *testing.T) {
+	var updateTime = time.Date(2020, time.October, 3, 12, 10, 0, 0, time.UTC)
+	vin := "TESTVIN123"
+	// redisMock := tester.NewRedisMock()
+	// redisPool := tester.NewMockClientPool(redisMock)
+	redisClientFakeConnection, clientMock := redismock.NewClientMock()
+	_ = clientMock
+	redisClient := redis.NewClient(redisClientFakeConnection)
+
+	testCases := map[string]struct {
+		sismemberResults map[string]map[string]interface{}
+		hgetallResults   map[string][]interface{}
+		expResp          common.CarState
+		expErr           error
+	}{
+		"correct": {
+			sismemberResults: map[string]map[string]interface{}{
+				redis.CarSessionsKey(): {
+					vin: int64(1),
+				},
+				redis.HMISessionsKey(): {
+					vin: int64(1),
+				},
+			},
+			hgetallResults: map[string][]interface{}{
+				fmt.Sprintf("car:%s:state", vin): {
+					[]byte("DSMC_DrvrSeatHeatgSts"), []byte("2"),
+					[]byte("ESP_VehSpd"), []byte("123.4"),
+					[]byte("BMS_RmChrgTi_TrgtSoC"), []byte("5000"),
+					[]byte("BMS_RmChrgTi_FullChrg"), []byte("6000"),
+					[]byte("VCU_VehChrgDchgMod"), []byte("DC_charging"),
+					[]byte("BCM_AP_FL_LeReWinPosnInfo"), []byte("30"),
+					[]byte("BCM_ReDefrstHeatgCmd"), []byte("1"),
+					[]byte("BCM_FrntHoodLidSts"), []byte("1"),
+					[]byte("BMS_Bat_SOH"), []byte("20"),
+					[]byte("ICC_TotMilg_ODO"), []byte("2345"),
+					[]byte("IBS_BatteryVoltage"), []byte("12.3"),
+					[]byte("TBOX_GPSHei"), []byte("16"),
+					[]byte("ECC_OutdT"), []byte("30"),
+					[]byte("PSM_PassSeatHeatgSts"), []byte("4"),
+					[]byte("TBOX_GPSLati"), []byte("35.831"),
+					[]byte("BCM_PasFrntDoorSts"), []byte("0"),
+					[]byte("BCM_CenLockSwtSts"), []byte("3"),
+					[]byte("BCM_RiReDoorSts"), []byte("1"),
+					[]byte("BCM_LeReDoorSts"), []byte("1"),
+					[]byte("VCU_DrvgMilg"), []byte("1234"),
+					[]byte("TBOX_GPSLongi"), []byte("-120.398"),
+					[]byte("BCM_AP_FL_RiReWinPosnInfo"), []byte("40"),
+					[]byte("BCM_FrntDrDoorLockSts"), []byte("1"),
+					[]byte("BCM_DrFrntDoorSts"), []byte("0"),
+					[]byte("BCM_AP_TL_LeReWinPosnInfo"), []byte("60"),
+					[]byte("ECC_RemTSetSts"), []byte("120"),
+					[]byte("BCM_AP_FL_RiFrntWinPosnInfo"), []byte("20"),
+					[]byte("BMS_PwrBattRmngCpSOC"), []byte("50"),
+					[]byte("BCM_AP_TL_RiReWinPosnInfo"), []byte("70"),
+					[]byte("BCM_HeatedSteerWhlSt"), []byte("1"),
+					[]byte("BCM_AP_RW_WinPosnInfo"), []byte("80"),
+					[]byte("ECC_InsdT"), []byte("30"),
+					[]byte("updated"), []byte(`"2020-10-03T12:10:00Z"`),
+					[]byte("BMS_Bat_SoC_usable"), []byte("10"),
+					[]byte("BCM_AP_FL_LeFrntWinPosnInfo"), []byte("10"),
+					[]byte("BCM_SunroofPosnInfo"), []byte("50"),
+					[]byte("BMS_BattAvrgT"), []byte("90"),
+					[]byte("dbc_version"), []byte("hash"),
+					[]byte("VCU_VehSt"), []byte("12"),
+					[]byte("VCU_VcuState"), []byte("18"),
+					[]byte("MCU_F_ActSafeSt"), []byte("4"),
+					[]byte("MCU_R_ActSafeSt"), []byte("2"),
+					[]byte("MCU_R_Decoup_State"), []byte("3"),
+					[]byte("MCU_F_CrtMod"), []byte("7"),
+					[]byte("MCU_R_CrtMod"), []byte("8"),
+					[]byte("VCU_RdyLamp"), []byte("1"),
+				},
+			},
+			expResp: common.CarState{
+				Online:    true,
+				OnlineHMI: true,
+				VehicleSpeed: &common.VehicleSpeed{
+					Speed: 123.4,
+				},
+				Battery: &common.Battery{
+					Percent:              50,
+					TotalMileageOdometer: 2345,
+					BatteryVoltage:       12.3,
+				},
+				MaxRange: &common.MaxRange{
+					MaxMiles: 1234,
+				},
+				Doors: &common.Doors{
+					Hood:       true,
+					LeftFront:  false,
+					LeftRear:   true,
+					RightFront: false,
+					RightRear:  true,
+				},
+				Location: &common.Location{
+					Altitude:  16,
+					Longitude: -120.398,
+					Latitude:  35.831,
+				},
+				Locks: &common.Locks{
+					Driver: false,
+					All:    false,
+				},
+				Windows: &common.Windows{
+					LeftFront:  10,
+					LeftRear:   30,
+					RightFront: 20,
+					RightRear:  40,
+				},
+				MiscWindows: &common.MiscWindows{
+					LeftRearQuarter:  60,
+					RightRearQuarter: 70,
+					RearWindshield:   80,
+				},
+				Sunroof: &common.Sunroof{
+					Sunroof: 50,
+				},
+				CabinClimate: &common.CabinClimate{
+					CabinTemperature:    120,
+					InternalTemperature: 30,
+				},
+				RearDefrost: &common.RearDefrost{
+					On: true,
+				},
+				DriverSeatHeat: &common.DriverSeatHeat{
+					Level: 2,
+				},
+				PassengerSeatHeat: &common.PassengerSeatHeat{
+					Level: 4,
+				},
+				CellTemperature: &common.CellTemperature{
+					AvgBatteryTemp: 90,
+				},
+				ChargingMetrics: &common.VCUChargingMetrics{
+					RemainingChargingTime:     5000,
+					RemainingChargingTimeFull: 6000,
+				},
+				SteeringWheelHeat: &common.SteeringWheelHeat{
+					On: true,
+				},
+				AmbientTemperature: &common.AmbientTemperature{
+					Temperature: 30,
+				},
+				VCU0x260: &common.VCU0x260Descriptor{
+					ChargeType: "DC_charging",
+				},
+				StateOfCharge: &common.StateOfCharge{
+					Usable: 10,
+					Health: 20,
+				},
+				DBCVersion: "hash",
+				UpdatedAt:  &updateTime,
+				SafeState: &common.SafeState{
+					VehicleSafeState:      false,
+					VCUSafeState:          true,
+					MCUFrontSafeState:     false,
+					MCURearSafeState:      true,
+					MCURearDecoupState:    false,
+					MCUFrontInverterError: true,
+					MCURearInverterError:  false,
+				},
+				VehicleReadyState: &common.VehicleReadyState{
+					IsVehicleReady: true,
+				},
+			},
+			expErr: nil,
+		},
+	}
+
+	parser := cache.NewVehicleState(redisClient)
+
+	for tName, tt := range testCases {
+		t.Run(tName, func(t *testing.T) {
+			// clientMock.ExpectSIsMember()
+			// redisMock.SISMEMBEResults = tt.sismemberResults
+			// redisMock.HGETALLResults = tt.hgetallResults
+			state, err := parser.Get(vin)
+			assert.Equal(t, tt.expErr, err)
+			assert.Equal(t, tt.expResp, state)
+		})
+	}
+}

pkg/cachev2/vehicle_state_towman.go

@@ -0,0 +1,40 @@
+package cachev2
+
+import (
+	"context"
+	"errors"
+
+	"fiskerinc.com/modules/common"
+	redis "fiskerinc.com/modules/redisv2"
+	"fiskerinc.com/modules/utils/elptr"
+)
+
+func GetTowManDigitalTwin(vin string, redisClient *redis.Connection)(tdt common.TowmanDigitalTwin, err error){
+	// TODO: Make this more efficient with specific gets
+	dTwins, errorList := GetVINListDigitalTwin([]string{vin}, redisClient)
+	if len(errorList) > 0 {
+		err = errorList[0]
+	}
+	if err != nil {
+		return
+	}
+
+	digitalTwin, ok := dTwins[vin]
+	if !ok {
+		err = errors.New("digital twin not found")
+		return
+	}
+
+	tdt.Gear = digitalTwin.GetGear()
+	tdt.Location = digitalTwin.GetLocation()
+	tdt.Online = digitalTwin.Online
+	tdt.Charging = elptr.ElPtr((digitalTwin.GetVCU0x260().ChargeType != "initial_value") && (digitalTwin.GetVCU0x260().ChargeType != ""))
+
+	return
+}
+
+func GetVehicleLocation(vin string, redisClient *redis.Connection)(location common.Location, err error){
+	res := redisClient.HGet(context.Background(), redis.CarLocationsKey(), vin)
+	err = res.Scan(&location)
+	return
+}

pkg/cachev2/verify.go

@@ -0,0 +1,60 @@
+package cachev2
+
+import (
+	"fmt"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	redis "fiskerinc.com/modules/redisv2"
+
+	"github.com/pkg/errors"
+)
+
+// VerifyCarToDriver checks cache and DB for car to driver relationship.
+// If relationship exists and not in cache, will cache value.
+//
+//	car:<VIN>:driver:<DRIVER_ID>
+func VerifyCarToDriver(clientPool redis.ClientInterface, db queries.CarsInterface, vin string, driverID string) (bool, error) {
+	key := redis.CarToDriverKey(vin, driverID)
+
+	ok, err := redisCheckGet(clientPool, key)
+	if err != nil {
+		return ok, err
+	}
+
+	if ok {
+		return ok, err
+	}
+
+	carToDrivers, err := db.SelectCarToDriver(&common.CarToDriver{VIN: vin, DriverID: driverID})
+	if err != nil {
+		return false, err
+	}
+	verified := len(carToDrivers) == 1
+	redisPlaceDriverCache(clientPool, key, verified)
+
+	return verified, err
+}
+
+func redisCheckGet(redisClient redis.ClientInterface, key string) (bool, error) {
+	value, err := redisClient.Get(key)
+	if err != nil {
+		if errors.Is(err, redis.ErrNil) {
+			err = nil
+		}
+		return false, err
+	}
+	v, ok := value.(bool)
+	if !ok {
+		err = fmt.Errorf("failed to convert %v interface to bool", value)
+	}
+	return v, err
+}
+
+func redisPlaceDriverCache(redisClient redis.ClientInterface, key string, verified bool) (err error) {
+	batch := redis.NewRedisBatchCommands()
+	batch.Add(redis.Command{Command: "SET", Arguments: []interface{}{key, verified}})
+	batch.Add(redis.Command{Command: "EXPIRE", Arguments: []interface{}{key, redisObjectExpire.Seconds()}})
+	_, err = redisClient.ExecuteBatch(batch)
+	return
+}

pkg/cachev2/verify_test.go

@@ -0,0 +1,55 @@
+package cachev2_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries/mocks"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/redis/tester"
+	"fiskerinc.com/modules/testhelper"
+
+	redigo "github.com/gomodule/redigo/redis"
+)
+
+type mockRedisCacheDriverToCars struct {
+	redis.Connection
+}
+
+func (c *mockRedisCacheDriverToCars) Execute(command ...interface{}) (interface{}, error) {
+	return []byte("1"), nil
+}
+
+type mockRedisEmptyCacheDriverToCars struct {
+	redis.Connection
+}
+
+func (c *mockRedisEmptyCacheDriverToCars) Execute(command ...interface{}) (interface{}, error) {
+	return nil, redigo.ErrNil
+}
+
+func (c *mockRedisEmptyCacheDriverToCars) ExecuteBatch(batch *redis.RedisBatchCommands) (interface{}, error) {
+	return nil, nil
+}
+
+func TestVerifyCarToDriver(t *testing.T) {
+	setupRedisMock()
+	mockDB := &mocks.MockCars{
+		SelectCarsForDrivers: []common.CarToDriver{{}},
+	}
+
+	mockRedis = &mockRedisCacheDriverToCars{}
+	redisPool := tester.NewMockClientPool(mockRedis)
+	_, err := cache.VerifyCarToDriver(redisPool, mockDB, "VALID_VIN", "VALID_ID")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", nil, err)
+	}
+
+	mockRedis = &mockRedisEmptyCacheDriverToCars{}
+	redisPool = tester.NewMockClientPool(mockRedis)
+	_, err = cache.VerifyCarToDriver(redisPool, mockDB, "VALID_VIN", "VALID_ID")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveAndCacheDriverIDs", nil, err)
+	}
+}

pkg/carcommand/common.go

@@ -0,0 +1,37 @@
+package carcommand
+
+import (
+	"github.com/pkg/errors"
+)
+
+var (
+	ErrNoICCIDForWakeUp     = errors.New("no ICCID for sending wake up SMS")
+	ErrWakeUpMessageNotSent = errors.New("wake up message wasn't delivered")
+)
+
+var acceptedCommands = map[string]struct{}{
+	"doors_lock":             {},
+	"doors_unlock":           {},
+	"vent_windows":           {},
+	"close_windows":          {},
+	"california_mode":        {},
+	"trunk_open":             {},
+	"trunk_close":            {},
+	"flash_headlights":       {},
+	"alert":                  {},
+	"temp_cabin":             {},
+	"defrost":                {},
+	"driver_seat_preheat":    {},
+	"passenger_seat_preheat": {},
+	"steering_wheel_preheat": {},
+	"precondition":           {},
+	"charging":               {},
+}
+
+func ValidateCommand(cmd string) error {
+	if _, ok := acceptedCommands[cmd]; !ok {
+		return errors.New("unknown command")
+	}
+
+	return nil
+}

pkg/carcommand/wake_up.go

@@ -0,0 +1,128 @@
+package carcommand
+
+import (
+	"context"
+
+	"fiskerinc.com/modules/cache"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/grpc/sms"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+	"fiskerinc.com/modules/utils/envtool"
+	"fiskerinc.com/modules/validator"
+
+	"github.com/pkg/errors"
+)
+
+type CarWakeUp struct {
+	db  queries.CarsInterface
+	sms sms.SMSServiceClient
+}
+
+func NewCarWakeUp(db queries.CarsInterface, sms sms.SMSServiceClient) *CarWakeUp {
+	return &CarWakeUp{
+		db:  db,
+		sms: sms,
+	}
+}
+
+func (c *CarWakeUp) WakeUp(vin string, await bool) error {
+	logger.Debug().Msgf("CarWakeUp.WakeUp %s", vin)
+	car, err := c.db.SelectByVIN(vin)
+	if err != nil {
+		return err
+	}
+
+	if car.ICCID == "" {
+		return errors.WithStack(ErrNoICCIDForWakeUp)
+	}
+
+	// KafkaServiceTarget is where the sms delivery status will be sent if await is false
+	// Need to be added to aftersales and ota_update
+	smsWakeUp := sms.SendSMSRequest{
+		ICCID:              car.ICCID,
+		MessageText:        ".",
+		Await:              await,
+		KafkaServiceTarget: envtool.GetEnv("SMS_SERVICE_KAFKA_CALLBACK", "attendant_service"),
+	}
+
+	if ok, err := validator.ValidateICCIDSimple(smsWakeUp.ICCID); !ok || err != nil {
+		err = errors.New("iccid " + smsWakeUp.ICCID + " is invalid")
+		logger.Warn().Err(errors.WithStack(err)).Send()
+		return err
+	}
+	_, err = c.sms.HandleSMSSend(context.Background(), &smsWakeUp)
+	if err != nil {
+		logger.Error().Err(errors.WithStack(err)).Send()
+		return errors.WithStack(ErrWakeUpMessageNotSent)
+	}
+
+	return nil
+}
+
+func (c *CarWakeUp) WakeUpQueue(vin string, await bool) (qs *sms.SMSQueueResponse, err error) {
+	logger.Debug().Msgf("CarWakeUp.WakeUpQueue %s", vin)
+	car, err := c.db.SelectByVIN(vin)
+	if err != nil {
+		return qs, err
+	}
+
+	// If we do not return an ICCID, we still send the message to the SMS service.
+	// The sms service will create a fake wake up message to be used on dev
+
+	smsWakeUp := sms.SendSMSRequest{
+		ICCID:       car.ICCID,
+		MessageText: ".",
+		Await:       await,
+	}
+
+	qs, err = c.sms.HandleSMSQueue(context.Background(), &smsWakeUp)
+	if err != nil {
+		logger.Error().Err(errors.WithStack(err)).Send()
+		return qs, errors.WithStack(ErrWakeUpMessageNotSent)
+	}
+
+	return
+}
+
+// The SMS service currently automatically always calls back to the attendant service, which is not great, as config updates are not sent that way
+func QueueSMSWakeUp(vin string, await bool, kafkaclient redis.Client, carDB queries.CarsInterface, sms sms.SMSServiceClient) (qs *sms.SMSQueueResponse, err error) {
+	// This new client with every request is bad. These are re-usable
+	wake := NewCarWakeUp(carDB, sms)
+	qs, err = wake.WakeUpQueue(vin, await)
+	if err != nil {
+		level := logger.Error()
+		if errors.Is(err, ErrNoICCIDForWakeUp) ||
+			errors.Is(err, ErrWakeUpMessageNotSent) {
+			level = logger.Warn()
+		}
+		logger.At(level, vin, "sms").Err(err).Send()
+	}
+	return
+}
+
+func SMSWakeUp(vin string, checkonline bool, clientPool redis.ClientPoolInterface, carDB queries.CarsInterface, sms sms.SMSServiceClient) (err error) {
+	if checkonline {
+		var online bool
+		online, err = cache.IsCarOnline(clientPool, vin)
+		if err != nil {
+			logger.At(logger.Error(), vin, "sms").Err(err).Send()
+			return
+		}
+		if online {
+			return nil
+		}
+	}
+
+	wake := NewCarWakeUp(carDB, sms)
+	err = wake.WakeUp(vin, true)
+	if err != nil {
+		level := logger.Error()
+		if errors.Is(err, ErrNoICCIDForWakeUp) ||
+			errors.Is(err, ErrWakeUpMessageNotSent) {
+			level = logger.Warn()
+		}
+		logger.At(level, vin, "sms").Err(err).Send()
+	}
+	return
+}

pkg/clickhouse/clickhouse.go

@@ -0,0 +1,257 @@
+package clickhouse
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/utils/envtool"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/pkg/errors"
+)
+
+var (
+	TIMEOUT               = envtool.GetEnvInt("CLICKHOUSE_TIMEOUT", 1)
+	MAX_CONNS             = envtool.GetEnvInt("CLICKHOUSE_MAX_CONNS", 1)
+	CLICKHOUSE_HOST       = envtool.GetEnv("CLICKHOUSE_HOST", "localhost")
+	CLICKHOUSE_PORT       = envtool.GetEnv("CLICKHOUSE_PORT", "9000")
+	CLICKHOUSE_DB         = envtool.GetEnv("CLICKHOUSE_DB", "default")
+	CLICKHOUSE_USER       = envtool.GetEnv("CLICKHOUSE_USER", "")
+	CLICKHOUSE_PASS       = envtool.GetEnv("CLICKHOUSE_PASS", "")
+	VEHICLE_FILTERS_TABLE = envtool.GetEnv("CLICKHOUSE_VEHICLE_FILTERS_TABLE", "can_filter_list_vin")
+	DEFAULT_FILTERS_TABLE = envtool.GetEnv("CLICKHOUSE_DEFAULT_FILTERS_TABLE", "can_filter_list_all")
+)
+
+func NewClient(conn ConnInterface) (ClientInterface, error) {
+	return &Client{conn: conn}, nil
+}
+
+func NewConn() (clickhouse.Conn, error) {
+	return clickhouse.Open(&clickhouse.Options{
+		Addr: []string{fmt.Sprintf("%s:%s", CLICKHOUSE_HOST, CLICKHOUSE_PORT)},
+		Auth: clickhouse.Auth{
+			Database: CLICKHOUSE_DB,
+			Username: CLICKHOUSE_USER,
+			Password: CLICKHOUSE_PASS,
+		},
+		DialTimeout:     time.Second * 60,
+		MaxOpenConns:    MAX_CONNS,
+		MaxIdleConns:    MAX_CONNS,
+		ConnMaxLifetime: 24 * time.Hour,
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+	})
+}
+
+type ClientInterface interface {
+	Select(result interface{}, query string) error
+	RetrieveDefaultFilters() ([]CANFilterSchema, error)
+	RetrieveFiltersForVehicle(vin string) ([]CANFilterSchema, error)
+	SaveDBCInfo(dbc common.DBCDesc) error
+	SaveDBCMessages(ms []common.MessageDesc) error
+	SaveDBCSignals(signals []common.SignalDesc) error
+	SelectDBCsByVersions(versions []string) ([]string, error)
+	SelectDBCSignals(dbc string, options PageQueryOptions) ([]common.SignalDescWithECU, int, error)
+	TruncateDBCDescs() error
+
+	SetConn(conn ConnInterface)
+	Exec(query string) error
+}
+
+type ConnInterface interface {
+	Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
+	PrepareBatch(ctx context.Context, query string) (driver.Batch, error)
+	AsyncInsert(ctx context.Context, query string, wait bool) error
+	QueryRow(ctx context.Context, query string, args ...interface{}) driver.Row
+	Query(ctx context.Context, query string, args ...interface{}) (driver.Rows, error)
+	Exec(ctx context.Context, query string, args ...interface{}) error
+}
+
+type Client struct {
+	conn ConnInterface
+}
+
+func (c *Client) Select(result interface{}, query string) error {
+	ctx := context.Background()
+
+	err := c.conn.Select(ctx, result, query)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+func (c *Client) RetrieveDefaultFilters() ([]CANFilterSchema, error) {
+	var result []CANFilterSchema
+
+	if err := c.Select(&result, fmt.Sprintf("SELECT ID, Period FROM %s", DEFAULT_FILTERS_TABLE)); err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+func (c *Client) RetrieveFiltersForVehicle(vin string) ([]CANFilterSchema, error) {
+	var result []CANFilterSchema
+
+	if err := c.Select(&result, fmt.Sprintf("SELECT ID, Period FROM %s WHERE VIN='%s'", VEHICLE_FILTERS_TABLE, vin)); err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+func (c *Client) SelectDBCsByVersions(versions []string) ([]string, error) {
+	if len(versions) == 0 {
+		return nil, nil
+	}
+
+	q := fmt.Sprintf("'%s'", strings.Join(versions, "','"))
+
+	var result []string
+	if err := c.Select(&result, fmt.Sprintf("SELECT dbc_name FROM dbcs where dbc_hash IN (%s)", q)); err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+func (c *Client) TruncateDBCDescs() error {
+	err := c.Exec("TRUNCATE TABLE dbc_signals_shard ON CLUSTER default")
+	if err != nil {
+		return err
+	}
+
+	err = c.Exec("TRUNCATE TABLE dbc_messages_shard ON CLUSTER default")
+	if err != nil {
+		return err
+	}
+
+	err = c.Exec("TRUNCATE TABLE dbcs_shard ON CLUSTER default")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) SaveDBCInfo(dbc common.DBCDesc) error {
+	query := fmt.Sprintf(`INSERT INTO dbcs (dbc_hash, dbc_name) VALUES ('%s', '%s')`, dbc.Hash, dbc.Name)
+	return errors.WithStack(c.conn.AsyncInsert(context.Background(), query, true))
+}
+
+func (c *Client) SaveDBCMessages(ms []common.MessageDesc) error {
+	batch, err := c.conn.PrepareBatch(context.Background(),
+		`INSERT INTO dbc_messages (dbc_hash, message_name, message_id, is_extended,
+					send_type, length, description, sender_node, cycle_time_ns, delay_time_ns, ecu_name)`)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	for _, m := range ms {
+		err = batch.Append(m.DBCHash, m.Name, m.ID, m.IsExtended, m.SendType, m.Length, m.Description,
+			m.SenderNode, m.CycleTime, m.DelayTime, m.ECUName)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	return errors.WithStack(batch.Send())
+}
+
+func (c *Client) SaveDBCSignals(signals []common.SignalDesc) error {
+	batch, err := c.conn.PrepareBatch(context.Background(),
+		`INSERT INTO dbc_signals (
+                         dbc_hash, message_id, signal_name, start,
+                         length, big_endian, signed, multiplexer, multiplexed,
+                         multiplexer_value, offset, scale, min, max, unit,
+                         description, value_descriptions, receiver_nodes, default_value, ecu_name)`)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	for _, s := range signals {
+		err = batch.Append(s.DBCHash, s.MessageID, s.Name, s.Start, s.Length, s.IsBigEndian, s.IsSigned,
+			s.IsMultiplexer, s.IsMultiplexed, s.MultiplexerValue, s.Offset, s.Scale, s.Min, s.Max,
+			s.Unit, s.Description, s.ValueDescriptions, s.ReceiverNodes, s.DefaultValue, s.ECUName)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	return batch.Send()
+}
+
+func (c *Client) SelectDBCSignals(dbc string, options PageQueryOptions) ([]common.SignalDescWithECU, int, error) {
+	var result []common.SignalDescWithECU
+	chCtx := clickhouse.Context(
+		context.Background(),
+		clickhouse.WithParameters(clickhouse.Parameters{
+			"dbc": dbc,
+
+			// we cannot use keywords like "offset" and "limit" as parameter names
+			"lim":  fmt.Sprint(options.Limit),
+			"offs": fmt.Sprint(options.Offset),
+		}))
+
+	query := CreateDBCSignalQuery(options)
+
+	if err := c.conn.Select(chCtx, &result, query); err != nil {
+		return nil, 0, errors.WithStack(err)
+	}
+
+	var count uint64
+	if err := c.conn.QueryRow(chCtx, "SELECT COUNT() FROM dbc_signals a WHERE a.dbc_hash = {dbc:String}").Scan(&count); err != nil {
+		return nil, 0, errors.WithStack(err)
+	}
+
+	return result, int(count), nil
+}
+
+func (c *Client) SetConn(conn ConnInterface) {
+	c.conn = conn
+}
+
+func (c *Client) Exec(query string) error {
+	ctx := context.Background()
+
+	err := c.conn.Exec(ctx, query)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+type CANFilterSchema struct {
+	ID     int16 `ch:"ID"`
+	Period int32 `ch:"Period"`
+}
+
+type MigrationSchema struct {
+	Version  int64  `ch:"version"`
+	Dirty    uint8  `ch:"dirty"`
+	Sequence uint64 `ch:"sequence"`
+}
+
+func CreateDBCSignalQuery(options PageQueryOptions) string {
+
+	initQuery := `select * from dbc_signals where dbc_hash = {dbc:String}`
+
+	query := initQuery
+	if options.Limit != 0 {
+		query += ` LIMIT {lim:UInt64}`
+	}
+
+	if options.Offset != 0 {
+		query += ` OFFSET {offs:UInt64}`
+	}
+
+	return query
+}

pkg/clickhouse/clickhouse_test.go

@@ -0,0 +1,125 @@
+package clickhouse_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/clickhouse"
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestRetrieveDefaultFilters(t *testing.T) {
+	filters := []clickhouse.CANFilterSchema{
+		{
+			ID:     1,
+			Period: 2,
+		},
+		{
+			ID:     3,
+			Period: 4,
+		},
+	}
+
+	conn := &clickhouse.MockConn{ExpectedResult: filters}
+	client := clickhouse.NewMockClient(conn)
+
+	defaults, err := client.RetrieveDefaultFilters()
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", nil, err)
+	}
+
+	for i := range filters {
+		if filters[i].ID != defaults[i].ID {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].ID, defaults[i].ID)
+		}
+		if filters[i].Period != defaults[i].Period {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].Period, defaults[i].Period)
+		}
+	}
+}
+
+func TestRetrieveFiltersForVehicle(t *testing.T) {
+	filters := []clickhouse.CANFilterSchema{
+		{
+			ID:     1,
+			Period: 2,
+		},
+		{
+			ID:     3,
+			Period: 4,
+		},
+	}
+
+	conn := &clickhouse.MockConn{ExpectedResult: filters}
+	client := clickhouse.NewMockClient(conn)
+
+	defaults, err := client.RetrieveFiltersForVehicle("TESTVIN123")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", nil, err)
+	}
+
+	for i := range filters {
+		if filters[i].ID != defaults[i].ID {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].ID, defaults[i].ID)
+		}
+		if filters[i].Period != defaults[i].Period {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].Period, defaults[i].Period)
+		}
+	}
+}
+
+func TestCreateDBCSignalQuery(t *testing.T) {
+
+	tests := []struct {
+		name   string
+		input  clickhouse.PageQueryOptions
+		result string
+		err    error
+	}{
+		{
+			name: "with no options",
+			input: clickhouse.PageQueryOptions{
+				Limit:  0,
+				Offset: 0,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String}`,
+			err: nil,
+		},
+		{
+			name: "with the offset option",
+			input: clickhouse.PageQueryOptions{
+				Limit:  0,
+				Offset: 10,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String} OFFSET {offs:UInt64}`,
+			err: nil,
+		},
+		{
+			name: "with the limit option",
+			input: clickhouse.PageQueryOptions{
+				Limit:  10,
+				Offset: 0,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String} LIMIT {lim:UInt64}`,
+			err: nil,
+		},
+		{
+			name: "with the offset and limit options",
+			input: clickhouse.PageQueryOptions{
+				Limit:  100,
+				Offset: 10,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String} LIMIT {lim:UInt64} OFFSET {offs:UInt64}`,
+			err: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ans := clickhouse.CreateDBCSignalQuery(test.input)
+			if ans != test.result {
+				t.Errorf("got %s, expected %s", ans, test.result)
+			}
+		})
+	}
+
+}

pkg/clickhouse/mock.go

@@ -0,0 +1,134 @@
+package clickhouse
+
+import (
+	"context"
+	"encoding/json"
+	"reflect"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"github.com/pkg/errors"
+)
+
+func NewMockClient(conn ConnInterface) ClientInterface {
+	c := &Client{}
+	c.SetConn(conn)
+	return c
+}
+
+type MockConn struct {
+	ExpectedResult   interface{}
+	PrepareBatchMock func(ctx context.Context, query string) (driver.Batch, error)
+	AsyncInsertMock  func(ctx context.Context, query string, wait bool) error
+	QueryRowtMock    func(ctx context.Context, query string, args ...interface{}) driver.Row
+	QueryMock        func(ctx context.Context, query string, args ...interface{}) (driver.Rows, error)
+	Client
+}
+
+func (c *MockConn) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	payload, err := json.Marshal(c.ExpectedResult)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	err = json.Unmarshal(payload, dest)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+func (c *MockConn) PrepareBatch(ctx context.Context, query string) (driver.Batch, error) {
+	return c.PrepareBatchMock(ctx, query)
+}
+
+func (c *MockConn) AsyncInsert(ctx context.Context, query string, wait bool) error {
+	return c.AsyncInsertMock(ctx, query, wait)
+}
+
+func (c *MockConn) QueryRow(ctx context.Context, query string, args ...interface{}) driver.Row {
+	return c.QueryRowtMock(ctx, query, args...)
+}
+
+func (c *MockConn) Query(ctx context.Context, query string, args ...interface{}) (driver.Rows, error) {
+	return RowsMock{}, nil
+}
+
+func (c *MockConn) Exec(ctx context.Context, query string, args ...interface{}) error {
+	return nil
+}
+
+type ColumnTypeMock struct{}
+
+func (c ColumnTypeMock) Name() string {
+	return "name"
+}
+
+func (c ColumnTypeMock) Nullable() bool {
+	return true
+}
+
+func (c ColumnTypeMock) ScanType() reflect.Type {
+	return reflect.TypeOf("")
+}
+
+func (c ColumnTypeMock) DatabaseTypeName() string {
+	return ""
+}
+
+type RowsMock struct {
+	RowsResult interface{}
+}
+
+func (r RowsMock) Next() bool {
+	return false
+}
+
+func (r RowsMock) Scan(dest ...interface{}) error {
+	return nil
+}
+
+func (r RowsMock) ScanStruct(dest interface{}) error {
+	return nil
+}
+
+func (r RowsMock) ColumnTypes() []driver.ColumnType {
+	return []driver.ColumnType{ColumnTypeMock{}}
+}
+
+func (r RowsMock) Totals(dest ...interface{}) error {
+	return nil
+}
+
+func (r RowsMock) Columns() []string {
+	return nil
+}
+
+func (r RowsMock) Close() error {
+	return nil
+}
+
+func (r RowsMock) Err() error {
+	return nil
+}
+
+type RowMock struct {
+	RowResult interface{}
+}
+
+func (r RowMock) Err() error {
+	return nil
+}
+
+func (r RowMock) Scan(dest ...interface{}) error {
+	if len(dest) != 0 {
+		dest[0] = r.RowResult
+	}
+
+	return nil
+}
+
+func (r RowMock) ScanStruct(dest interface{}) error {
+	return nil
+}

pkg/clickhouse/options.go

@@ -0,0 +1,31 @@
+package clickhouse
+
+import (
+	"net/http"
+
+	"fiskerinc.com/modules/validator"
+
+	"github.com/gorilla/schema"
+)
+
+const PageQueryOptionsLimitMaximum = 100
+
+type PageQueryOptions struct {
+	Limit  int `json:"limit" validate:"gte=0,lte=100"`
+	Offset int `json:"offset" validate:"gte=0"`
+}
+
+// ParsePageQuery parses PageQueryOptions from http request
+func ParsePageQuery(r *http.Request) (PageQueryOptions, error) {
+	decoder := schema.NewDecoder()
+	options := PageQueryOptions{}
+
+	decoder.SetAliasTag("json")
+	decoder.Decode(&options, r.URL.Query())
+	err := validator.ValidateStruct(options)
+	if err == nil && options.Limit == 0 {
+		options.Limit = PageQueryOptionsLimitMaximum
+	}
+
+	return options, err
+}

pkg/common/actionlogger/actionlogger.go

@@ -0,0 +1,37 @@
+package actionlogger
+
+import (
+	"time"
+
+	"fiskerinc.com/modules/common/dbbasemodel"
+	"github.com/google/uuid"
+)
+
+// Use Action Log to track actions taken against a car including remote commands, car updates etc. Mostly worried about car commands for now
+type ActionLog struct {
+	VIN            string
+	UserIdentifier string
+	//UserType       string // Not sure how different users can be identified between fisker customers and api admins
+	Action         Action // Short Hand of description
+	Description    string // JSON string explaining full action
+	// TrackingID     *uuid.UUID // If we want to log the same action as it goes through, this tracking ID will follow the same request through
+	CallLocation string // Informative field where we created this. Can use runtime reflection if we want, but is more expensive
+	dbbasemodel.DBModelBase
+}
+
+type Action string
+
+const (
+	RemoteCommand Action = "RemoteCommand"
+	CarConfigurationUpdate Action = "CarConfigurationUpdate"
+	CarUpdate Action = "CarUpdate"
+)
+
+type ActionLogFilter struct {
+	VINs    []string `pg:",array"`
+	Actions []string `pg:",array"`
+	Before  time.Time
+	After   time.Time
+	Limit   int
+	TrackingID *uuid.UUID
+}

pkg/common/add_car_request.go

@@ -0,0 +1,28 @@
+package common
+
+type AddCarRequest struct {
+	VIN         string   `json:"vin" validate:"required,vin"`
+	LogLevel    LogLevel `json:"log_level,omitempty" swaggertype:"string"`
+	CANBus      *CANBus  `json:"canbus,omitempty"`
+	IDPSEnabled bool     `json:"idps_enabled,omitempty"`
+}
+
+type UpdateCarRequest struct {
+	VIN         string   `json:"vin" validate:"required,vin"`
+	ICCID       string   `json:"iccid,omitempty" validate:"omitempty,max=50"`
+	Year        int      `json:"year,omitempty" validate:"required,gte=1000,lte=9999"`
+	Model       string   `json:"model,omitempty" validate:"required,max=256"`
+	Trim        string   `json:"trim,omitempty" validate:"required,max=256"`
+	Country     string   `json:"country,omitempty" validate:"max=256"`
+	Powertrain  string   `json:"powertrain,omitempty" validate:"max=256"`
+	Restraint   string   `json:"restraint,omitempty" validate:"max=256"`
+	BodyType    string   `json:"body_type,omitempty" validate:"max=256"`
+	LogLevel    LogLevel `json:"log_level,omitempty" swaggertype:"string"`
+	DLTEnabled  bool     `json:"dlt_enabled,omitempty" swaggertype:"boolean"`
+	DLTLevel    int      `json:"dlt_level,omitempty" validate:"oneof=0 1 2 3 4 5 6 255"`
+	CANBus      *CANBus  `json:"canbus,omitempty"`
+	IDPSEnabled bool     `json:"idps_enabled,omitempty"`
+	DebugMask   string   `json:"debug_mask,omitempty"`
+	Tags        []string `json:"tags,omitempty"`
+	SUMSVersion string   `json:"sums_version,omitempty"`
+}

pkg/common/apicalls.go

@@ -0,0 +1,26 @@
+package common
+
+import "time"
+
+type AccessType string
+
+const AccessTypeJWT = "jwt"
+const AccessTypeAPIToken = "api_token"
+
+type APICall struct {
+	// ClientID can be email or api_token.
+	ClientID string `json:"client_id" pg:"client_id"`
+
+	// Check allowed access types above.
+	AccessType string     `json:"access_type" pg:"access_type"`
+	Method     string     `json:"method" pg:"method"`
+	Endpoint   string     `json:"endpoint" pg:"endpoint"`
+	CreatedAt  *time.Time `json:"created_at" pg:"default:now()"`
+}
+
+// APICallsSearch is supposed to be used for calls only.
+type APICallsSearch struct {
+	Search string
+	From   *time.Time
+	To     *time.Time
+}

pkg/common/apitoken.go

@@ -0,0 +1,19 @@
+package common
+
+import (
+	"fmt"
+	"time"
+	"fiskerinc.com/modules/common/dbbasemodel"
+)
+
+type APIToken struct {
+	Token       string     `json:"token" validate:"required,max=1000" pg:",pk"`
+	Roles       string     `json:"roles" validate:"required,max=10000"`
+	Description string     `json:"description" validate:"required,max=1000"`
+	ExpiresAt   *time.Time `json:"expires_at" pg:"expires_at"`
+	dbbasemodel.DBModelBase
+}
+
+func (a *APIToken) String() string {
+	return fmt.Sprintf("APIToken<%s>", a.Token)
+}

pkg/common/approval_update.go

@@ -0,0 +1,32 @@
+package common
+
+func NewApprovalUpdates(cu *CarUpdate) ApprovalUpdate {
+	a := ApprovalUpdate{}
+	a.Update(cu)
+	return a
+}
+
+type ApprovalUpdate struct {
+	ID           int64  `json:"id,omitempty"`
+	VIN          string `json:"vin"`
+	Name         string `json:"name,omitempty"`
+	Version      string `json:"version,omitempty"`
+	Description  string `json:"description,omitempty"`
+	ReleaseNotes string `json:"release_notes,omitempty"`
+}
+
+func (a *ApprovalUpdate) Update(cu *CarUpdate) {
+	a.ID = cu.ID
+	a.VIN = cu.VIN
+
+	if cu.UpdateManifest == nil {
+		return
+	}
+
+	m := cu.UpdateManifest
+
+	a.Name = m.Name
+	a.Version = m.Version
+	a.Description = m.Description
+	a.ReleaseNotes = m.ReleaseNotes
+}

pkg/common/authproviders/authproviders.go

@@ -0,0 +1,12 @@
+package authproviders
+
+import "fiskerinc.com/modules/utils/envtool"
+
+const (
+	Default      = "Default" // This is for any provider
+	FiskerAD     = "Fisker"
+	FiskerQA     = "Fisker-QA"
+	FiskerAPIKey = "FiskerAPIKey"
+)
+
+var Magna = envtool.GetEnv("MAGNA_PROVIDER", "REPLACE_ME")

pkg/common/binary_hex._test.go

@@ -0,0 +1,54 @@
+package common_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/testhelper"
+)
+
+type testHexBinary struct {
+	Data *common.BinaryHex `json:"data"`
+}
+
+func TestBinaryHexMarshalJSON(t *testing.T) {
+	expected := `"0000ffff"`
+	value := common.BinaryHex{00, 00, 0xff, 0xff}
+
+	json, err := value.MarshalJSON()
+
+	testhelper.NoError(t, "MarshalJSON error", err)
+	testhelper.Equal(t, "MarshalJSON json", expected, string(json))
+}
+
+func TestBinaryHexUnmarshalJSON(t *testing.T) {
+	expected := []byte{00, 00, 0xff, 0xff}
+	value := common.BinaryHex{}
+
+	err := value.UnmarshalJSON([]byte(`"0000ffff"`))
+
+	testhelper.NoError(t, "UnmarshalJSON error", err)
+	testhelper.EqualByteArray(t, "UnmarshalJSON len", expected, value)
+}
+
+func TestBinaryHexStructMarshalJSON(t *testing.T) {
+	expected := `{"data":"0000ffff"}`
+	value := testHexBinary{Data: &common.BinaryHex{00, 00, 0xff, 0xff}}
+
+	json, err := json.Marshal(value)
+
+	testhelper.NoError(t, "MarshalJSON error", err)
+	testhelper.Equal(t, "MarshalJSON json", expected, string(json))
+}
+
+func TestBinaryHexStructUnmarshalJSON(t *testing.T) {
+	obj := testHexBinary{}
+	data := `{"data":"0e00ffff"}`
+	expected := []byte{0x0e, 00, 0xff, 0xff}
+
+	err := json.Unmarshal([]byte(data), &obj)
+
+	testhelper.NoError(t, "UnmarshalJSON error", err)
+	testhelper.EqualByteArray(t, "UnmarshalJSON json", *obj.Data, expected)
+}

pkg/common/binary_hex.go

@@ -0,0 +1,56 @@
+package common
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strings"
+)
+
+func NewBinaryHex(data []byte) BinaryHex {
+	var result BinaryHex = data
+	return result
+}
+
+type BinaryHex []byte
+
+func (bh BinaryHex) Bytes() []byte {
+	return []byte(bh)
+}
+
+func (bh *BinaryHex) SetBytes(value []byte) {
+	v := BinaryHex(value)
+	*bh = v
+}
+
+func (bh BinaryHex) String() string {
+	return hex.EncodeToString(bh)
+}
+
+// TODO Hack to render []byte as int array for the JSON RPC. Otherwise Go renders in base64
+func (bh BinaryHex) UintArray() []uint {
+	bytes := bh.Bytes()
+	result := make([]uint, len(bytes))
+	for i, d := range bytes {
+		result[i] = uint(d)
+	}
+
+	return result
+}
+
+func (bh *BinaryHex) SetHex(data string) error {
+	x, err := hex.DecodeString(strings.Trim(string(data), `"`))
+	if err != nil {
+		return err
+	}
+	*bh = x
+
+	return nil
+}
+
+func (bh BinaryHex) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, bh.String())), nil
+}
+
+func (bh *BinaryHex) UnmarshalJSON(data []byte) error {
+	return bh.SetHex(string(data))
+}

pkg/common/bulk_car_commands.go

@@ -0,0 +1,7 @@
+package common
+
+// BulkCarCommands for sending commands to multiple cars
+type BulkCarCommands struct {
+	VINs []string `json:"vins,omitempty" validate:"required,max=1000,dive,vin"`
+	RemoteCommandSource
+}

pkg/common/can.go

@@ -0,0 +1,112 @@
+package common
+
+import (
+	"time"
+
+	"fiskerinc.com/modules/grpc/kafka_grpc"
+)
+
+// CANFrame provides struct for can bus messages
+type CANFrame struct {
+	TimestampUSec int    `json:"epoch_usec" parquet:"name=epoch_usec, type=INT64"`
+	ID            int    `json:"id" parquet:"name=id, type=INT32"`
+	Data          string `json:"data" parquet:"name=data, type=BYTE_ARRAY"`
+}
+
+type CANBusMessage struct {
+	EpochUsec int        `json:"epoch_usec"`
+	Dropped   int        `json:"dropped"`
+	Filtered  int        `json:"filtered"`
+	Frames    []CANFrame `json:"frames"`
+}
+
+// CANSignal provides struct for parsed can bus messages
+type CANSignal struct {
+	VIN         string  `json:"vin"`
+	Timestamp   float64 `json:"timestamp" parquet:"name=timestamp, type=FLOAT"`
+	ID          int     `json:"id" parquet:"name=id, type=INT32"`
+	Name        string  `json:"name" parquet:"name=name, type=BYTE_ARRAY"`
+	Value       float64 `json:"value" parquet:"name=value, type=FLOAT"`
+	Description string  `json:"description,omitempty"`
+}
+
+type CANSignalExport struct {
+	VIN              string    `json:"vin"`
+	Timestamp        time.Time `json:"timestamp"`
+	Name             string    `json:"name"`
+	Value            float64   `json:"value"`
+	TimestampInMilli int64     `json:"tm"`
+}
+
+// "missing destination name \"BCM_FrntDrDoorLockSts\" in *common.CANSignal"
+type CANSignalQuery struct {
+	VIN            string   `json:"vin" validate:"required"`
+	TimestampStart float64  `json:"timestamp_start" validate:"required"`
+	TimestampEnd   float64  `json:"timestamp_end" validate:"required"`
+	Offset         int      `json:"offset"`
+	Limit          int      `json:"limit"`
+	SelectAll      bool     `json:"select_all"`
+	CanSignals     []string `json:"can_signals"`
+}
+
+type ExportCANSignalQuery struct {
+	VIN            string   `json:"vin" validate:"required"`
+	TimestampStart int64    `json:"timestamp_start" validate:"required"`
+	TimestampEnd   int64    `json:"timestamp_end" validate:"required"`
+	Offset         int      `json:"offset"`
+	Limit          int      `json:"limit"`
+	SelectAll      bool     `json:"select_all"`
+	CanSignals     []string `json:"can_signals"`
+}
+
+type CANSignalNameList struct {
+	Signal_Name string `json:"signal_name"`
+}
+
+type CANSignalData struct {
+	cansignals []CANSignal `json:"cansignals"`
+}
+
+type CANSignalBatchPayload struct {
+	Data CANSignalData `json:"data"`
+}
+
+// CANFilter provides struct for filtering can messages based on ID
+type CANFilter struct {
+	CANID    string     `json:"can_id" bson:"can_id" validate:"required,can_id"`
+	Interval *int       `json:"interval,omitempty" bson:"interval" validate:"omitempty,gte=0"`
+	EdgeMask *BinaryHex `json:"edge_mask,omitempty" bson:"edge_mask,omitempty" validate:"omitempty,lte=10000"`
+}
+
+// CANFilterWithFleet is used only for rendering vehicle's fleets' filters.
+type CANFilterWithFleet struct {
+	CANID    string     `json:"can_id" bson:"can_id" validate:"required,can_id"`
+	Interval *int       `json:"interval,omitempty" bson:"interval" validate:"gte=0"`
+	Fleet    string     `json:"fleet,omitempty" bson:"fleet,omitempty"`
+	EdgeMask *BinaryHex `json:"edge_mask,omitempty" bson:"edge_mask,omitempty" validate:"omitempty,lte=10000"`
+}
+
+func (c *CANSignalBatchPayload) ToGrpc(data []CANSignal) *kafka_grpc.GRPC_CANSignalBatchPayload {
+	grpccansignals := make([]*kafka_grpc.GRPC_CANSignal, len(data))
+
+	msg := kafka_grpc.GRPC_CANSignalData{
+		Cansignals: grpccansignals,
+	}
+
+	for i, cs := range data {
+		msg.Cansignals[i] = &kafka_grpc.GRPC_CANSignal{
+			Vin:         cs.VIN,
+			Timestamp:   cs.Timestamp,
+			Id:          int32(cs.ID),
+			Name:        cs.Name,
+			Value:       cs.Value,
+			Description: cs.Description,
+		}
+	}
+
+	batchPayload := kafka_grpc.GRPC_CANSignalBatchPayload{
+		Data: &msg,
+	}
+
+	return &batchPayload
+}

pkg/common/car.go

@@ -0,0 +1,61 @@
+package common
+
+import (
+	"fmt"
+
+	"fiskerinc.com/modules/common/dbbasemodel"
+)
+
+const (
+	InfoSourceAutoCreated = "autocreated"
+	CarSoldStatusRetailed = "Retailed"
+)
+
+type RegionCode string
+
+const (
+	US RegionCode = "US"
+	EU RegionCode = "EU"
+)
+
+// Car schema
+type Car struct {
+	VIN         string           `pg:",pk" json:"vin" validate:"required,vin"`
+	Region      RegionCode       `json:"region,omitempty"`
+	Country     string           `json:"country,omitempty" validate:"max=256"`
+	Year        int              `json:"year,omitempty" validate:"required,gte=1000,lte=9999"`
+	Model       string           `json:"model,omitempty" validate:"required,max=256"`
+	Trim        string           `json:"trim,omitempty" validate:"required,max=256"`
+	Powertrain  string           `json:"powertrain,omitempty" validate:"max=256"`
+	Restraint   string           `json:"restraint,omitempty" validate:"max=256"`
+	BodyType    string           `json:"body_type,omitempty" validate:"max=256"`
+	ECUList     string           `json:"ecu_list,omitempty"`
+	ICCID       string           `json:"iccid,omitempty" validate:"omitempty,max=50"`
+	InfoSource  string           `pg:"info_source" json:"-"`
+	Blocked     bool             `pg:"blocked" json:"-"`
+	Tags        []string         `json:"tags,omitempty" pg:"tags,array" validate:"omitempty,max=50"`
+	Drivers     []CarToDriver    `pg:"-" json:"drivers,omitempty"`
+	Manifests   []StatusManifest `pg:"-" json:"manifests,omitempty"`
+	Document    string           `pg:"-" json:"document,omitempty"`
+	SoldStatus  string           `pg:"sold_status" json:"sold_status,omitempty"`
+	SUMSVersion string           `pg:"sums_version" json:"sums_version,omitempty"`
+	OSVersion   string           `pg:"-" json:"os_version,omitempty"`
+	Flashpack   string           `json:"flashpack,omitempty"`
+	dbbasemodel.DBModelBase
+}
+
+// CarToDriver table storing cars-to-drivers
+type CarToDriver struct {
+	ID            int64          `json:"id"`
+	VIN           string         `pg:",unique:carid_driverid" json:"vin,omitempty" validate:"required,vin"`
+	DriverID      string         `pg:",unique:carid_driverid" json:"driverid,omitempty" validate:"required,max=256"`
+	DriverRole    string         `json:"role" validate:"required,max=100"`
+	BLEKey        string         `pg:"ble_key" json:"ble_key,omitempty" validation:"hexdecimal,max=32"`
+	Settings      []CarSetting   `pg:"-" json:"settings,omitempty"`
+	Subscriptions []Subscription `pg:"rel:has-many" json:"subscriptions,omitempty"`
+	dbbasemodel.DBModelBase
+}
+
+func (c Car) String() string {
+	return fmt.Sprintf("Car<%s %s %d>", c.VIN, c.Model, c.Year)
+}

pkg/common/car_command_locks.go

@@ -0,0 +1,14 @@
+package common
+
+const (
+	LockDoor   string = "lock"
+	UnlockDoor string = "unlock"
+)
+
+type CarCommandLocks struct {
+	LeftFront  string `json:"left_front,omitempty"`
+	RightFront string `json:"right_front,omitempty"`
+	LeftRear   string `json:"left_rear,omitempty"`
+	RightRear  string `json:"right_rear,omitempty"`
+	Trunk      string `json:"trunk,omitempty"`
+}

pkg/common/car_command_wins.go

@@ -0,0 +1,16 @@
+package common
+
+const (
+	OpenWindow  string = "open"
+	CloseWindow string = "close"
+)
+
+type CarCommandWindows struct {
+	LeftFront        string `json:"left_front,omitempty"`
+	RightFront       string `json:"right_front,omitempty"`
+	LeftRear         string `json:"left_rear,omitempty"`
+	RightRear        string `json:"right_rear,omitempty"`
+	LeftRearQuarter  string `json:"left_rear_quarter,omitempty"`
+	RightRearQuarter string `json:"right_rear_quarter,omitempty"`
+	RearWindshield   string `json:"rear_windshield,omitempty"`
+}

pkg/common/car_data.go

@@ -0,0 +1,76 @@
+package common
+
+import (
+	"encoding/json"
+
+	"fiskerinc.com/modules/grpc/kafka_grpc"
+	"github.com/pkg/errors"
+)
+
+type CANBusFrame struct {
+	EpochUsec int64  `json:"epoch_usec"`
+	ID        int    `json:"id"`
+	Data      string `json:"data"`
+}
+type CANBusMessageRaw struct {
+	EpochUsec int64         `json:"epoch_usec"`
+	Dropped   int           `json:"dropped"`
+	Filtered  int           `json:"filtered"`
+	Frames    []CANBusFrame `json:"frames"`
+}
+type CarDataBatchPayloadRaw struct {
+	Handler string           `json:"handler"`
+	Data    CANBusMessageRaw `json:"data"`
+	Version string           `json:"version"`
+}
+
+// CarDataBatchPayload is a payload received from T.Rex
+//
+//	it contains batches of CANMessages (can.go)
+type CarDataBatchPayload struct {
+	Handler string        `json:"handler"`
+	Data    CANBusMessage `json:"data"`
+	Version string        `json:"version"`
+}
+
+func (c *CarDataBatchPayload) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*c)
+	return data, errors.WithStack(err)
+
+}
+
+func (c *CarDataBatchPayload) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, c)
+	return errors.WithStack(err)
+}
+
+func (c *CarDataBatchPayload) ToGrpc(data MessageRawJSON, vin string) (*kafka_grpc.GRPC_BatchPayload, error) {
+	var payload CANBusMessageRaw
+	err := json.Unmarshal(data.Data, &payload)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	frames := make([]*kafka_grpc.GRPC_CANFrame, len(payload.Frames))
+	msg := kafka_grpc.GRPC_CANData{
+		EpochUsec: payload.EpochUsec,
+		Dropped:   int32(payload.Dropped),
+		Filtered:  int32(payload.Filtered),
+		Frames:    frames,
+		Vin:       vin,
+	}
+
+	for i, frame := range payload.Frames {
+		msg.Frames[i] = &kafka_grpc.GRPC_CANFrame{
+			Epoch: frame.EpochUsec,
+			ID:    int32(frame.ID),
+			Value: []byte(frame.Data),
+		}
+	}
+	batch_payload := kafka_grpc.GRPC_BatchPayload{
+		Handler: data.Handler,
+		Data:    &msg,
+		Version: data.Version,
+	}
+	return &batch_payload, nil
+}

pkg/common/car_driver.go

@@ -0,0 +1,9 @@
+package common
+
+type CarToDriverModel struct {
+	User          UserProfile    `json:"user,omitempty"`
+	DriverID      string         `json:"driver_id"`
+	Role          string         `json:"role"`
+	Settings      []CarSetting   `json:"settings"`
+	Subscriptions []Subscription `json:"subscriptions"`
+}

pkg/common/car_ecu.go

@@ -0,0 +1,102 @@
+package common
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+
+	"fiskerinc.com/modules/common/dbbasemodel"
+)
+
+type CarECU struct {
+	VIN               string `json:"vin,omitempty" pg:",unique:vin_ecu" validate:"required,vin"`
+	ECU               string `json:"ecu" pg:",unique:vin_ecu" validate:"required,max=100"`
+	Version           string `json:"sw_version" validate:"max=255"`
+	SerialNumber      string `json:"serial_number,omitempty" validate:"max=1024"`
+	HWVersion         string `json:"hw_version,omitempty" validate:"max=1024"`
+	BootLoaderVersion string `json:"boot_loader_version,omitempty" validate:"max=1024"`
+	Fingerprint       string `json:"fingerprint,omitempty" validate:"max=1024"`
+	// cloud/attendant/handlers/car_update_state.go JSON message config was renamed to code_data_string
+	Config            string `json:"code_data_string,omitempty" pg:"code_data_string" validate:"max=2048"` // config was renamed to code_data_string
+	Vendor            string `json:"vendor,omitempty" validate:"max=1024"`
+	SupplierSWVersion string `json:"supplier_sw_version,omitempty" validate:"max=1024"`
+	Epoch_usec        int64  `json:"epoch_usec" pg:"epoch_usec"`
+	ASSYNumber        string `json:"assy_number,omitempty" pg:"assy_number"`
+	dbbasemodel.DBModelBase
+}
+
+func (c *CarECU) CacheKey() string {
+	return fmt.Sprintf("%s:%s", c.VIN, c.ECU)
+}
+
+func (c *CarECU) HashValues() string {
+	data := []byte(fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", c.Version, c.SerialNumber, c.HWVersion, c.BootLoaderVersion, c.Fingerprint, c.Config, c.Vendor, c.SupplierSWVersion, c.ASSYNumber))
+	hash := sha256.Sum256(data)
+	signature := hex.EncodeToString(hash[:])
+	return signature
+}
+
+// Ensure we always have the correct car_ecu name for OTA
+func (c *CarECU) TransformECUName() {
+	replacement, ok := OTAUpdateECUReplacement[c.ECU]
+	if ok {
+		c.ECU = replacement
+	}
+}
+
+type CarECUFilter struct {
+	VIN                  string
+	ECUs                 []string
+	Search               string
+	Unique               bool
+	FlashPackNumberExist bool
+	HWVersionRequired    bool // Ensure that the hw_version has a value. This may not retrieve the latest entry for the ecu, but until we find out why hw_versions are being inserted as empty
+}
+
+type CarFlashpackVersion struct {
+	Flashpack     string `json:"flashpack" validate:"required,numeric"`
+	CarModel      string `json:"car_model" validate:"required"`
+	CarTrim       string `json:"car_trim" validate:"required"`
+	CarYear       int    `json:"car_year" validate:"required"`
+	CarECUName    string `json:"car_ecu_name" validate:"required"`
+	CarECUVersion string `json:"car_ecu_version" validate:"required"`
+	dbbasemodel.DBModelBase
+}
+
+type CarECUVersion struct {
+	CarECUName    string `json:"car_ecu_name"`
+	CarECUVersion string `json:"car_ecu_version"`
+}
+
+type CarFlashpackVersionRequest struct {
+	Flashpack string `json:"flashpack" validate:"required,numeric"`
+	CarModel  string `json:"car_model" validate:"required"`
+	CarTrim   string `json:"car_trim" validate:"required"`
+	CarYear   int    `json:"car_year" validate:"required"`
+}
+
+type ECUVersionRequest struct {
+	CarECUName    string `json:"car_ecu_name" validate:"required"`
+	CarECUVersion string `json:"car_ecu_version" validate:"required"`
+}
+
+type CarFlashpackVersionAddRequest struct {
+	Flashpack   string              `json:"flashpack" validate:"required,numeric"`
+	CarModel    string              `json:"car_model" validate:"required"`
+	CarTrim     string              `json:"car_trim" validate:"required"`
+	CarYear     int                 `json:"car_year" validate:"required"`
+	ECUVersions []ECUVersionRequest `json:"ecu_versions" validate:"required"`
+}
+
+type CarFlashpackVersionResponse struct {
+	Flashpack string `json:"flashpack"`
+	CarModel  string `json:"car_model"`
+	CarTrim   string `json:"car_trim"`
+	CarYear   int    `json:"car_year"`
+}
+
+type CarFlashpackVersionInfoResponse struct {
+	Flashpack     string          `json:"flashpack"`
+	NextFlashpack string          `json:"next_flashpack"`
+	ECUVersions   []CarECUVersion `json:"ecu_versions"`
+}

pkg/common/car_location.go

@@ -0,0 +1,41 @@
+package common
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
+// Location represents the location state of the car
+type Location struct {
+	Altitude  float64 `json:"altitude" redis:"altitude"`
+	Longitude float64 `json:"longitude" redis:"longitude"`
+	Latitude  float64 `json:"latitude" redis:"latitude"`
+	Heading   float64 `json:"-" redis:"heading"`
+}
+
+func (l *Location) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*l)
+	return data, errors.WithStack(err)
+}
+
+func (l *Location) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, l)
+	return errors.WithStack(err)
+}
+
+func (it *Location) MarshalBinary() ([]byte, error) {
+	return json.Marshal(it)
+}
+
+func (it *Location) UnmarshalBinary(data []byte) error {
+	return json.Unmarshal(data, it)
+}
+
+type JSONCarLocation struct {
+	VIN       string  `json:"vin"`
+	Altitude  float64 `json:"altitude"`
+	Latitude  float64 `json:"latitude"`
+	Longitude float64 `json:"longitude"`
+	Heading   float64 `json:"heading"`
+}

pkg/common/car_search.go

@@ -0,0 +1,15 @@
+package common
+
+type CarSearch struct {
+	Search string `json:"search" validate:"max=1024"`
+	VINs   string `json:"vins" validate:"omitempty"`
+	Online *CarOnlineFilter
+	NoEU   bool `json:"no_eu", validate:"omitempty"`
+	Car
+}
+
+type CarOnlineFilter struct {
+	Online     *bool
+	HMI        *bool
+	VINsOnline []string
+}

pkg/common/car_setting.go

@@ -0,0 +1,80 @@
+package common
+
+import (
+	"fiskerinc.com/modules/common/dbbasemodel"
+)
+
+type CarSetting struct {
+	VIN      string `pg:"vin" json:"-"`
+	DriverID string `pg:"driver_id" json:"-"`
+	Name     string `pg:",pk" json:"name"`
+	Value    string `json:"value"`
+	Type     string `json:"type"`
+	dbbasemodel.DBModelBase
+}
+
+type MobileSettingsUpdate struct {
+	VIN      string       `json:"vin"`
+	Settings []CarSetting `json:"settings"`
+}
+
+type HMISettingsUpdate struct {
+	DriverID string       `json:"driver_id"`
+	Settings []CarSetting `json:"settings"`
+}
+
+/*
+It would be nice if we could do this easily, but requires a cast to string which isn't so nice
+type CarSettingEnum string
+*/
+const (
+	SEQUENCE_NUMBER      string = "SEQUENCE_NUMBER"
+	BODY_COLOR           string = "BODY_COLOR"
+	DELIVERY_DESTINATION string = "DELIVERY_DESTINATION"
+)
+
+// Take in the feature codes for the car, and convert it to body color string, will probably change
+func FeatureCodeToBodyColor(VehicleFeatures []FeatureCodes) (bodyColor string) {
+	var colorCode string
+	for x := range VehicleFeatures {
+		if VehicleFeatures[x].FamilyCode == "0103" {
+			colorCode = VehicleFeatures[x].FeatureCode
+			break
+		}
+	}
+
+	switch colorCode {
+	case "010300":
+		return "PRIMERED"
+	case "010301":
+		return "SOLID_WHITE"
+	case "010302":
+		return "SOLID_BLACK"
+	case "010303":
+		return "BLUE_GREY_MET"
+	case "010304":
+		return "MID_BLUE_GLOSS"
+	case "010305":
+		return "MID_BLUE_MATTE"
+	case "010306":
+		return "VIVID_BLUE"
+	case "010307":
+		return "SPE_COOL_SILVER"
+	case "010308":
+		return "STEALTH_GREEN"
+	case "010309":
+		return "VIVID_ORANGE"
+	case "010310":
+		return "EARTH_COPPER"
+	case "010311":
+		return "METALLIC_BLUE_BLACK"
+	case "010312":
+		return "WHITE_PEARL"
+	case "010313":
+		return "STEALTH_GREEN_GLOSS"
+	case "010314":
+		return "SOLID_RED"
+	default:
+		return "MISSING_COLOR"
+	}
+}

pkg/common/car_state.go

@@ -0,0 +1,552 @@
+package common
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type CarState struct {
+	Online                bool                `json:"online"`
+	OnlineHMI             bool                `json:"online_hmi"`
+	Battery               *Battery            `json:"battery,omitempty"`
+	MaxRange              *MaxRange           `json:"max_range,omitempty"`
+	Doors                 *Doors              `json:"doors,omitempty"`
+	Location              *Location           `json:"location,omitempty"`
+	Locks                 *Locks              `json:"door_locks,omitempty"`
+	Windows               *Windows            `json:"windows,omitempty"`
+	MiscWindows           *MiscWindows        `json:"misc_windows,omitempty"`
+	Sunroof               *Sunroof            `json:"sunroof,omitempty"`
+	CabinClimate          *CabinClimate       `json:"cabin_climate,omitempty"`
+	RearDefrost           *RearDefrost        `json:"rear_defrost,omitempty"`
+	DriverSeatHeat        *DriverSeatHeat     `json:"driver_seat_heat,omitempty"`
+	PassengerSeatHeat     *PassengerSeatHeat  `json:"passenger_seat_heat,omitempty"`
+	SteeringWheelHeat     *SteeringWheelHeat  `json:"steering_wheel_heat,omitempty"`
+	AmbientTemperature    *AmbientTemperature `json:"ambient_temperature,omitempty"`
+	CellTemperature       *CellTemperature    `json:"cell_temp,omitempty"`
+	VehicleSpeed          *VehicleSpeed       `json:"vehicle_speed,omitempty"`
+	VCU0x260              *VCU0x260Descriptor `json:"vcu0x260,omitempty"`
+	ChargingMetrics       *VCUChargingMetrics `json:"charging_metrics,omitempty"`
+	Gear                  *Gear               `json:"gear,omitempty"`
+	StateOfCharge         *StateOfCharge      `json:"state_of_charge,omitempty"`
+	TRexVersion           string              `json:"trex_version,omitempty"`
+	DBCVersion            string              `json:"dbc_version,omitempty"`
+	IP                    string              `json:"ip,omitempty"`
+	UpdatedAt             *time.Time          `json:"updated,omitempty"`
+	SafeState             *SafeState          `json:"safe_state,omitempty"`
+	DriverOccupySeatState *int                `json:"driver_occupy_seat_state,omitempty"`
+	PowerMode             *int                `json:"power_mode,omitempty"`
+	ChargingStatus        *int                `json:"charging_status,omitempty"`
+	VehicleReadyState     *VehicleReadyState  `json:"vehicle_ready_state,omitempty"`
+	Battery12V            *Battery12V         `json:"battery_12v,omitempty"` // maybe make it Battery12v
+	ExpandedSignals       *ExpandedSignals    `json:"expanded_signals,omitempty"`
+}
+
+func (c *CarState) GetBattery() *Battery {
+	if c.Battery == nil {
+		c.Battery = &Battery{}
+	}
+
+	return c.Battery
+}
+
+func (c *CarState) GetMaxRange() *MaxRange {
+	if c.MaxRange == nil {
+		c.MaxRange = &MaxRange{}
+	}
+
+	return c.MaxRange
+}
+
+func (c *CarState) GetDoors() *Doors {
+	if c.Doors == nil {
+		c.Doors = &Doors{}
+	}
+
+	return c.Doors
+}
+
+func (c *CarState) UpdateLocation(value []byte) error {
+	loc := Location{}
+	err := loc.Unmarshal(value)
+	if err != nil {
+		return err
+	}
+
+	location := c.GetLocation()
+	location.Latitude = loc.Latitude
+	location.Longitude = loc.Longitude
+
+	// Altitude could have already been set from GPS_ALTITUDE so do not overwrite unless it is a non-zero
+	if loc.Altitude != 0 {
+		location.Altitude = loc.Altitude
+	}
+
+	return nil
+}
+
+func (c *CarState) GetLocation() *Location {
+	if c.Location == nil {
+		c.Location = &Location{}
+	}
+
+	return c.Location
+}
+
+func (c *CarState) GetLocks() *Locks {
+	if c.Locks == nil {
+		c.Locks = &Locks{}
+	}
+
+	return c.Locks
+}
+
+func (c *CarState) GetWindows() *Windows {
+	if c.Windows == nil {
+		c.Windows = &Windows{}
+	}
+
+	return c.Windows
+}
+
+func (c *CarState) GetMiscWindows() *MiscWindows {
+	if c.MiscWindows == nil {
+		c.MiscWindows = &MiscWindows{}
+	}
+
+	return c.MiscWindows
+}
+
+func (c *CarState) GetSunroof() *Sunroof {
+	if c.Sunroof == nil {
+		c.Sunroof = &Sunroof{}
+	}
+
+	return c.Sunroof
+}
+
+func (c *CarState) GetCabinClimate() *CabinClimate {
+	if c.CabinClimate == nil {
+		c.CabinClimate = &CabinClimate{}
+	}
+
+	return c.CabinClimate
+}
+
+func (c *CarState) GetRearDefrost() *RearDefrost {
+	if c.RearDefrost == nil {
+		c.RearDefrost = &RearDefrost{}
+	}
+
+	return c.RearDefrost
+}
+
+func (c *CarState) GetDriverSeatHeat() *DriverSeatHeat {
+	if c.DriverSeatHeat == nil {
+		c.DriverSeatHeat = &DriverSeatHeat{}
+	}
+
+	return c.DriverSeatHeat
+}
+
+func (c *CarState) GetPassengerSeatHeat() *PassengerSeatHeat {
+	if c.PassengerSeatHeat == nil {
+		c.PassengerSeatHeat = &PassengerSeatHeat{}
+	}
+
+	return c.PassengerSeatHeat
+}
+
+func (c *CarState) GetSteeringWheelHeat() *SteeringWheelHeat {
+	if c.SteeringWheelHeat == nil {
+		c.SteeringWheelHeat = &SteeringWheelHeat{}
+	}
+
+	return c.SteeringWheelHeat
+}
+
+func (c *CarState) GetAmbientTemperature() *AmbientTemperature {
+	if c.AmbientTemperature == nil {
+		c.AmbientTemperature = &AmbientTemperature{}
+	}
+
+	return c.AmbientTemperature
+}
+
+func (c *CarState) GetCellTemperature() *CellTemperature {
+	if c.CellTemperature == nil {
+		c.CellTemperature = &CellTemperature{}
+	}
+
+	return c.CellTemperature
+}
+
+func (c *CarState) GetVCU0x260() *VCU0x260Descriptor {
+	if c.VCU0x260 == nil {
+		c.VCU0x260 = &VCU0x260Descriptor{}
+	}
+
+	return c.VCU0x260
+}
+
+func (c *CarState) GetChargingMetrics() *VCUChargingMetrics {
+	if c.ChargingMetrics == nil {
+		c.ChargingMetrics = &VCUChargingMetrics{}
+	}
+
+	return c.ChargingMetrics
+}
+
+func (c *CarState) GetGear() *Gear {
+	if c.Gear == nil {
+		c.Gear = &Gear{}
+	}
+
+	return c.Gear
+}
+
+func (c *CarState) GetStateOfCharge() *StateOfCharge {
+	if c.StateOfCharge == nil {
+		c.StateOfCharge = &StateOfCharge{}
+	}
+
+	return c.StateOfCharge
+}
+
+func (c *CarState) GetVehicleSpeed() *VehicleSpeed {
+	if c.VehicleSpeed == nil {
+		c.VehicleSpeed = &VehicleSpeed{}
+	}
+
+	return c.VehicleSpeed
+}
+
+// Battery represents the battery state of the car
+type Battery struct {
+	Percent              int     `json:"percent" redis:"percent"`
+	TotalMileageOdometer int     `json:"total_mileage_odometer"`
+	BatteryVoltage       float64 `json:"battery_voltage"` // The fact that this is called battery voltage is really dumb. Its the 12 volt battery, not the high voltage one
+}
+
+func (b *Battery) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*b)
+	return data, errors.WithStack(err)
+}
+
+func (b *Battery) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, b)
+	return errors.WithStack(err)
+}
+
+// StateOfCharge represents the battery state of charge
+type StateOfCharge struct {
+	Usable int `json:"usable"`
+	Health int `json:"health"`
+}
+
+type SafeState struct {
+	VehicleSafeState      bool `json:"vehicle_safe_state" redis:"vehicle_safe_state"`
+	VCUSafeState          bool `json:"vcu_safe_state" redis:"vcu_safe_state"`
+	MCUFrontSafeState     bool `json:"mcu_front_safe_state" redis:"mcu_front_safe_state"`
+	MCURearSafeState      bool `json:"mcu_rear_safe_state" redis:"mcu_rear_safe_state"`
+	MCURearDecoupState    bool `json:"mcu_rear_decoup_state" redis:"mcu_rear_decoup_state"`
+	MCUFrontInverterError bool `json:"mcu_front_inverter_error" redis:"mcu_front_inverter_error"`
+	MCURearInverterError  bool `json:"mcu_rear_inverter_error" redis:"mcu_rear_inverter_error"`
+}
+
+func (s *SafeState) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*s)
+	return data, errors.WithStack(err)
+}
+
+func (s *SafeState) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, s)
+	return errors.WithStack(err)
+}
+
+func (c *CarState) GetSafeState() *SafeState {
+	if c.SafeState == nil {
+		c.SafeState = &SafeState{}
+	}
+
+	return c.SafeState
+}
+
+// MaxRange represents the predicted max range of the car
+type MaxRange struct {
+	MaxMiles int `json:"max_miles" redis:"max_miles"`
+}
+
+type VehicleReadyState struct {
+	IsVehicleReady bool `json:"is_vehicle_ready" redis:"is_vehicle_ready"`
+}
+
+func (c *CarState) GetVehicleReadyState() *VehicleReadyState {
+	if c.VehicleReadyState == nil {
+		c.VehicleReadyState = &VehicleReadyState{}
+	}
+	return c.VehicleReadyState
+}
+
+func (b *MaxRange) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*b)
+	return data, errors.WithStack(err)
+}
+
+func (b *MaxRange) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, b)
+	return errors.WithStack(err)
+}
+
+// Doors represents the doors state of the car
+// false means closed, true means open
+type Doors struct {
+	Hood       bool `json:"hood" redis:"hood"`
+	LeftFront  bool `json:"left_front" redis:"left_front"`
+	LeftRear   bool `json:"left_rear" redis:"left_rear"`
+	RightFront bool `json:"right_front" redis:"right_front"`
+	RightRear  bool `json:"right_rear" redis:"right_rear"`
+	Trunk      bool `json:"trunk" redis:"trunk"`
+}
+
+func (d *Doors) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*d)
+	return data, errors.WithStack(err)
+}
+
+func (d *Doors) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, d)
+	return errors.WithStack(err)
+}
+
+// Locks represents the lock state of the car
+type Locks struct {
+	Driver bool `json:"driver" redis:"driver"`
+	All    bool `json:"all" redis:"all"`
+}
+
+func (l *Locks) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*l)
+	return data, errors.WithStack(err)
+}
+
+func (l *Locks) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, l)
+	return errors.WithStack(err)
+}
+
+// Windows represents the windows state of the car
+//
+//	value is a percentage 0-100 in increments of 0.5
+type Windows struct {
+	LeftFront  int `json:"left_front" redis:"left_front"`
+	LeftRear   int `json:"left_rear" redis:"left_rear"`
+	RightFront int `json:"right_front" redis:"right_front"`
+	RightRear  int `json:"right_rear" redis:"right_rear"`
+}
+
+func (w *Windows) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *Windows) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+// MiscWindows represents the windows state of the car for misc windows (left / right rear quarter and rear windshield)
+// value is a percentage 0-100 in increments of 0.5
+type MiscWindows struct {
+	LeftRearQuarter  int `json:"left_rear_quarter" redis:"left_rear_quarter"`
+	RightRearQuarter int `json:"right_rear_quarter" redis:"right_rear_quarter"`
+	RearWindshield   int `json:"rear_windshield" redis:"rear_windshield"`
+}
+
+func (w *MiscWindows) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *MiscWindows) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type Sunroof struct {
+	Sunroof int `json:"sunroof" redis:"sunroof"`
+}
+
+func (s *Sunroof) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*s)
+	return data, errors.WithStack(err)
+}
+
+func (s *Sunroof) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, s)
+	return errors.WithStack(err)
+}
+
+type CabinClimate struct {
+	CabinTemperature    int `json:"cabin_temperature" redis:"cabin_temperature"`
+	InternalTemperature int `json:"internal_temperature" redis:"internal_temperature"`
+}
+
+func (w *CabinClimate) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *CabinClimate) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type RearDefrost struct {
+	On bool `json:"on" redis:"on"`
+}
+
+func (w *RearDefrost) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *RearDefrost) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type DriverSeatHeat struct {
+	Level int `json:"level" redis:"level"`
+}
+
+func (w *DriverSeatHeat) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *DriverSeatHeat) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type PassengerSeatHeat struct {
+	Level int `json:"level" redis:"level"`
+}
+
+func (w *PassengerSeatHeat) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *PassengerSeatHeat) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type SteeringWheelHeat struct {
+	On bool `json:"on" redis:"on"`
+}
+
+func (w *SteeringWheelHeat) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *SteeringWheelHeat) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type AmbientTemperature struct {
+	Temperature int `json:"temperature" redis:"temperature"`
+}
+
+func (w *AmbientTemperature) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *AmbientTemperature) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type VehicleSpeed struct {
+	Speed float64 `json:"speed" redis:"speed"`
+}
+
+func (w *VehicleSpeed) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *VehicleSpeed) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}
+
+type CellTemperature struct {
+	AvgBatteryTemp int `json:"avg_battery_temp"`
+}
+
+type VCU0x260Descriptor struct {
+	ChargeType string `json:"charge_type"`
+}
+
+type VCUChargingMetrics struct {
+	RemainingChargingTime     int `json:"remaining_charging_time"`
+	RemainingChargingTimeFull int `json:"remaining_charging_time_full"`
+}
+
+type Gear struct {
+	InPark bool `json:"in_park"`
+	Immobilizer string `json:"immobilizer,omitempty"`
+}
+
+type Battery12V struct {
+	IBS_BatteryVoltage *float64 `json:"voltage,omitempty"`        // 12 Volt battery voltage
+	IBS_StateOfCharge  *float64 `json:"percent_charge,omitempty"` // Percentages of the voltage out of about 15.5 Volts
+	IBS_StateOfHealth  *int     `json:"health,omitempty"`         // estimated health of the 12v battery
+}
+
+func (c *CarState) GetBattery12V() *Battery12V {
+	if c.Battery12V == nil {
+		c.Battery12V = &Battery12V{}
+	}
+
+	return c.Battery12V
+}
+
+type ExpandedSignals struct {
+	// IBS_SOCUpperTolerance *float64 //unconfirmed
+	// IBS_SOCLowerTolerance *float64 //unconfirmed
+	IBS_NominalCapacity   *int     `json:",omitempty"`
+	IBS_AvailableCapacity *int     `json:",omitempty"`
+	BCM_TotMilg_ODO       *float64 `json:",omitempty"`
+	BMS_SwVersS           *int     `json:",omitempty"`
+	BMS_SwVersM           *int     `json:",omitempty"`
+	BMS_SwVers            *int     `json:",omitempty"`
+	BMS_AccueDchaTotAh    *int     `json:",omitempty"`
+	BMS_AccueChrgTotAh    *int     `json:",omitempty"`
+}
+
+func (c *CarState) GetExpandedSignals() *ExpandedSignals {
+	if c.ExpandedSignals == nil {
+		c.ExpandedSignals = &ExpandedSignals{}
+	}
+
+	return c.ExpandedSignals
+}
+
+// I am quite certain there is no reason to have these custom marshalers, but want to keep with the current form incase of unexpected side effects
+func (w *ExpandedSignals) Marshal() ([]byte, error) {
+	data, err := json.Marshal(*w)
+	return data, errors.WithStack(err)
+}
+
+func (w *ExpandedSignals) Unmarshal(data []byte) error {
+	err := json.Unmarshal(data, w)
+	return errors.WithStack(err)
+}

pkg/common/car_state_al.go

@@ -0,0 +1,16 @@
+package common
+
+// CarStateAL builds on top of the normal car state, and includes the parsing of a few additional fields
+type CarStateAL struct {
+	*CarState
+	PKCVersion string `json:"pkc_version"`
+	SumsVersion string `json:"sums_version"`
+	OSVersion string `json:"os_version"`
+}
+
+type CarPKCOSVersion struct {
+	Vin         string
+	PKCVersion  string
+	SumsVersion string
+	OSVersion   string
+}

pkg/common/car_state_towman.go

@@ -0,0 +1,33 @@
+package common
+
+type TowmanDigitalTwin struct {
+	Online   bool           `json:"online"`
+	Location *Location      `json:"location"`
+	Gear     *Gear          `json:"gear"`
+	Charging *bool `json:"charging"`
+}
+
+func (c *TowmanDigitalTwin) GetLocation() *Location {
+	if c.Location == nil {
+		c.Location = &Location{}
+	}
+
+	return c.Location
+}
+
+func (c *TowmanDigitalTwin) GetGear() *Gear {
+	if c.Gear == nil {
+		c.Gear = &Gear{}
+	}
+
+	return c.Gear
+}
+
+func (c *TowmanDigitalTwin) GetCharging() *bool {
+	if c.Charging == nil {
+		temp := false
+		c.Charging = &temp
+	}
+
+	return c.Charging
+}

pkg/common/car_state_update.go

@@ -0,0 +1,5 @@
+package common
+
+type CarStateUpdate struct {
+	ECUs map[string]CarECU `json:"ecus" validate:"required,min=1"`
+}

pkg/common/car_update.go

@@ -0,0 +1,37 @@
+package common
+
+import (
+	"fmt"
+	"fiskerinc.com/modules/common/dbbasemodel"
+)
+
+// CarUpdate schema
+type CarUpdate struct {
+	ID               int64           `json:"id,omitempty"`
+	VIN              string          `pg:",unique:vin_update_manifest" json:"vin" validate:"required,max=17"`
+	UpdateManifestID int64           `pg:",unique:vin_update_manifest" json:"manifest_id,omitempty" validate:"required"`
+	Status           string          `pg:"default:'pending'" json:"status,omitempty" validate:"max=100"`
+	ErrorCode        int             `json:"err,omitempty"`
+	Info             string          `json:"info,omitempty" pg:"info" validate:"max=1000"`
+	Username         string          `json:"username,omitempty" validate:"required"`
+	UpdateManifest   *UpdateManifest `pg:"rel:has-one" json:"updatemanifest,omitempty"`
+	UpdateSource     string
+	dbbasemodel.DBModelBase
+}
+
+func (cu CarUpdate) String() string {
+	return fmt.Sprintf("CarUpdate<%d %d %s %s>", cu.ID, cu.UpdateManifestID, cu.VIN, cu.Status)
+}
+
+func (cu *CarUpdate) Scrub() {
+	cu.UpdateManifestID = 0
+	cu.UpdateManifest = nil
+	cu.CreatedAt = nil
+	cu.UpdatedAt = nil
+}
+
+const (
+	UPDATE_SOURCE_OTA        = "OTA"        // The cloud has deployed this update
+	UPDATE_SOURCE_AFTERSALES = "AFTERSALES" // An update generated to be sent by aftersales.
+	UPDATE_SOURCE_FLASHPACK  = "FLASHPACK"
+)

pkg/common/car_update_progress.go

@@ -0,0 +1,29 @@
+package common
+
+// CarUpdateProgress represents multi-file update download progress
+// If you change this structure and it relevant database entry, please update LogStatusIfNotARepeat
+// cloud/modules_go/db/queries/carupdates.go:194
+type CarUpdateProgress struct {
+	FileCurrent    uint64 `json:"file_current" redis:"file_size"`
+	FileTotal      uint64 `json:"file_total" redis:"file_total"`
+	PackageCurrent uint64 `json:"package_current" redis:"current_size"`
+	PackageTotal   uint64 `json:"package_total" redis:"total_size"`
+	InstalledFiles int    `json:"installed" redis:"installed"`
+	TotalFiles     int    `json:"total_files" redis:"total_files"`
+	CarUpdateID    int64  `json:"car_update_id" redis:"id"`
+	ECU            string `json:"ecu" redis:"ecu" validate:"max=100"`
+	Status         string `json:"msg" redis:"status,omitempty" validate:"max=1000"`
+	Info           string `json:"extra_info,omitempty" redis:"info,omitempty" validate:"max=1000"`
+	ErrorCode      int    `json:"err" redis:"errorcode"`
+}
+
+func (cu *CarUpdateProgress) Combine(status *CarUpdateProgress) {
+	if status == nil {
+		return
+	}
+
+	cu.PackageCurrent += status.PackageCurrent
+	cu.PackageTotal += status.PackageTotal
+	cu.InstalledFiles += status.InstalledFiles
+	cu.TotalFiles += status.TotalFiles
+}

pkg/common/car_update_progress_test.go

@@ -0,0 +1,37 @@
+package common_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestCarUpdateProgressCombine(t *testing.T) {
+	status1 := common.CarUpdateProgress{
+		InstalledFiles: 0,
+		TotalFiles:     1,
+		PackageCurrent: 100,
+		PackageTotal:   200,
+	}
+	status2 := common.CarUpdateProgress{
+		InstalledFiles: 1,
+		TotalFiles:     2,
+		PackageCurrent: 200,
+		PackageTotal:   300,
+	}
+
+	status1.Combine(&status2)
+	if status1.InstalledFiles != 1 {
+		t.Errorf(testhelper.TestErrorTemplate, "InstalledFiles", 1, status1.InstalledFiles)
+	}
+	if status1.TotalFiles != 3 {
+		t.Errorf(testhelper.TestErrorTemplate, "TotalFiles", 3, status1.TotalFiles)
+	}
+	if status1.PackageCurrent != 300 {
+		t.Errorf(testhelper.TestErrorTemplate, "PackageCurrent", 300, status1.PackageCurrent)
+	}
+	if status1.PackageTotal != 500 {
+		t.Errorf(testhelper.TestErrorTemplate, "PackageTotal", 500, status1.PackageTotal)
+	}
+}

pkg/common/car_update_status.go

@@ -0,0 +1,17 @@
+package common
+
+import (
+	"fiskerinc.com/modules/common/dbbasemodel"
+)
+
+// CarUpdateStatus database model for logging history of car updates
+// If this model is changed for the database, please update LogStatusIfNotARepeat
+// cloud/modules_go/db/queries/carupdates.go:194
+type CarUpdateStatus struct {
+	ID          int64  `json:"id" pg:",pk"`
+	CarUpdateID int64  `json:"carupdate_id"`
+	Status      string `json:"status" validate:"max=100"`
+	ErrorCode   int    `json:"error_code"`
+	Info        string `json:"info,omitempty" pg:"info" validate:"max=1000"`
+	dbbasemodel.DBModelBase
+}

pkg/common/car_version_log.go

@@ -0,0 +1,20 @@
+package common
+
+import "time"
+
+type VersionSource string
+
+const (
+	DBCVersionSource       VersionSource = "DBC"
+	TREXVersionSource      VersionSource = "TREX"
+	FlashpackVersionSource VersionSource = "Flashpack"
+)
+
+// CarVersionLogs is used for logging dbc version changes to DB.
+type CarVersionLogs struct {
+	ID            int64         `json:"id" pg:"id"`
+	VIN           string        `json:"vin" pg:"vin"`
+	VersionSource VersionSource `json:"version_source" pg:"version_source"`
+	Version       string        `json:"version" pg:"version"`
+	CreatedAt     *time.Time    `json:"created_at" pg:"created_at"`
+}

pkg/common/carupdatestatus/carupdatestatus.go

@@ -0,0 +1,60 @@
+package carupdatestatus
+
+const (
+	ManifestReceived            = "manifest_received"
+	ManifestAccepted            = "manifest_accepted"
+	ManifestRejected            = "manifest_rejected"
+	ManifestCancelPending       = "manifest_cancel_pending"
+	ManifestCancelReceived      = "manifest_cancel_received"
+	ManifestCancelAccepted      = "manifest_cancel_accepted"
+	ManifestCancelRejected      = "manifest_cancel_rejected"
+	ManifestValidationSucceeded = "manifest_validation_succeeded"
+	ManifestValidationFailed    = "manifest_validation_failed"
+	DownloadStarted             = "download_started"
+	Downloading                 = "downloading"
+	DownloadCompleted           = "download_completed"
+	DownloadFailed              = "download_failed"
+	InstallApprovalAwait        = "install_approval_await"
+	InstallApprovalReceived     = "install_approval_received"
+	InstallStarted              = "install_started"
+	Installing                  = "installing"
+	InstallSucceeded            = "install_succeeded"
+	InstallFailed               = "install_failed"
+	RollbackStarted             = "rollback_started"
+	RollbackSucceeded           = "rollback_succeeded"
+	RollbackFailed              = "rollback_failed"
+	CleanupSucceeded            = "cleanup_succeeded"
+	CleanupFailed               = "cleanup_failed"
+	ManifestError               = "manifest_error"
+	ManifestRollback            = "manifest_rollback"
+	ManifestSucceeded           = "manifest_succeeded"
+	ManifestCanceled            = "manifest_canceled"
+	ManifestPending             = "manifest_pending"
+	Pending                     = "pending"
+	Sent                        = "sent"
+	RequirementsFailed          = "requirements_failed"
+	RequirementsAwait           = "requirements_await"
+	InstallScheduled            = "install_scheduled"
+	InitialFlashPack            = "initial_flashpack_install"
+)
+
+// These final update statuses are ones that we will want to filter out
+var FINAL_UPDATE_STATUS = []string{
+	ManifestSucceeded,
+	ManifestCanceled,
+	ManifestError,
+	DownloadFailed,
+	ManifestCancelPending,
+	RollbackSucceeded,
+	ManifestRejected,
+	RollbackFailed,
+	CleanupSucceeded,
+}
+
+var NoRepeatUpdateStatus = map[string]struct{}{}
+
+func init() {
+	for _, status := range FINAL_UPDATE_STATUS {
+		NoRepeatUpdateStatus[status] = struct{}{}
+	}
+}

pkg/common/certificate.go

@@ -0,0 +1,57 @@
+package common
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"time"
+
+	"fiskerinc.com/modules/common/dbbasemodel"
+)
+
+// CertificateRequest schema
+const (
+	CertCharging   string = "CHARGING"
+	CertICC        string = "ICC"
+	CertTBOX       string = "TBOX"
+	CertAftersales string = "AFTERSALES"
+)
+
+type Certificate struct {
+	PublicKey    string `json:"public_key"`
+	CommonName   string `json:"-" validate:"required"`
+	PrivateKey   string `json:"private_key,omitempty" pg:"-"`
+	EncryptedKey []byte `json:"-" pg:"encrypted_key"`
+	SerialNumber string `json:"serial_number" pg:",pk"`
+	Type         string `json:"type"`
+	Valid        bool   `json:"-" pg:",use_zero"`
+	CreatedBy    string `json:"-" pg:"created_by"`
+	dbbasemodel.DBModelBase
+}
+
+func (cert Certificate) String() string {
+	return fmt.Sprintf("Certificate for Common Name:<%s>", cert.CommonName)
+}
+
+func (cert Certificate) IsExpiredOrInvalidAtTime(t time.Time, certDaysBeforeExp int) (bool, error) {
+	if !cert.Valid {
+		return true, nil
+	}
+
+	if cert.PublicKey != "" {
+		p, _ := pem.Decode([]byte(cert.PublicKey))
+		if p != nil {
+			c, err := x509.ParseCertificate(p.Bytes)
+			if err != nil {
+				return true, err
+			}
+
+			day := c.NotAfter.AddDate(0, 0, 0-certDaysBeforeExp)
+			if t.After(day) {
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
+}

pkg/common/certificate_request.go

@@ -0,0 +1,32 @@
+package common
+
+import "fmt"
+
+// CertificateRequest schema
+type CertificateRequest struct {
+	CommonName      string `json:"common_name" validate:"required,max=100"`
+	CertificateType string `json:"type" validate:"max=100"`
+}
+
+type CertificateRevokeRequest struct {
+	Serial          string `json:"serial_number" validate:"required,serial,max=1000"`
+	CertificateType string `json:"type" validate:"max=100"`
+}
+
+type CertificateRenewRequest struct {
+	Type       string `json:"type" validate:"required,max=10000"`
+	CommonName string `json:"common_name" validate:"max=100"`
+}
+
+type UpdateCert struct {
+	SSLCertBase64 string `json:"ssl_cert_base64"`
+}
+
+type CertificateInstallRequest struct {
+	VIN   string `pg:",pk" json:"vin" validate:"required,vin"`
+	ICCID string `json:"iccid,omitempty" validate:"omitempty,max=50"`
+}
+
+func (c CertificateRequest) String() string {
+	return fmt.Sprintf("CertificateRequest for Common Name<%s>", c.CommonName)
+}

pkg/common/certificate_test.go

@@ -0,0 +1,63 @@
+package common_test
+
+import (
+	"testing"
+	"time"
+
+	"fiskerinc.com/modules/common"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCertIsExpiredOrInvalidAtTime(t *testing.T) {
+	// this is a fake example cert generated at https://www.samltool.com/self_signed_certs.php
+	var testcert = `-----BEGIN CERTIFICATE-----
+MIIDUzCCAjqgAwIBAgIBADANBgkqhkiG9w0BAQsFADBDMQswCQYDVQQGEwJ1czEL
+MAkGA1UECAwCQ0ExFDASBgNVBAoMC0Zpc2tlciBJbmMuMREwDwYDVQQDDAh0ZXN0
+Y2VydDAeFw0yMzAxMzExOTM5MjZaFw0yNDAxMzExOTM5MjZaMEMxCzAJBgNVBAYT
+AnVzMQswCQYDVQQIDAJDQTEUMBIGA1UECgwLRmlza2VyIEluYy4xETAPBgNVBAMM
+CHRlc3RjZXJ0MIIBIzANBgkqhkiG9w0BAQEFAAOCARAAMIIBCwKCAQIA13BpkJvp
+tqqGTwnMq+t+A50tzENZ3tmtKLIMeuprTux3oqT9PiUHRTLl0zp2r6X+T0A98P+/
+Ad2ybhKtd3qCBEIOkV+M84+q5ecOy2majNQJOgpHNSOtHiAqaZyUslCEtQrLX/Cj
+TLT8RvepzxWf7wB9iIj1hYiUFSXWYqWx07TrtcYEdoGiOd8syjRSHr2nMYjOr/K8
+4Ihyrze9g5j5Dosp943j2WjPETmGebu6bdi5SsoGbkm6dgtKbTKihuo5RBYKMS7t
+xis22jjq4nJigDz506aqY7zRn2Ph1B1CwqxP1O21c7nS78sUmewyKKJY2SX2yB9S
+XcfS4uYjFWC+9GcCAwEAAaNQME4wHQYDVR0OBBYEFMnlDS32ShOeQVUahFE3GUoX
+p/kEMB8GA1UdIwQYMBaAFMnlDS32ShOeQVUahFE3GUoXp/kEMAwGA1UdEwQFMAMB
+Af8wDQYJKoZIhvcNAQELBQADggECAJXUtgm9zuXsDGI1x2zzNY8gjIjsrhToWNAN
+tZKIR2eQETEWwzGLVuz/fmpbSdFN/jnlxLQUjaX2YqlU4gSqHcp4ypYLygs+UEbp
+tfdFDDfxw/1Oc8BRxAxygt6hnFsGM/uMingc6ON4qKg6UeFx9NTfq4jco+/5YDHL
+DNiAv8KUPxreR19bODue6+OKCU6JIkZbMa1/sKzTLkzHbUlHAsxe1JmoqquRvI5z
+a/6nNNka6vwyoSSH6PABU976DkPgDS4tSUvz0yUTwss7an6v5YM+i4T+VpA1nMTA
+LrSlbsmC+whMPAkl4DE9JtmrM3TQTO10bdWmcpMQuOuQpTmdyCfu
+-----END CERTIFICATE-----`
+
+	cert1 := common.Certificate{
+		PublicKey: testcert,
+		Valid:     true,
+	}
+
+	// unexpired cert
+	result1, err := cert1.IsExpiredOrInvalidAtTime(time.Date(2023, 2, 14, 0, 0, 0, 0, time.Local), 30)
+	assert.Nil(t, err)
+	assert.False(t, result1)
+
+	// expired cert
+	result2, err := cert1.IsExpiredOrInvalidAtTime(time.Date(2024, 2, 14, 0, 0, 0, 0, time.Local), 30)
+	assert.Nil(t, err)
+	assert.True(t, result2)
+
+	// less than 30 days before expiration
+	result3, err := cert1.IsExpiredOrInvalidAtTime(time.Date(2024, 1, 14, 0, 0, 0, 0, time.Local), 30)
+	assert.Nil(t, err)
+	assert.True(t, result3)
+
+	cert2 := common.Certificate{
+		PublicKey: testcert,
+		Valid:     false,
+	}
+
+	// invalid cert
+	result4, err := cert2.IsExpiredOrInvalidAtTime(time.Date(2023, 2, 14, 0, 0, 0, 0, time.Local), 30)
+	assert.Nil(t, err)
+	assert.True(t, result4)
+}