admin/cloud-services
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add depot, attendant, jetfire, optimus, ota services with kustomize overlays

Browse Source
This commit is contained in:
Chris Rai
2026-01-31 15:35:07 -05:00
parent a0ec642ca1
commit 9a5cb2f547
404 changed files with 38817 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
services/jetfire/tests/bench_test.go Normal file
View File

@@ -0,0 +1,103 @@
package tests
/*
This file implements benchmarks for
1. Batch handling of messages.
The intention is to measure performance of appending rows to InsertBuffer
2. Batch serialization of messages
The intention is to measure performance of preparing data for clickhouse insertion
Because of the size of the input data for this benchmark, it is recommended to set
JETFIRE_BUFFER_MAX_BYTES=1073741824
Or more; otherwise the benchmark may hang while waiting for the nonexistant inserter thread to flush the buffer.
*/
import (
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
"github.com/fiskerinc/cloud-services/services/jetfire/services"
"os"
"testing"
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
"github.com/ClickHouse/ch-go/proto"
"github.com/intel-go/fastjson"
)
var benchmarkJSONPayload = []byte{}
var benchmarkBatchPayload = []kafka_grpc.GRPC_CANSignal{}
var benchmarkBatchPtrs = []*kafka_grpc.GRPC_CANSignal{}
func benchInit() {
//1176 messages long
jsonPath := "test-batch-msg.json"
os.Chdir("./tests/")
data, err := os.ReadFile(jsonPath)
if err != nil {
panic(err)
}
benchmarkJSONPayload = data
fastjson.Unmarshal(data, &benchmarkBatchPayload)
benchmarkBatchPtrs = make([]*kafka_grpc.GRPC_CANSignal, len(benchmarkBatchPayload))
for i := range benchmarkBatchPayload {
benchmarkBatchPtrs[i] = &benchmarkBatchPayload[i]
}
services.ResetCacheVars()
}
func benchmarkMessageHandler(batchData []*kafka_grpc.GRPC_CANSignal, b *testing.B) {
cache := services.GetVehicleCache()
for i := 0; i < b.N; i++ {
handlers.HandleSignalBatch(batchData, cache, nil)
}
b.StopTimer()
}
func BenchmarkMessageHandler(b *testing.B) {
benchInit()
b.ResetTimer()
benchmarkMessageHandler(benchmarkBatchPtrs, b)
}
func BenchmarkSignalSerialization(b *testing.B) {
benchInit()
cache := services.GetVehicleCache()
handlers.HandleSignalBatch(benchmarkBatchPtrs, cache, nil)
b.ResetTimer()
dummy := proto.Input{}
serializedLength := 0
rowsLength := 0
for i := 0; i < b.N; i++ {
dummy = services.GetVehicleSignalBatch().GetInput()
serializedLength += len(dummy)
rowsLength += services.GetVehicleSignalBatch().Len()
}
b.StopTimer()
}
func BenchmarkFeatureSerialization(b *testing.B) {
benchInit()
cache := services.GetVehicleCache()
handlers.HandleSignalBatch(benchmarkBatchPtrs, cache, nil)
b.ResetTimer()
dummy := proto.Input{}
serializedLength := 0
rowsLength := 0
for i := 0; i < b.N; i++ {
dummy = services.GetFeatureBatch().GetInput()
serializedLength += len(dummy)
rowsLength += services.GetFeatureBatch().Len()
}
b.StopTimer()
}

251
services/jetfire/tests/cache_test.go Normal file
View File

@@ -0,0 +1,251 @@
package tests
import (
"fmt"
"github.com/fiskerinc/cloud-services/services/jetfire/services"
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
"testing"
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
"github.com/stretchr/testify/assert"
)
const testVIN = "TESTVIN1234567890"
var testCanSignalBatch = []kafka_grpc.GRPC_CANSignal{
{Vin: testVIN, Timestamp: 600.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 100},
{Vin: testVIN, Timestamp: 601.0, Id: 792, Name: "ESP_VehSpd", Value: 0},
{Vin: testVIN, Timestamp: 602.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 90},
{Vin: testVIN, Timestamp: 603.0, Id: 792, Name: "ESP_VehSpd", Value: 10},
{Vin: testVIN, Timestamp: 604.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 80},
{Vin: testVIN, Timestamp: 605.0, Id: 792, Name: "ESP_VehSpd", Value: 20},
{Vin: testVIN, Timestamp: 606.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 80},
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 30},
{Vin: testVIN, Timestamp: 608.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 70},
{Vin: testVIN, Timestamp: 609.0, Id: 792, Name: "ESP_VehSpd", Value: 40},
{Vin: testVIN, Timestamp: 608.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 60},
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 50},
{Vin: testVIN, Timestamp: 610.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 50},
{Vin: testVIN, Timestamp: 611.0, Id: 792, Name: "ESP_VehSpd", Value: 60},
{Vin: testVIN, Timestamp: 612.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 40},
{Vin: testVIN, Timestamp: 613.0, Id: 792, Name: "ESP_VehSpd", Value: 70},
{Vin: testVIN, Timestamp: 614.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 30},
{Vin: testVIN, Timestamp: 615.0, Id: 792, Name: "ESP_VehSpd", Value: 80},
{Vin: testVIN, Timestamp: 616.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 20},
{Vin: testVIN, Timestamp: 617.0, Id: 792, Name: "ESP_VehSpd", Value: 90},
{Vin: testVIN, Timestamp: 618.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 60},
{Vin: testVIN, Timestamp: 619.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
{Vin: testVIN, Timestamp: 650.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
{Vin: testVIN, Timestamp: 800.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
{Vin: testVIN, Timestamp: 1500.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 20},
{Vin: testVIN, Timestamp: 1501.0, Id: 792, Name: "ESP_VehSpd", Value: 90},
{Vin: testVIN, Timestamp: 1502.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 45},
{Vin: testVIN, Timestamp: 1503.0, Id: 792, Name: "ESP_VehSpd", Value: 13},
{Vin: testVIN, Timestamp: 1510.0, Id: 819, Name: "BCM_PwrMod", Value: 0},
{Vin: testVIN, Timestamp: 1511.0, Id: 819, Name: "BCM_PwrMod", Value: 2},
{Vin: testVIN, Timestamp: 1512.0, Id: 792, Name: "ESP_VehSpd", Value: 11},
}
var testCanSignalOrderBatch = []kafka_grpc.GRPC_CANSignal{
{Vin: testVIN, Timestamp: 700.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 100},
{Vin: testVIN, Timestamp: 601.0, Id: 792, Name: "ESP_VehSpd", Value: 0},
{Vin: testVIN, Timestamp: 603.0, Id: 792, Name: "ESP_VehSpd", Value: 10},
{Vin: testVIN, Timestamp: 704.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 80},
{Vin: testVIN, Timestamp: 605.0, Id: 792, Name: "ESP_VehSpd", Value: 20},
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 30},
{Vin: testVIN, Timestamp: 708.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 70},
{Vin: testVIN, Timestamp: 609.0, Id: 792, Name: "ESP_VehSpd", Value: 40},
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 50},
{Vin: testVIN, Timestamp: 710.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 50},
{Vin: testVIN, Timestamp: 611.0, Id: 792, Name: "ESP_VehSpd", Value: 60},
{Vin: testVIN, Timestamp: 613.0, Id: 792, Name: "ESP_VehSpd", Value: 70},
{Vin: testVIN, Timestamp: 714.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 30},
{Vin: testVIN, Timestamp: 615.0, Id: 792, Name: "ESP_VehSpd", Value: 80},
{Vin: testVIN, Timestamp: 617.0, Id: 792, Name: "ESP_VehSpd", Value: 90},
{Vin: testVIN, Timestamp: 718.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 60},
{Vin: testVIN, Timestamp: 619.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
}
func TestVehicleCache(t *testing.T) {
cache := services.GetVehicleCache()
cache.Clear()
for i, signal := range testCanSignalBatch {
cache.UpdateSignal(&signal, 0x0)
if i == 4 {
// testing signal aggregation
state, containsState := cache.Cache[testVIN]
assert.True(t, containsState)
assert.NotNil(t, state)
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(604.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 80.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 10.0)
}
if i == 11 {
// testing out of order messages
state := cache.Cache[testVIN]
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(609.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 40.0)
}
if i == 17 {
// testing signal aggregation after message order is restored
state := cache.Cache[testVIN]
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(615.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 30.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 80.0)
}
if i == 22 {
state := cache.Cache[testVIN]
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(650.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 100.0)
}
if i == 23 {
state := cache.Cache[testVIN]
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(800.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 100.0)
}
if i == 25 {
state := cache.Cache[testVIN]
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(1501.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(1500.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 20.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 90.0)
}
}
assert.Equal(t, len(cache.Cache), 1)
// testing large timestamp gap; trigger new trip
state := cache.Cache[testVIN]
tripStartTime := utils.FloatToTime(1511.0)
assert.WithinDuration(t, state.TripStart, tripStartTime, 1e8)
assert.Equal(t, state.TripID, fmt.Sprintf("%s_%d", testVIN, 1511))
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(1512.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 45.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 11.0)
}
func TestVehicleCacheOrderly(t *testing.T) {
cache := services.GetVehicleCache()
cache.Clear()
for i, signal := range testCanSignalOrderBatch {
cache.UpdateSignal(&signal, 0x0)
if i == 8 {
state := cache.Cache[testVIN]
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(708.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(700.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 70.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 40.0)
}
}
state := cache.Cache[testVIN]
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(718.0), 1e8)
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(700.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 100.0)
}
func TestVehicleCacheList(t *testing.T) {
cache := services.GetVehicleCache()
cache.Clear()
signal := kafka_grpc.GRPC_CANSignal{
Vin: "TESTVIN1",
Timestamp: 0.0,
Id: 816,
Name: "ESP_VehSpd",
Value: 792,
}
cache.UpdateSignal(&signal, 0x0)
signal = kafka_grpc.GRPC_CANSignal{
Vin: "TESTVIN2",
Timestamp: 0.0,
Id: 816,
Name: "ESP_VehSpd",
Value: 792,
}
cache.UpdateSignal(&signal, 0x0)
signal = kafka_grpc.GRPC_CANSignal{
Vin: "TESTVIN3",
Timestamp: 0.0,
Id: 816,
Name: "ESP_VehSpd",
Value: 792,
}
cache.UpdateSignal(&signal, 0x0)
expectedVins := []string{"TESTVIN1", "TESTVIN2", "TESTVIN3"}
node := cache.StatesListHead
i := 0
for node != nil {
assert.Equal(t, expectedVins[i], node.VIN)
node = node.Next
i++
}
node2 := cache.PopLeft()
expectedVins = []string{"TESTVIN2", "TESTVIN3"}
node = cache.StatesListHead
i = 0
for node != nil {
assert.Equal(t, expectedVins[i], node.VIN)
node = node.Next
i++
}
cache.ReinsertRight(node2)
expectedVins = []string{"TESTVIN2", "TESTVIN3", "TESTVIN1"}
node = cache.StatesListHead
i = 0
for node != nil {
assert.Equal(t, expectedVins[i], node.VIN)
node = node.Next
i++
}
cache.ReinsertRight(cache.Cache["TESTVIN3"])
expectedVins = []string{"TESTVIN2", "TESTVIN1", "TESTVIN3"}
node = cache.StatesListHead
i = 0
for node != nil {
assert.Equal(t, expectedVins[i], node.VIN)
node = node.Next
i++
}
}

151
services/jetfire/tests/integration_test.go Normal file
View File

@@ -0,0 +1,151 @@
package tests
import (
"context"
"fmt"
"github.com/fiskerinc/cloud-services/services/jetfire/server"
"github.com/fiskerinc/cloud-services/services/jetfire/services"
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
"os"
"testing"
"time"
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
"github.com/fiskerinc/cloud-services/pkg/kafka"
"github.com/fiskerinc/cloud-services/pkg/logger"
"github.com/intel-go/fastjson"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/proto"
)
var batchDataSignal = []kafka_grpc.GRPC_CANSignal{}
var dataToPublish kafka_grpc.GRPC_CANSignalBatchPayload
var startTimestamp = time.Now().UTC()
func TestIntegration(t *testing.T) {
t.Skip()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
//cleaning up previous run
conn, err := services.GetClickhouseConnection()
if err != nil {
logger.Error().Err(err)
}
startTimestamp = time.Now().UTC()
timestampString := fmt.Sprintf(
"%d-%d-%d %d:%d:%d",
startTimestamp.Year(),
startTimestamp.Month(),
startTimestamp.Day(),
startTimestamp.Hour(),
startTimestamp.Minute(),
startTimestamp.Second(),
)
println("cleaning up previous run")
conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s DELETE WHERE VIN=='%s' AND Timestamp<'%s'", services.FEATURE_TABLE, testVIN, timestampString))
conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s DELETE WHERE VIN=='%s' AND Timestamp<'%s'", services.VEHICLE_SIGNAL_TABLE, testVIN, timestampString))
// conn.Ping(ctx)
//initialization
cache := services.GetVehicleCache()
cache.Clear()
readTestData()
producer, err := kafka.NewAsyncProducer(ctx)
assert.Nil(t, err)
// runJetfire(ctx)
publishDuration := int64(10)
testDuration := time.Duration(publishDuration*2) * time.Second //batch inserts take some time to run. give it some time...
// begin publishing kafka data. Update message timestamps first.
grpcData, err := proto.Marshal(&dataToPublish)
assert.Nil(t, err)
err = producer.ProduceBinary(kafka.VehicleSignal, testVIN, grpcData, nil)
assert.Nil(t, err)
//wait a while since kafka has to rebalance, wait for clickhouse inserts to trigger
time.Sleep(testDuration)
//check clickhouse, count rows
println("querying clickhouse feature...")
query := fmt.Sprintf("SELECT VIN, Timestamp FROM %s WHERE VIN=='%s' AND Timestamp>='%s'",
services.FEATURE_TABLE,
testVIN,
timestampString,
)
checkRows(query, 1, t)
println("querying clickhouse vehicle_signal...")
query = fmt.Sprintf("SELECT VIN, Timestamp FROM %s WHERE VIN=='%s' AND Timestamp>='%s'",
services.VEHICLE_SIGNAL_TABLE,
testVIN,
timestampString,
)
checkRows(query, 1000, t)
}
func checkRows(query string, expected int, t *testing.T) {
conn, err := services.GetClickhouseConnection()
if err != nil {
logger.Error().Err(err)
return
}
fmt.Println(query)
rows, err := conn.Query(context.Background(), query)
assert.Nil(t, err)
count := int(0)
defer rows.Close()
for rows.Next() {
count++
}
assert.GreaterOrEqual(t, count, expected)
}
func readTestData() {
//1176 messages long
jsonPath := "./test-batch-msg.json"
data, err := os.ReadFile(jsonPath)
if err != nil {
panic(err)
}
fastjson.Unmarshal(data, &batchDataSignal)
dataPtr := make([]*kafka_grpc.GRPC_CANSignal, len(batchDataSignal))
offset := -1.0
for i := range batchDataSignal {
dataPtr[i] = &batchDataSignal[i]
// find min timestamp in batch data
if offset < 0 || offset > batchDataSignal[i].Timestamp {
offset = dataPtr[i].Timestamp
}
dataPtr[i].Vin = testVIN // in case we need to chagne the test vin to follow proper pattern
}
for i := range batchDataSignal {
dataPtr[i].Timestamp += utils.TimeToFloat(startTimestamp) - offset
}
dataToPublish = kafka_grpc.GRPC_CANSignalBatchPayload{Data: &kafka_grpc.GRPC_CANSignalData{
Cansignals: dataPtr,
}}
}
func runJetfire(ctx context.Context) {
// first initialize and run jetfire application loops
services.ResetCacheVars()
go server.StartConsumer(ctx, kafka.VehicleSignal)
}

39
services/jetfire/tests/message_test.go Normal file
View File

@@ -0,0 +1,39 @@
package tests
import (
"fmt"
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
"github.com/fiskerinc/cloud-services/services/jetfire/services"
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
"testing"
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
"github.com/stretchr/testify/assert"
)
func TestHandleSignalMessage(t *testing.T) {
cache := services.GetVehicleCache()
ptrArray := make([]*kafka_grpc.GRPC_CANSignal, len(testCanSignalBatch))
for i := range testCanSignalBatch {
ptrArray[i] = &testCanSignalBatch[i]
}
services.ResetCacheVars()
cache = services.GetVehicleCache()
cache.Clear()
handlers.HandleSignalBatch(ptrArray, cache, nil)
assert.Equal(t, len(cache.Cache), 1)
// testing large timestamp gap; trigger new trip
state := cache.Cache[testVIN]
tripStartTime := utils.FloatToTime(1511.0)
assert.WithinDuration(t, state.TripStart, tripStartTime, 1e8)
assert.Equal(t, state.TripID, fmt.Sprintf("%s_%d", testVIN, 1511))
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(1512.0), 1e8)
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 45.0)
assert.Equal(t, state.StateValues["ESP_VehSpd"], 11.0)
}

26
services/jetfire/tests/reset_test.go Normal file
View File

@@ -0,0 +1,26 @@
//go:build reset
// +build reset
package tests
import (
"net/http"
"testing"
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
th "github.com/fiskerinc/cloud-services/pkg/testhelper"
)
func TestResetSchema(t *testing.T) {
tests := []th.BasicHttpTest{
{
Name: "Reset",
Request: th.MakeTestRequest(http.MethodGet, "http://example.com/reset", nil),
ExpectedStatus: http.StatusOK,
ExpectedResponse: "",
},
}
th.RunBasicHttpTests(t, tests, handlers.ResetSchemaDefinitions)
}

1
services/jetfire/tests/test-batch-msg.json Normal file
View File

File diff suppressed because one or more lines are too long