Add depot, attendant, jetfire, optimus, ota services with kustomize overlays
This commit is contained in:
Chris Rai
164
services/jetfire/server/server_consumer.go
Normal file
164
services/jetfire/server/server_consumer.go
Normal file
@@ -0,0 +1,164 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/models"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/fiskerinc/cloud-services/pkg/kafka"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/loggerdataresp"
|
||||
|
||||
"github.com/intel-go/fastjson"
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// StartConsumer runs consumer and puts vehicle signals into a channel for router
|
||||
func StartConsumer(ctx context.Context, topic string) {
|
||||
logger.Debug().Str("StartConsumer", "").Send()
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
logger.Error().Msgf("PanicConsumer %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
producerChannel := make(chan models.InsertCommand, 100)
|
||||
signalsChannel := make(chan *kafka.Message)
|
||||
rebalanceChannel := make(chan struct{})
|
||||
|
||||
consumer, err := services.GetKafkaConsumer()
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
logger.Debug().Msgf("Starting routeEvents and InserterLoop...")
|
||||
|
||||
go routeEvents(signalsChannel, rebalanceChannel, producerChannel)
|
||||
|
||||
go InserterLoop(producerChannel, ctx)
|
||||
|
||||
for {
|
||||
logger.Debug().Msgf("Calling ConsumeOrRebalancedCatch...")
|
||||
consumer.Subscribe([]string{topic})
|
||||
err = consumer.ConsumeOrRebalancedCatch([]string{topic}, signalsChannel, rebalanceChannel)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
if !loggerdataresp.BadDataError(err, loggerdataresp.EofErrorCheck) {
|
||||
check := consumer.Check(ctx)
|
||||
if check != nil {
|
||||
logger.Error().Err(check).Send()
|
||||
}
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// reset the kafka consumer and gc the old one
|
||||
consumer = nil
|
||||
for consumer == nil || err != nil {
|
||||
consumer, err = services.ResetKafkaConsumer()
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processes signals in batch, handles caching of vehicle state and appending to insertion batch caches
|
||||
func routeEvents(signalsChannel chan *kafka.Message, rebalanceChannel <-chan struct{}, producerChannel chan models.InsertCommand) {
|
||||
var jsonErr error
|
||||
var protoErr error
|
||||
var batchData []*kafka_grpc.GRPC_CANSignal
|
||||
|
||||
initialized := false
|
||||
|
||||
//Consumer loop
|
||||
rebalanceFlag := true
|
||||
for {
|
||||
select {
|
||||
case signal := <-signalsChannel:
|
||||
if signal == nil {
|
||||
continue
|
||||
}
|
||||
rebalanceFlag = true
|
||||
|
||||
if !initialized {
|
||||
// init cache after rebalancing
|
||||
logger.Debug().Msgf("INITIALIZING CACHE...")
|
||||
time.Sleep(time.Second)
|
||||
services.InitCacheFromClickhouse()
|
||||
|
||||
initialized = true
|
||||
logger.Debug().Msgf("INITIALIZED...")
|
||||
}
|
||||
|
||||
batchData, protoErr = unmarshalProtobufSignal(signal)
|
||||
if protoErr != nil {
|
||||
batchData, jsonErr = unmarshalJSONSignal(signal)
|
||||
if jsonErr != nil {
|
||||
logger.Error().Err(protoErr).Msg("Failed to unmarshal signal as either Protobuf or JSON")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err := handlers.HandleSignalBatch(batchData, services.GetVehicleCache(), producerChannel)
|
||||
if err != nil {
|
||||
logger.Error().Err(errors.WithStack(err)).Send()
|
||||
}
|
||||
|
||||
case <-rebalanceChannel:
|
||||
if rebalanceFlag {
|
||||
rebalanceFlag = false
|
||||
|
||||
logger.Info().Msgf("kafka rebalancing...")
|
||||
initialized = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func unmarshalProtobufSignal(event *kafka.Message) ([]*kafka_grpc.GRPC_CANSignal, error) {
|
||||
if event == nil {
|
||||
err := errors.Errorf("trying to unmarshall null event ptr")
|
||||
return nil, err
|
||||
}
|
||||
batchData := kafka_grpc.GRPC_CANSignalBatchPayload{}
|
||||
err := proto.Unmarshal(event.Value, &batchData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return batchData.Data.Cansignals, nil
|
||||
}
|
||||
|
||||
func unmarshalJSONSignal(event *kafka.Message) ([]*kafka_grpc.GRPC_CANSignal, error) {
|
||||
if event == nil {
|
||||
err := errors.Errorf("trying to unmarshall null event ptr")
|
||||
return nil, err
|
||||
}
|
||||
//JSON handling is generally slower;
|
||||
//not only is payload much larger but character insertions to fix json format from optimus is very slow.
|
||||
|
||||
batchData := []kafka_grpc.GRPC_CANSignal{}
|
||||
dataBuffer := make([]byte, len(event.Value)+2)
|
||||
copy(dataBuffer[1:], event.Value)
|
||||
dataBuffer[0] = '['
|
||||
dataBuffer[len(event.Value)+1] = ']'
|
||||
|
||||
err := fastjson.Unmarshal(dataBuffer, &batchData)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ptrs := make([]*kafka_grpc.GRPC_CANSignal, len(batchData))
|
||||
for i := range batchData {
|
||||
ptrs[i] = &batchData[i]
|
||||
}
|
||||
return ptrs, nil
|
||||
}
|
||||
25
services/jetfire/server/server_http.go
Normal file
25
services/jetfire/server/server_http.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/httphandlers"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
|
||||
const port string = ":8077"
|
||||
|
||||
func StartHTTPServer() {
|
||||
router := httprouter.New()
|
||||
router.PanicHandler = httphandlers.HttpRouterPanicHandler
|
||||
addHandler(router, http.MethodGet, "/reset", handlers.ResetSchemaDefinitions)
|
||||
logger.Fatal().AnErr("http.ListenAndServe", http.ListenAndServe(port, router)).Send()
|
||||
}
|
||||
|
||||
func addHandler(router *httprouter.Router, method string, path string, handler http.HandlerFunc) {
|
||||
router.HandlerFunc(method, httphandlers.HttpRouterHandleBaseURL(path), handler)
|
||||
}
|
||||
99
services/jetfire/server/server_inserter.go
Normal file
99
services/jetfire/server/server_inserter.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/models"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/ClickHouse/ch-go"
|
||||
"github.com/ClickHouse/ch-go/proto"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
||||
)
|
||||
|
||||
var (
|
||||
flushBufferCmd chan interface{}
|
||||
|
||||
schemaResetPeriod = envtool.GetEnvDuration("JETFIRE_SCHEMA_RESET_PERIOD_MS", 3600000*12) * time.Millisecond
|
||||
)
|
||||
|
||||
// loop for Inserter goroutine.
|
||||
// This goroutine is responsible for inserting data into clickhouse
|
||||
func InserterLoop(producerChannel chan models.InsertCommand, ctx context.Context) {
|
||||
resetSchemaTicker := time.NewTicker(schemaResetPeriod)
|
||||
for {
|
||||
select {
|
||||
// reset cache vars every hour
|
||||
case <-resetSchemaTicker.C:
|
||||
logger.Debug().Msgf("<-resetSchemaTicker")
|
||||
services.ResetCacheVars()
|
||||
|
||||
// process flush buffer command
|
||||
case <-flushBufferCmd:
|
||||
logger.Debug().Msgf("<-flushBufferCmd")
|
||||
|
||||
//get clickhouse client
|
||||
client := services.GetShardClient()
|
||||
for client == nil || client.IsClosed() {
|
||||
logger.Error().Err(errors.Errorf("bad chgo client , retrying connection...")).Send()
|
||||
time.Sleep(time.Second)
|
||||
services.InitShardClients()
|
||||
client = services.GetShardClient()
|
||||
}
|
||||
|
||||
logger.Debug().Msgf("InsertFlushAllBuffers??")
|
||||
services.InsertFlushAllBuffers(client)
|
||||
|
||||
// process data to be inserted to clickhouse
|
||||
case command := <-producerChannel:
|
||||
logger.Debug().Msgf("<-producerChannel")
|
||||
// get clickhouse client
|
||||
client := services.GetShardClient()
|
||||
for client == nil || client.IsClosed() {
|
||||
logger.Error().Err(errors.Errorf("bad chgo client , retrying connection...")).Send()
|
||||
time.Sleep(time.Second)
|
||||
services.InitShardClients()
|
||||
client = services.GetShardClient()
|
||||
}
|
||||
|
||||
//insert the queued buffer and block in command
|
||||
var err error
|
||||
retryInsert := true
|
||||
count := 0
|
||||
insertTime := time.Now()
|
||||
for retryInsert {
|
||||
count, err = models.InsertAndFlush(command, ctx, client.GetClient(), client.GetBreaker(), nil)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
logger.Error().Err(errors.WithStack(err)).Send()
|
||||
// if the error is a mismatching schema error, then pull new schemas
|
||||
if ch.IsErr(err,
|
||||
proto.ErrIncompatibleColumns,
|
||||
proto.ErrNoSuchColumnInTable,
|
||||
proto.ErrThereIsNoColumn,
|
||||
proto.ErrIncorrectNumberOfColumns,
|
||||
) {
|
||||
services.ResetCacheVars()
|
||||
} else {
|
||||
// client.Connect() //close and restart the clickhouse connection
|
||||
client = services.GetShardClient() //grab a new shard client to retry insert
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
//log messages relating to clickhouse insertion
|
||||
logger.Debug().Msgf("done flush Buffer %s, %dms", command.Buffer.TableName, (time.Since(insertTime))/time.Millisecond)
|
||||
if time.Since(insertTime) > 10*time.Second {
|
||||
logger.Warn().Msgf("slow row insertions: took %s to insert %d rows for %s", time.Since(insertTime), count, command.Buffer.TableName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user