Add depot, attendant, jetfire, optimus, ota services with kustomize overlays
This commit is contained in:
Chris Rai
518
services/attendant/controllers/car_update_progress.go
Normal file
518
services/attendant/controllers/car_update_progress.go
Normal file
@@ -0,0 +1,518 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/attendant/services"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/cache"
|
||||
"github.com/fiskerinc/cloud-services/pkg/common"
|
||||
s "github.com/fiskerinc/cloud-services/pkg/common/carupdatestatus"
|
||||
"github.com/fiskerinc/cloud-services/pkg/db/queries"
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/sms"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/manifestsender"
|
||||
"github.com/fiskerinc/cloud-services/pkg/redis"
|
||||
vconfig "github.com/fiskerinc/cloud-services/pkg/vehicleconfig"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/hwversion"
|
||||
"github.com/go-pg/pg/v10"
|
||||
r "github.com/gomodule/redigo/redis"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const redisObjectExpire = 3600
|
||||
|
||||
const (
|
||||
PackageDownloadStart = "package_download_start"
|
||||
PackageDownloadComplete = "package_download_complete"
|
||||
PackageInstallStart = "package_install_start"
|
||||
PackageInstallComplete = "package_install_complete"
|
||||
InstallError = "install_error"
|
||||
)
|
||||
|
||||
var RepeatedStatus = errors.New("RepeatedStatus")
|
||||
|
||||
// CarUpdateProgress takes in a car update message and saves it to our database
|
||||
// This includes setting the status of a car update, and telling the car and SAP that the update is done
|
||||
func NewCarUpdateProgress(clientPool redis.ClientPoolInterface, ka *services.KeepAwake, db *services.DB, device common.Device) CarUpdateProgressInterface {
|
||||
if device == common.TRex {
|
||||
return &CarUpdateProgress{
|
||||
RedisClientPool: clientPool,
|
||||
DB: db,
|
||||
ka: ka,
|
||||
}
|
||||
}
|
||||
|
||||
if device == common.HMI {
|
||||
return &HMICarUpdateProgress{
|
||||
conf: services.GetVehicleConfig(),
|
||||
sms: services.GetSMSClient(),
|
||||
ka: ka,
|
||||
CarUpdateProgress: CarUpdateProgress{
|
||||
RedisClientPool: clientPool,
|
||||
DB: db,
|
||||
ka: ka,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type CarUpdateProgressInterface interface {
|
||||
Process(vin string, data []byte) error
|
||||
ProcessStatus(vin string, status common.CarUpdateProgress) error
|
||||
Dispose()
|
||||
}
|
||||
|
||||
type CarUpdateProgress struct {
|
||||
RedisClientPool redis.ClientPoolInterface
|
||||
DB *services.DB
|
||||
ka *services.KeepAwake
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) Process(vin string, data []byte) error {
|
||||
var status common.CarUpdateProgress
|
||||
|
||||
err := json.Unmarshal(data, &status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return cu.ProcessStatus(vin, status)
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) ProcessStatus(vin string, status common.CarUpdateProgress) (err error) {
|
||||
if cu.transformDBCarUpdateProgress(&status) {
|
||||
err = cu.logStatusDB(status)
|
||||
// If the error is the repeated, we can just exit early
|
||||
if err != nil {
|
||||
if errors.Is(err, queries.RepeatedStatus) {
|
||||
err = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
cu.cancelTheCANAwake(vin, status)
|
||||
|
||||
batch := redis.NewRedisBatchCommands()
|
||||
|
||||
cu.transformRedisCarUpdateProgress(&status)
|
||||
cu.BatchCacheRedis(batch, redis.CarUpdateStatusHashKey(status.CarUpdateID), &status)
|
||||
|
||||
client := cu.RedisClientPool.GetFromPool()
|
||||
defer client.Close()
|
||||
_, err = client.ExecuteBatch(batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// do not send car update status for internal cloud statuses
|
||||
if cu.isInternalStatus(status) {
|
||||
return nil
|
||||
}
|
||||
|
||||
msg := cu.getMessage(&status)
|
||||
err = cu.publishStatusHMI(vin, &msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
msgMobile := cu.getMessageForMobile(&status, vin)
|
||||
err = cu.publishStatusMobile(vin, &msgMobile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = cu.onUpdateManifestComplete(&status, vin)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// We will try and cancel sending CAN status stuff
|
||||
func (cu *CarUpdateProgress) cancelTheCANAwake(vin string, status common.CarUpdateProgress) {
|
||||
switch status.Status {
|
||||
case s.DownloadFailed, s.InstallFailed, s.ManifestCancelAccepted, s.ManifestCancelRejected,
|
||||
s.ManifestError, s.ManifestRejected, s.ManifestValidationFailed, s.RequirementsFailed, s.ManifestCanceled:
|
||||
logger.Info().Msgf("canceling CAN Awake for %s because %s", vin, status.Status)
|
||||
cu.ka.RemoveKeepAwakeMessage(vin)
|
||||
}
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) isInternalStatus(status common.CarUpdateProgress) bool {
|
||||
return status.Status == s.Sent || status.Status == s.Pending
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) transformDBCarUpdateProgress(status *common.CarUpdateProgress) bool {
|
||||
switch status.Status {
|
||||
case s.DownloadStarted:
|
||||
if status.PackageCurrent == 0 {
|
||||
status.Status = PackageDownloadStart
|
||||
}
|
||||
return true
|
||||
case s.DownloadCompleted:
|
||||
if status.PackageCurrent == status.PackageTotal {
|
||||
status.Status = PackageDownloadComplete
|
||||
}
|
||||
return true
|
||||
case s.InstallStarted:
|
||||
if status.InstalledFiles == 0 && status.TotalFiles > 0 {
|
||||
status.Status = PackageInstallStart
|
||||
}
|
||||
return true
|
||||
case s.InstallSucceeded:
|
||||
if status.InstalledFiles == status.TotalFiles && status.TotalFiles > 0 {
|
||||
status.Status = PackageInstallComplete
|
||||
}
|
||||
return true
|
||||
case InstallError:
|
||||
status.Status = s.InstallFailed
|
||||
return true
|
||||
case s.Installing:
|
||||
return false
|
||||
case s.Downloading:
|
||||
// these status updates do not need to be saved in the database
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) transformRedisCarUpdateProgress(status *common.CarUpdateProgress) {
|
||||
switch status.Status {
|
||||
case s.DownloadStarted, s.DownloadCompleted, PackageDownloadStart:
|
||||
status.Status = s.Downloading
|
||||
case s.InstallStarted, s.InstallSucceeded, PackageInstallStart:
|
||||
status.Status = s.Installing
|
||||
}
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) logStatusDB(status common.CarUpdateProgress) (err error) {
|
||||
// If we are one of these status's we want to ignore, then we need to do some extra database steps, otherwise insert normally
|
||||
carUpdate := common.CarUpdate{
|
||||
ID: status.CarUpdateID,
|
||||
Status: status.Status,
|
||||
ErrorCode: status.ErrorCode,
|
||||
Info: strings.TrimSpace(fmt.Sprintf("%s %s", status.ECU, status.Info)),
|
||||
}
|
||||
|
||||
if _, ok := s.NoRepeatUpdateStatus[status.Status]; ok {
|
||||
_, err = cu.DB.GetCarUpdates().UpdateStatusIfNotRepeat(&carUpdate)
|
||||
return
|
||||
}
|
||||
_, err = cu.DB.GetCarUpdates().UpdateStatus(&carUpdate)
|
||||
return err
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) GetCache(key string) (*common.CarUpdateProgress, error) {
|
||||
client := cu.RedisClientPool.GetFromPool()
|
||||
defer client.Close()
|
||||
status := common.CarUpdateProgress{}
|
||||
err := client.GetObject(key, &status)
|
||||
return &status, err
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) BatchCacheRedis(batch *redis.RedisBatchCommands, key string, status *common.CarUpdateProgress) {
|
||||
batch.Add(r.Args{}.Add("HSET").Add(key).AddFlat(status)...)
|
||||
batch.Add("EXPIRE", key, redisObjectExpire)
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) getMessage(status *common.CarUpdateProgress) common.Message {
|
||||
return common.Message{
|
||||
Handler: "car_update_status",
|
||||
Data: status,
|
||||
}
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) getMessageForMobile(status *common.CarUpdateProgress, vin string) common.Message {
|
||||
type mobileData struct {
|
||||
VIN string `json:"vin"`
|
||||
*common.CarUpdateProgress
|
||||
}
|
||||
|
||||
return common.Message{
|
||||
Handler: "car_update_status",
|
||||
Data: mobileData{vin, status},
|
||||
}
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) publishStatusHMI(vin string, msg *common.Message) error {
|
||||
client := cu.RedisClientPool.GetFromPool()
|
||||
defer client.Close()
|
||||
// redis publish to HMI
|
||||
hmiKey := common.HMI.Key(vin)
|
||||
// Add VIN
|
||||
err := client.SafePublishMessage(hmiKey, msg)
|
||||
return err
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) publishStatusMobile(vin string, msg *common.Message) error {
|
||||
drivers := cache.NewDriversCache(cu.RedisClientPool, cu.DB.GetCars())
|
||||
|
||||
// redis publish to mobile devices
|
||||
driverIDs, err := drivers.RetrieveDriverIDs(vin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Change thos for loop to isntead create a batch and execute it all at once
|
||||
client := cu.RedisClientPool.GetFromPool()
|
||||
defer client.Close()
|
||||
for _, d := range driverIDs {
|
||||
mobileKey := common.Mobile.Key(d)
|
||||
err = client.SafePublishMessage(mobileKey, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) Dispose() {
|
||||
cu.DB = nil
|
||||
}
|
||||
|
||||
type HMICarUpdateProgress struct {
|
||||
conf vconfig.ConfigServiceInterface
|
||||
sms sms.SMSServiceClient
|
||||
ka *services.KeepAwake
|
||||
CarUpdateProgress
|
||||
}
|
||||
|
||||
func (h *HMICarUpdateProgress) Process(vin string, data []byte) error {
|
||||
var status common.CarUpdateProgress
|
||||
|
||||
err := json.Unmarshal(data, &status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if h.downloadComplete(&status) {
|
||||
// stop calling the sendKeepAwakeMessage
|
||||
h.ka.RemoveKeepAwakeMessage(vin)
|
||||
_, err = h.sendManifestToTRex(vin, &status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.logStatusDB(common.CarUpdateProgress{
|
||||
CarUpdateID: status.CarUpdateID,
|
||||
Status: s.Sent,
|
||||
Info: "TBOX",
|
||||
})
|
||||
}
|
||||
|
||||
return h.ProcessStatus(vin, status)
|
||||
}
|
||||
|
||||
func (h *HMICarUpdateProgress) downloadComplete(status *common.CarUpdateProgress) bool {
|
||||
return status.Status == s.DownloadCompleted
|
||||
}
|
||||
|
||||
func (h *HMICarUpdateProgress) getManifest(status *common.CarUpdateProgress) (*common.UpdateManifest, error) {
|
||||
update := common.CarUpdate{ID: status.CarUpdateID}
|
||||
err := h.DB.GetCarUpdates().Load(&update)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
update.UpdateManifest.CarUpdateID = status.CarUpdateID
|
||||
|
||||
return update.UpdateManifest, nil
|
||||
}
|
||||
|
||||
func (h *HMICarUpdateProgress) sendManifestToTRex(vin string, status *common.CarUpdateProgress) (msgID string, err error) {
|
||||
logger.Info().Msgf("HMICarUpdateProgress sendManifestToTRex car_update_id %d", status.CarUpdateID)
|
||||
manifest, err := h.getManifest(status)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !manifest.HasSelfDownload() {
|
||||
logger.Error().Msgf("%s download_completed for non-self-download manifest", vin)
|
||||
return
|
||||
}
|
||||
|
||||
err = hwversion.SetHWVersion(manifest, vin, services.GetDB().GetCars())
|
||||
if err != nil {
|
||||
// An error here is very unexpected. The hw versioning should have been confirmed earlier before ICC was updated
|
||||
err = errors.WithStack(err)
|
||||
logger.Err(err).Str("VIN", vin).Int64("UpdateID", status.CarUpdateID).Msg("failed to set hw versions for a manifest after ICC complete update")
|
||||
err = nil
|
||||
}
|
||||
manifest.SortECUs()
|
||||
manifest.FilterCompatibleECUs(vin)
|
||||
|
||||
// This code is going to be removed by mny other PR so not going to mess with it for now
|
||||
client := h.RedisClientPool.GetFromPool()
|
||||
defer client.Close()
|
||||
trex := manifestsender.NewTBOXManifestSender(client, h.conf, h.DB, h.sms, nil)
|
||||
defer trex.Close()
|
||||
|
||||
msgID, err = trex.ProcessSoftwareUpdate(vin, manifest, services.GetDB().GetCarConfigData())
|
||||
return
|
||||
}
|
||||
|
||||
func (h *HMICarUpdateProgress) GetRedisHashKey(status *common.CarUpdateProgress) string {
|
||||
return redis.CarUpdateStatusHMIHashKey(status.CarUpdateID)
|
||||
}
|
||||
|
||||
// Car Update Done
|
||||
func (cu *CarUpdateProgress) onUpdateManifestComplete(status *common.CarUpdateProgress, vin string) (err error) {
|
||||
success := false
|
||||
final := false
|
||||
submitSAP := false
|
||||
|
||||
switch status.Status {
|
||||
case s.ManifestSucceeded:
|
||||
success = true
|
||||
submitSAP = true
|
||||
final = true
|
||||
case s.ManifestCanceled, s.ManifestError, s.ManifestRejected:
|
||||
success = false
|
||||
submitSAP = true
|
||||
final = true
|
||||
case s.DownloadFailed, s.ManifestCancelPending, s.RollbackSucceeded, s.RollbackFailed, s.CleanupSucceeded:
|
||||
final = true
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
carUpdatesDB := cu.DB.GetCarUpdates()
|
||||
carUpdate, err := carUpdatesDB.SelectByID(status.CarUpdateID)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
if carUpdate != nil {
|
||||
// Notify car user of in progress update through FOA API
|
||||
fs := services.GetFoaService()
|
||||
foaResp, err := fs.OtaUpdateStatus(vin, carUpdate, status)
|
||||
if err != nil || (foaResp != nil && foaResp.StatusCode != http.StatusOK) {
|
||||
bodyBytes, _ := io.ReadAll(foaResp.Body)
|
||||
bodyString := string(bodyBytes)
|
||||
logger.Err(err).Msgf("notify FOA for update manifest %d final state %s for %s failed with http status %d and message %s", carUpdate.UpdateManifestID, status.Status, vin, foaResp.StatusCode, bodyString)
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info().Msgf("Manifest update completed for %s with status of %s", vin, status.Status)
|
||||
|
||||
if submitSAP {
|
||||
logger.Info().Msg("SAP: No Longer Submit Updates")
|
||||
// sap := services.GetSapService()
|
||||
// err = sap.SubmitResult(vin, success)
|
||||
// if err != nil {
|
||||
// requestBody := struct {
|
||||
// VIN string
|
||||
// Success bool
|
||||
// CarUpdateProgress common.CarUpdateProgress
|
||||
// }{VIN: vin, Success: success, CarUpdateProgress: *status}
|
||||
// logger.Err(err).Interface("body", requestBody).Msgf("failed to call sap submit result")
|
||||
// err = nil
|
||||
// }
|
||||
}
|
||||
|
||||
if success {
|
||||
// If we are successful, we want to possibly update the cars sums version
|
||||
// Need to pull the manifest to check it has a sums version, and then update the car
|
||||
err = cu.updateCarsSUMSVersion(status)
|
||||
if err != nil {
|
||||
logger.Err(err).Msgf("failed to update car sums version for manifest with CarUpdateID %d", status.CarUpdateID)
|
||||
err = nil
|
||||
}
|
||||
|
||||
// Send the read_ecu_versions remote command so that the ECU data is updated in postgres ASAP
|
||||
client := services.RedisClientPool().GetFromPool()
|
||||
defer client.Close()
|
||||
err = client.SafePublishMessage(
|
||||
common.TRex.Key(vin),
|
||||
common.Message{
|
||||
Handler: "read_ecu_versions",
|
||||
Data: common.RemoteReadVersionsCommandArgs{
|
||||
ECUName: "*",
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
logger.Err(err).Msgf("failed to send read_ecu_versions command to vin %s", vin)
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
if final {
|
||||
// if the manifest is in a final state
|
||||
// then delete the redundant requirements_await rows from car_update_statuses, to avoid overcrowding the table
|
||||
err = cu.truncateRequirementsAwaitForUpdate(status)
|
||||
if err != nil {
|
||||
logger.Err(err).Msgf("failed to delete redundant requirements_await rows from car_update_statuses for manifest with CarUpdateID %d", status.CarUpdateID)
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (cu *CarUpdateProgress) truncateRequirementsAwaitForUpdate(status *common.CarUpdateProgress) error {
|
||||
logger.Info().Msgf("Manifest with CarUpdateID %d successful with status %s. Deleting redundant requirements_await rows from car_update_statuses", status.CarUpdateID, status.Status)
|
||||
|
||||
_, err := cu.DB.GetCarUpdates().TruncateRequirementsAwaitForUpdate(status.CarUpdateID)
|
||||
if err != nil && !errors.Is(err, pg.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the car update, and it gives the update manifest
|
||||
// If the manifest has a sums version, apply it to the car
|
||||
func (cu *CarUpdateProgress) updateCarsSUMSVersion(status *common.CarUpdateProgress) (err error) {
|
||||
carUpdatesDB := cu.DB.GetCarUpdates()
|
||||
carUpdate, err := carUpdatesDB.SelectByID(status.CarUpdateID)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
if carUpdate.UpdateManifest == nil {
|
||||
err = errors.New("failed to pull car updates update manifest")
|
||||
return
|
||||
}
|
||||
|
||||
um := carUpdate.UpdateManifest
|
||||
// So if we have have a sums version we want to update
|
||||
if um.SUMS == "" {
|
||||
return
|
||||
}
|
||||
|
||||
carsDB := cu.DB.GetCars()
|
||||
filter := common.Car{
|
||||
VIN: carUpdate.VIN,
|
||||
}
|
||||
|
||||
cars, err := carsDB.Select(&filter, nil)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(cars) != 1 {
|
||||
err = fmt.Errorf("did not receive only one car, received: %d", len(cars))
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
car := cars[0]
|
||||
car.SUMSVersion = um.SUMS
|
||||
_, err = carsDB.Update(&car)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user