cloud-services/services/attendant/controllers/car_update_progress.go

package controllers

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/fiskerinc/cloud-services/services/attendant/services"

	"github.com/fiskerinc/cloud-services/pkg/cache"
	"github.com/fiskerinc/cloud-services/pkg/common"
	s "github.com/fiskerinc/cloud-services/pkg/common/carupdatestatus"
	"github.com/fiskerinc/cloud-services/pkg/db/queries"
	"github.com/fiskerinc/cloud-services/pkg/grpc/sms"
	"github.com/fiskerinc/cloud-services/pkg/logger"
	"github.com/fiskerinc/cloud-services/pkg/manifestsender"
	"github.com/fiskerinc/cloud-services/pkg/redis"
	vconfig "github.com/fiskerinc/cloud-services/pkg/vehicleconfig"

	"github.com/fiskerinc/cloud-services/pkg/hwversion"
	"github.com/go-pg/pg/v10"
	r "github.com/gomodule/redigo/redis"
	"github.com/pkg/errors"
)

const redisObjectExpire = 3600

const (
	PackageDownloadStart    = "package_download_start"
	PackageDownloadComplete = "package_download_complete"
	PackageInstallStart     = "package_install_start"
	PackageInstallComplete  = "package_install_complete"
	InstallError            = "install_error"
)

var RepeatedStatus = errors.New("RepeatedStatus")

// CarUpdateProgress takes in a car update message and saves it to our database
// This includes setting the status of a car update, and telling the car and SAP that the update is done
func NewCarUpdateProgress(clientPool redis.ClientPoolInterface, ka *services.KeepAwake, db *services.DB, device common.Device) CarUpdateProgressInterface {
	if device == common.TRex {
		return &CarUpdateProgress{
			RedisClientPool: clientPool,
			DB:              db,
			ka:              ka,
		}
	}

	if device == common.HMI {
		return &HMICarUpdateProgress{
			conf: services.GetVehicleConfig(),
			sms:  services.GetSMSClient(),
			ka:   ka,
			CarUpdateProgress: CarUpdateProgress{
				RedisClientPool: clientPool,
				DB:              db,
				ka:              ka,
			},
		}
	}

	return nil
}

type CarUpdateProgressInterface interface {
	Process(vin string, data []byte) error
	ProcessStatus(vin string, status common.CarUpdateProgress) error
	Dispose()
}

type CarUpdateProgress struct {
	RedisClientPool redis.ClientPoolInterface
	DB              *services.DB
	ka              *services.KeepAwake
}

func (cu *CarUpdateProgress) Process(vin string, data []byte) error {
	var status common.CarUpdateProgress

	err := json.Unmarshal(data, &status)
	if err != nil {
		return err
	}

	return cu.ProcessStatus(vin, status)
}

func (cu *CarUpdateProgress) ProcessStatus(vin string, status common.CarUpdateProgress) (err error) {
	if cu.transformDBCarUpdateProgress(&status) {
		err = cu.logStatusDB(status)
		// If the error is the repeated, we can just exit early
		if err != nil {
			if errors.Is(err, queries.RepeatedStatus) {
				err = nil
			}
			return
		}
	}

	cu.cancelTheCANAwake(vin, status)

	batch := redis.NewRedisBatchCommands()

	cu.transformRedisCarUpdateProgress(&status)
	cu.BatchCacheRedis(batch, redis.CarUpdateStatusHashKey(status.CarUpdateID), &status)

	client := cu.RedisClientPool.GetFromPool()
	defer client.Close()
	_, err = client.ExecuteBatch(batch)
	if err != nil {
		return err
	}

	// do not send car update status for internal cloud statuses
	if cu.isInternalStatus(status) {
		return nil
	}

	msg := cu.getMessage(&status)
	err = cu.publishStatusHMI(vin, &msg)
	if err != nil {
		return err
	}

	msgMobile := cu.getMessageForMobile(&status, vin)
	err = cu.publishStatusMobile(vin, &msgMobile)
	if err != nil {
		return err
	}

	err = cu.onUpdateManifestComplete(&status, vin)

	return err
}

// We will try and cancel sending CAN status stuff
func (cu *CarUpdateProgress) cancelTheCANAwake(vin string, status common.CarUpdateProgress) {
	switch status.Status {
	case s.DownloadFailed, s.InstallFailed, s.ManifestCancelAccepted, s.ManifestCancelRejected,
		s.ManifestError, s.ManifestRejected, s.ManifestValidationFailed, s.RequirementsFailed, s.ManifestCanceled:
		logger.Info().Msgf("canceling CAN Awake for %s because %s", vin, status.Status)
		cu.ka.RemoveKeepAwakeMessage(vin)
	}
}

func (cu *CarUpdateProgress) isInternalStatus(status common.CarUpdateProgress) bool {
	return status.Status == s.Sent || status.Status == s.Pending
}

func (cu *CarUpdateProgress) transformDBCarUpdateProgress(status *common.CarUpdateProgress) bool {
	switch status.Status {
	case s.DownloadStarted:
		if status.PackageCurrent == 0 {
			status.Status = PackageDownloadStart
		}
		return true
	case s.DownloadCompleted:
		if status.PackageCurrent == status.PackageTotal {
			status.Status = PackageDownloadComplete
		}
		return true
	case s.InstallStarted:
		if status.InstalledFiles == 0 && status.TotalFiles > 0 {
			status.Status = PackageInstallStart
		}
		return true
	case s.InstallSucceeded:
		if status.InstalledFiles == status.TotalFiles && status.TotalFiles > 0 {
			status.Status = PackageInstallComplete
		}
		return true
	case InstallError:
		status.Status = s.InstallFailed
		return true
	case s.Installing:
		return false
	case s.Downloading:
		// these status updates do not need to be saved in the database
		return false
	}

	return true
}

func (cu *CarUpdateProgress) transformRedisCarUpdateProgress(status *common.CarUpdateProgress) {
	switch status.Status {
	case s.DownloadStarted, s.DownloadCompleted, PackageDownloadStart:
		status.Status = s.Downloading
	case s.InstallStarted, s.InstallSucceeded, PackageInstallStart:
		status.Status = s.Installing
	}
}

func (cu *CarUpdateProgress) logStatusDB(status common.CarUpdateProgress) (err error) {
	// If we are one of these status's we want to ignore, then we need to do some extra database steps, otherwise insert normally
	carUpdate := common.CarUpdate{
		ID:        status.CarUpdateID,
		Status:    status.Status,
		ErrorCode: status.ErrorCode,
		Info:      strings.TrimSpace(fmt.Sprintf("%s %s", status.ECU, status.Info)),
	}

	if _, ok := s.NoRepeatUpdateStatus[status.Status]; ok {
		_, err = cu.DB.GetCarUpdates().UpdateStatusIfNotRepeat(&carUpdate)
		return
	}
	_, err = cu.DB.GetCarUpdates().UpdateStatus(&carUpdate)
	return err
}

func (cu *CarUpdateProgress) GetCache(key string) (*common.CarUpdateProgress, error) {
	client := cu.RedisClientPool.GetFromPool()
	defer client.Close()
	status := common.CarUpdateProgress{}
	err := client.GetObject(key, &status)
	return &status, err
}

func (cu *CarUpdateProgress) BatchCacheRedis(batch *redis.RedisBatchCommands, key string, status *common.CarUpdateProgress) {
	batch.Add(r.Args{}.Add("HSET").Add(key).AddFlat(status)...)
	batch.Add("EXPIRE", key, redisObjectExpire)
}

func (cu *CarUpdateProgress) getMessage(status *common.CarUpdateProgress) common.Message {
	return common.Message{
		Handler: "car_update_status",
		Data:    status,
	}
}

func (cu *CarUpdateProgress) getMessageForMobile(status *common.CarUpdateProgress, vin string) common.Message {
	type mobileData struct {
		VIN string `json:"vin"`
		*common.CarUpdateProgress
	}

	return common.Message{
		Handler: "car_update_status",
		Data:    mobileData{vin, status},
	}
}

func (cu *CarUpdateProgress) publishStatusHMI(vin string, msg *common.Message) error {
	client := cu.RedisClientPool.GetFromPool()
	defer client.Close()
	// redis publish to HMI
	hmiKey := common.HMI.Key(vin)
	// Add VIN
	err := client.SafePublishMessage(hmiKey, msg)
	return err
}

func (cu *CarUpdateProgress) publishStatusMobile(vin string, msg *common.Message) error {
	drivers := cache.NewDriversCache(cu.RedisClientPool, cu.DB.GetCars())

	// redis publish to mobile devices
	driverIDs, err := drivers.RetrieveDriverIDs(vin)
	if err != nil {
		return err
	}

	// Change thos for loop to isntead create a batch and execute it all at once
	client := cu.RedisClientPool.GetFromPool()
	defer client.Close()
	for _, d := range driverIDs {
		mobileKey := common.Mobile.Key(d)
		err = client.SafePublishMessage(mobileKey, msg)
		if err != nil {
			return err
		}
	}

	return nil
}

func (cu *CarUpdateProgress) Dispose() {
	cu.DB = nil
}

type HMICarUpdateProgress struct {
	conf vconfig.ConfigServiceInterface
	sms  sms.SMSServiceClient
	ka   *services.KeepAwake
	CarUpdateProgress
}

func (h *HMICarUpdateProgress) Process(vin string, data []byte) error {
	var status common.CarUpdateProgress

	err := json.Unmarshal(data, &status)
	if err != nil {
		return err
	}

	if h.downloadComplete(&status) {
		// stop calling the sendKeepAwakeMessage
		h.ka.RemoveKeepAwakeMessage(vin)
		_, err = h.sendManifestToTRex(vin, &status)
		if err != nil {
			return err
		}
		h.logStatusDB(common.CarUpdateProgress{
			CarUpdateID: status.CarUpdateID,
			Status:      s.Sent,
			Info:        "TBOX",
		})
	}

	return h.ProcessStatus(vin, status)
}

func (h *HMICarUpdateProgress) downloadComplete(status *common.CarUpdateProgress) bool {
	return status.Status == s.DownloadCompleted
}

func (h *HMICarUpdateProgress) getManifest(status *common.CarUpdateProgress) (*common.UpdateManifest, error) {
	update := common.CarUpdate{ID: status.CarUpdateID}
	err := h.DB.GetCarUpdates().Load(&update)
	if err != nil {
		return nil, err
	}

	update.UpdateManifest.CarUpdateID = status.CarUpdateID

	return update.UpdateManifest, nil
}

func (h *HMICarUpdateProgress) sendManifestToTRex(vin string, status *common.CarUpdateProgress) (msgID string, err error) {
	logger.Info().Msgf("HMICarUpdateProgress sendManifestToTRex car_update_id %d", status.CarUpdateID)
	manifest, err := h.getManifest(status)
	if err != nil {
		return
	}

	if !manifest.HasSelfDownload() {
		logger.Error().Msgf("%s download_completed for non-self-download manifest", vin)
		return
	}

	err = hwversion.SetHWVersion(manifest, vin, services.GetDB().GetCars())
	if err != nil {
		// An error here is very unexpected. The hw versioning should have been confirmed earlier before ICC was updated
		err = errors.WithStack(err)
		logger.Err(err).Str("VIN", vin).Int64("UpdateID", status.CarUpdateID).Msg("failed to set hw versions for a manifest after ICC complete update")
		err = nil
	}
	manifest.SortECUs()
	manifest.FilterCompatibleECUs(vin)

	// This code is going to be removed by mny other PR so not going to mess with it for now
	client := h.RedisClientPool.GetFromPool()
	defer client.Close()
	trex := manifestsender.NewTBOXManifestSender(client, h.conf, h.DB, h.sms, nil)
	defer trex.Close()

	msgID, err = trex.ProcessSoftwareUpdate(vin, manifest, services.GetDB().GetCarConfigData())
	return
}

func (h *HMICarUpdateProgress) GetRedisHashKey(status *common.CarUpdateProgress) string {
	return redis.CarUpdateStatusHMIHashKey(status.CarUpdateID)
}

// Car Update Done
func (cu *CarUpdateProgress) onUpdateManifestComplete(status *common.CarUpdateProgress, vin string) (err error) {
	success := false
	final := false
	submitSAP := false

	switch status.Status {
	case s.ManifestSucceeded:
		success = true
		submitSAP = true
		final = true
	case s.ManifestCanceled, s.ManifestError, s.ManifestRejected:
		success = false
		submitSAP = true
		final = true
	case s.DownloadFailed, s.ManifestCancelPending, s.RollbackSucceeded, s.RollbackFailed, s.CleanupSucceeded:
		final = true
	default:
		return nil
	}

	carUpdatesDB := cu.DB.GetCarUpdates()
	carUpdate, err := carUpdatesDB.SelectByID(status.CarUpdateID)
	if err != nil {
		err = errors.WithStack(err)
		return
	}
	if carUpdate != nil {
		// Notify car user of in progress update through FOA API
		fs := services.GetFoaService()
		foaResp, err := fs.OtaUpdateStatus(vin, carUpdate, status)
		if err != nil || (foaResp != nil && foaResp.StatusCode != http.StatusOK) {
			bodyBytes, _ := io.ReadAll(foaResp.Body)
			bodyString := string(bodyBytes)
			logger.Err(err).Msgf("notify FOA for update manifest %d final state %s for %s failed with http status %d and message %s", carUpdate.UpdateManifestID, status.Status, vin, foaResp.StatusCode, bodyString)
			err = nil
		}
	}

	logger.Info().Msgf("Manifest update completed for %s with status of %s", vin, status.Status)

	if submitSAP {
		logger.Info().Msg("SAP: No Longer Submit Updates")
		// sap := services.GetSapService()
		// err = sap.SubmitResult(vin, success)
		// if err != nil {
		// 	requestBody := struct {
		// 		VIN               string
		// 		Success           bool
		// 		CarUpdateProgress common.CarUpdateProgress
		// 	}{VIN: vin, Success: success, CarUpdateProgress: *status}
		// 	logger.Err(err).Interface("body", requestBody).Msgf("failed to call sap submit result")
		// 	err = nil
		// }
	}

	if success {
		// If we are successful, we want to possibly update the cars sums version
		// Need to pull the manifest to check it has a sums version, and then update the car
		err = cu.updateCarsSUMSVersion(status)
		if err != nil {
			logger.Err(err).Msgf("failed to update car sums version for manifest with CarUpdateID %d", status.CarUpdateID)
			err = nil
		}

		// Send the read_ecu_versions remote command so that the ECU data is updated in postgres ASAP
		client := services.RedisClientPool().GetFromPool()
		defer client.Close()
		err = client.SafePublishMessage(
			common.TRex.Key(vin),
			common.Message{
				Handler: "read_ecu_versions",
				Data: common.RemoteReadVersionsCommandArgs{
					ECUName: "*",
				},
			},
		)
		if err != nil {
			logger.Err(err).Msgf("failed to send read_ecu_versions command to vin %s", vin)
			err = nil
		}
	}

	if final {
		// if the manifest is in a final state
		// then delete the redundant requirements_await rows from car_update_statuses, to avoid overcrowding the table
		err = cu.truncateRequirementsAwaitForUpdate(status)
		if err != nil {
			logger.Err(err).Msgf("failed to delete redundant requirements_await rows from car_update_statuses for manifest with CarUpdateID %d", status.CarUpdateID)
			err = nil
		}
	}

	return err
}

func (cu *CarUpdateProgress) truncateRequirementsAwaitForUpdate(status *common.CarUpdateProgress) error {
	logger.Info().Msgf("Manifest with CarUpdateID %d successful with status %s. Deleting redundant requirements_await rows from car_update_statuses", status.CarUpdateID, status.Status)

	_, err := cu.DB.GetCarUpdates().TruncateRequirementsAwaitForUpdate(status.CarUpdateID)
	if err != nil && !errors.Is(err, pg.ErrNoRows) {
		return err
	}

	return nil
}

// Find the car update, and it gives the update manifest
// If the manifest has a sums version, apply it to the car
func (cu *CarUpdateProgress) updateCarsSUMSVersion(status *common.CarUpdateProgress) (err error) {
	carUpdatesDB := cu.DB.GetCarUpdates()
	carUpdate, err := carUpdatesDB.SelectByID(status.CarUpdateID)
	if err != nil {
		err = errors.WithStack(err)
		return
	}

	if carUpdate.UpdateManifest == nil {
		err = errors.New("failed to pull car updates update manifest")
		return
	}

	um := carUpdate.UpdateManifest
	// So if we have have a sums version we want to update
	if um.SUMS == "" {
		return
	}

	carsDB := cu.DB.GetCars()
	filter := common.Car{
		VIN: carUpdate.VIN,
	}

	cars, err := carsDB.Select(&filter, nil)
	if err != nil {
		err = errors.WithStack(err)
		return
	}

	if len(cars) != 1 {
		err = fmt.Errorf("did not receive only one car, received: %d", len(cars))
		err = errors.WithStack(err)
		return
	}

	car := cars[0]
	car.SUMSVersion = um.SUMS
	_, err = carsDB.Update(&car)
	if err != nil {
		err = errors.WithStack(err)
	}
	return
}