cloud-services/pkg/tmobile/queue.go

package tmobile

import (
	"context"
	"encoding/json"
	"time"

	"fiskerinc.com/modules/grpc/kafka_grpc"

	"fiskerinc.com/modules/kafka"
	"fiskerinc.com/modules/logger"
	"fiskerinc.com/modules/utils/envtool"
	"github.com/pkg/errors"
	"google.golang.org/protobuf/proto"
)

var (
	TMOBILE_MESSAGE_RETRY_MILLISECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_RETRY_MILLISECONDS", 300)
	TMOBILE_MESSAGE_RETRY_ATTEMPTS     int = envtool.GetEnvInt("TMOBILE_MESSAGE_RETRY_ATTEMPTS", 3) // Number of times we will try to send a message before giving up
	TMOBILE_MESSAGE_TIMEOUT_SECONDS    int = envtool.GetEnvInt("TMOBILE_MESSAGE_TIMEOUT_SECONDS", 10)
	TMOBILE_MESSAGE_CHECK_MILLISECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_CHECK_MILLISECONDS", 500) // About how often we want to check on a message

	// If we need to send even more messages at one time, I would suggest using an external message queue from either redis or maybe even kafka
	TMOBILE_MESSAGE_QUEUE_SIZE int = envtool.GetEnvInt("TMOBILE_MESSAGE_QUEUE_SIZE", 5000) // Number of messages we can wait on at one time
	TMOBILE_READ_THREADS       int = envtool.GetEnvInt("TMOBILE_READ_THREADS", 3)          // Number of goroutines that will be used to check on tmboile messages
)

// When we send an sms, we want to send it off, and then check on it every second to see if it has been delivered yet.
// After it has been delivered, we send a message in kafka to say that that message is done, and continue on
// This is for messages that we don't care if we deliver the message right away.
// Like if we are waking the car up for an update, we don't we can wait for the car to wake up nicely
// for remote commands, we want the car to be awoken and we return right away. Can maybe be its own faster queue

type RunningQueue struct {
	client        TMobClienter
	messageQueues []chan messageTimeout // Need to lock and unlock this array
	toSendThread  int                   // which thread we will try to send to
	kafka         kafka.ProducerInterface
}

type messageTimeout struct {
	MessageID string
	Timeout   time.Time
	NextCheck time.Time
}

func NewQueue(client TMobClienter, kafkaProducer kafka.ProducerInterface) (rq *RunningQueue) {
	rq = &RunningQueue{}
	rq.client = client
	rq.kafka = kafkaProducer

	for x := 0; x < TMOBILE_READ_THREADS; x++ {
		rq.messageQueues = append(rq.messageQueues, make(chan messageTimeout, TMOBILE_MESSAGE_QUEUE_SIZE))
		go rq.queueChecker(x)
	}

	return
}

type SendSMS struct {
	Message            string
	ICCID              string // The target of our message
	CheckDelivery      bool   // Do we care if this message is delivered or not
	KafkaServiceTarget string // The kafka service that will get the sms delivery status
}

// try to send message, if we fail to even send the message, that can be notified back right away
// then we thread out of here to wait for actual message delivery checks
func (rq *RunningQueue) SendSMS(ctx context.Context, req *SendSMSRequest) (messageID string, err error) {
	msg := SendSMS{
		Message:            req.MessageText,
		ICCID:              req.ICCID,
		CheckDelivery:      req.await,
		KafkaServiceTarget: req.KafkaServiceTarget,
	}
	messageID, err = rq.sendSMS(msg)
	if err != nil {
		return
	}

	// If we don't want to await for the message delivery, we will skip checking on the message
	if !req.await {
		return
	}

	queueMSG := messageTimeout{
		MessageID: messageID,
		Timeout:   time.Now().Add(time.Second * time.Duration(TMOBILE_MESSAGE_TIMEOUT_SECONDS)),
		NextCheck: time.Now().Add(time.Millisecond * time.Duration(TMOBILE_MESSAGE_CHECK_MILLISECONDS)),
	}
	rq.addToQueue(queueMSG)

	return
}

func (rq *RunningQueue) addToQueue(msg messageTimeout) {
	rq.messageQueues[rq.toSendThread] <- msg

	// Lightweight round robin, due to the multithreadedness, we might not send one message to each queue, but I am fine with that
	rq.toSendThread = (rq.toSendThread + 1) % TMOBILE_READ_THREADS
}

// Send down the kafka line if this message has been successfully delivered or not
// This should probably be moved outside of tmobile and into the sms service
func (rq *RunningQueue) UpdateMessageStatusKafka(messageID string, status MessageStatusEnum) {
	// We want to use grpc, but the other attendant services would also need a conversion
	/* msgGRPC := kafka_grpc.SMSStatus{
		MessageID: messageID,
		Status:    0,
	}
	//grpcCanSignal := cansignal.ToGrpc(batch)
	grpcData, err := proto.Marshal(&msgGRPC)
	if err != nil {
		logger.Err(err).Msgf("failed to protobuff sms message id %s", messageID)
		return
	} */

	msg := &kafka_grpc.GRPC_AttendantPayload_MessageStatus{
		MessageStatus: &kafka_grpc.MessageStatus{
			MessageId: messageID,
			Status:    kafka_grpc.EmumStatus(kafka_grpc.EmumStatus_value[string(status)]),
		},
	}
	kafkaMSG := kafka_grpc.GRPC_AttendantPayload{
		Handler: "sms_delivery_status_manifest",
		Data:    msg,
	}

	binaryPayload, _ := proto.Marshal(&kafkaMSG)
	err := rq.kafka.ProduceBinary(kafka.AttendantServiceGRPCKafka, "", binaryPayload, nil)
	if err != nil {
		err = errors.WithStack(err)
		logger.Err(err).Msgf("failed to produce kafka message for sms id %s", messageID)
	}
}

func (rq *RunningQueue) sendSMS(msg SendSMS) (messageID string, err error) {
	in := SendSMSRequest{
		ICCID:       msg.ICCID,
		MessageText: msg.Message,
	}

	var smr *SendSMSResponse
	// Giving 3 tries for a gateway failure
	// Sometimes we fail to deliver a message to tmobile for some reason, so we have to retry sending the message
	for x := 0; x < 3; x++ {
		smr, err = rq.client.SendSMS(context.Background(), &in)
		if err != nil {
			if errors.Is(err, ErrBadGatewayCode) {
				logger.Warn().Err(err).Msgf("ICCID %s gateway failure sending sms", in.ICCID)
				time.Sleep(time.Millisecond * time.Duration(TMOBILE_MESSAGE_RETRY_MILLISECONDS)) // 300 milliseconds seems to work best for me
				continue
			}
			if errors.Is(err, ErrAccessTokenExpired) {
				logger.Err(err).Msgf("Access token for sms expired, should never happen")
			}
			return "", errors.WithMessagef(err, "failed to send SMS to ICCID: %s", in.ICCID)
		}
		break
	}

	if smr == nil {
		return "", errors.WithMessagef(err, "failed to send SMS to ICCID: %s", in.ICCID)
	}

	messageID = smr.SmsMessageID
	return
}

// Continually read from the queue and check the status of the given message
func (rq *RunningQueue) queueChecker(index int) {
	messageQueue := rq.messageQueues[index]
	for msg := range messageQueue {
		// This will only happen when we are not sending messages
		if !time.Now().After(msg.NextCheck) {
			time.Sleep(time.Until(msg.NextCheck))
		}
		logger.Debug().Msgf("Checking msgID %s for delivery status", msg.MessageID)
		go func(msg messageTimeout) {
			delivered, err := rq.checkIfMessageDelivered(msg.MessageID)
			if err == nil {
				if delivered {
					rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_DELIVERED)
					return
				}
				// Not delivered, but not an error
				if time.Now().After(msg.Timeout) {
					rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_TIMEOUT)
					return
				}
				// If we get here, we need to check on the message again
				msg.NextCheck = time.Now().Add(time.Millisecond * time.Duration(TMOBILE_MESSAGE_CHECK_MILLISECONDS))
				rq.addToQueue(msg)
			} else {
				rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_FAILED)
			}
		}(msg)
	}
}

func (rq *RunningQueue) checkIfMessageDelivered(msgID string) (delivered bool, err error) {
	if !IsRealSMSID(msgID) {
		return true, nil
	}

	var out *SMSDetailsResponse
	for x := 0; x < 3; x++ {
		out, err = rq.client.Details(context.Background(), msgID)
		if err != nil {
			if errors.Is(err, ErrBadGatewayCode) {
				logger.Warn().Err(err).Msgf("MSG id %s gateway failure sending sms", msgID)
				time.Sleep(time.Millisecond * time.Duration(TMOBILE_MESSAGE_RETRY_MILLISECONDS))
				continue
			}
			if errors.Is(err, ErrAccessTokenExpired) {
				logger.Err(err).Msgf("Access token for sms expired, should never happen")
			}

			return false, errors.WithMessagef(err, "failed to check status of SMS: %s", msgID)
		}
	}

	switch out.Status {
	case Pending, CancelPending:
		return false, nil
	case Delivered:
		return true, nil
	default:
		logger.Warn().Msgf("Fell through to default of sms status id: %s status: %s", msgID, out.Status)
		return false, errors.WithMessagef(
			ErrBadMsgStatus,
			"message with id %s failed with status %s",
			out.SmsMsgID,
			out.Status,
		)
	}
}

type MessageStatus struct {
	MessageID string            `json:"MessageID"`
	Status    MessageStatusEnum `json:"Status"`
}

func GRPCToTMessage(payload *kafka_grpc.GRPC_AttendantPayload) []byte {
	if payload.Data == nil {
		return nil
	}
	data := payload.Data.(*kafka_grpc.GRPC_AttendantPayload_MessageStatus)
	msg := MessageStatus{
		MessageID: data.MessageStatus.GetMessageId(),
		Status:    MessageStatusEnum(data.MessageStatus.GetStatus().String()),
	}
	bytes, _ := json.Marshal(msg)
	return bytes
}

type MessageStatusEnum string

const (
	MESSAGE_STATUS_DELIVERED MessageStatusEnum = "DELIVERED"
	MESSAGE_STATUS_TIMEOUT   MessageStatusEnum = "TIMEOUT"
	MESSAGE_STATUS_FAILED    MessageStatusEnum = "FAILED"
)