Initial cloud-services repo - gateway service + pkg modules
This commit is contained in:
Chris Rai
263
pkg/tmobile/queue.go
Normal file
263
pkg/tmobile/queue.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package tmobile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"fiskerinc.com/modules/grpc/kafka_grpc"
|
||||
|
||||
"fiskerinc.com/modules/kafka"
|
||||
"fiskerinc.com/modules/logger"
|
||||
"fiskerinc.com/modules/utils/envtool"
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
var (
|
||||
TMOBILE_MESSAGE_RETRY_MILLISECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_RETRY_MILLISECONDS", 300)
|
||||
TMOBILE_MESSAGE_RETRY_ATTEMPTS int = envtool.GetEnvInt("TMOBILE_MESSAGE_RETRY_ATTEMPTS", 3) // Number of times we will try to send a message before giving up
|
||||
TMOBILE_MESSAGE_TIMEOUT_SECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_TIMEOUT_SECONDS", 10)
|
||||
TMOBILE_MESSAGE_CHECK_MILLISECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_CHECK_MILLISECONDS", 500) // About how often we want to check on a message
|
||||
|
||||
// If we need to send even more messages at one time, I would suggest using an external message queue from either redis or maybe even kafka
|
||||
TMOBILE_MESSAGE_QUEUE_SIZE int = envtool.GetEnvInt("TMOBILE_MESSAGE_QUEUE_SIZE", 5000) // Number of messages we can wait on at one time
|
||||
TMOBILE_READ_THREADS int = envtool.GetEnvInt("TMOBILE_READ_THREADS", 3) // Number of goroutines that will be used to check on tmboile messages
|
||||
)
|
||||
|
||||
// When we send an sms, we want to send it off, and then check on it every second to see if it has been delivered yet.
|
||||
// After it has been delivered, we send a message in kafka to say that that message is done, and continue on
|
||||
// This is for messages that we don't care if we deliver the message right away.
|
||||
// Like if we are waking the car up for an update, we don't we can wait for the car to wake up nicely
|
||||
// for remote commands, we want the car to be awoken and we return right away. Can maybe be its own faster queue
|
||||
|
||||
type RunningQueue struct {
|
||||
client TMobClienter
|
||||
messageQueues []chan messageTimeout // Need to lock and unlock this array
|
||||
toSendThread int // which thread we will try to send to
|
||||
kafka kafka.ProducerInterface
|
||||
}
|
||||
|
||||
type messageTimeout struct {
|
||||
MessageID string
|
||||
Timeout time.Time
|
||||
NextCheck time.Time
|
||||
}
|
||||
|
||||
func NewQueue(client TMobClienter, kafkaProducer kafka.ProducerInterface) (rq *RunningQueue) {
|
||||
rq = &RunningQueue{}
|
||||
rq.client = client
|
||||
rq.kafka = kafkaProducer
|
||||
|
||||
for x := 0; x < TMOBILE_READ_THREADS; x++ {
|
||||
rq.messageQueues = append(rq.messageQueues, make(chan messageTimeout, TMOBILE_MESSAGE_QUEUE_SIZE))
|
||||
go rq.queueChecker(x)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type SendSMS struct {
|
||||
Message string
|
||||
ICCID string // The target of our message
|
||||
CheckDelivery bool // Do we care if this message is delivered or not
|
||||
KafkaServiceTarget string // The kafka service that will get the sms delivery status
|
||||
}
|
||||
|
||||
// try to send message, if we fail to even send the message, that can be notified back right away
|
||||
// then we thread out of here to wait for actual message delivery checks
|
||||
func (rq *RunningQueue) SendSMS(ctx context.Context, req *SendSMSRequest) (messageID string, err error) {
|
||||
msg := SendSMS{
|
||||
Message: req.MessageText,
|
||||
ICCID: req.ICCID,
|
||||
CheckDelivery: req.await,
|
||||
KafkaServiceTarget: req.KafkaServiceTarget,
|
||||
}
|
||||
messageID, err = rq.sendSMS(msg)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// If we don't want to await for the message delivery, we will skip checking on the message
|
||||
if !req.await {
|
||||
return
|
||||
}
|
||||
|
||||
queueMSG := messageTimeout{
|
||||
MessageID: messageID,
|
||||
Timeout: time.Now().Add(time.Second * time.Duration(TMOBILE_MESSAGE_TIMEOUT_SECONDS)),
|
||||
NextCheck: time.Now().Add(time.Millisecond * time.Duration(TMOBILE_MESSAGE_CHECK_MILLISECONDS)),
|
||||
}
|
||||
rq.addToQueue(queueMSG)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (rq *RunningQueue) addToQueue(msg messageTimeout) {
|
||||
rq.messageQueues[rq.toSendThread] <- msg
|
||||
|
||||
// Lightweight round robin, due to the multithreadedness, we might not send one message to each queue, but I am fine with that
|
||||
rq.toSendThread = (rq.toSendThread + 1) % TMOBILE_READ_THREADS
|
||||
}
|
||||
|
||||
// Send down the kafka line if this message has been successfully delivered or not
|
||||
// This should probably be moved outside of tmobile and into the sms service
|
||||
func (rq *RunningQueue) UpdateMessageStatusKafka(messageID string, status MessageStatusEnum) {
|
||||
// We want to use grpc, but the other attendant services would also need a conversion
|
||||
/* msgGRPC := kafka_grpc.SMSStatus{
|
||||
MessageID: messageID,
|
||||
Status: 0,
|
||||
}
|
||||
//grpcCanSignal := cansignal.ToGrpc(batch)
|
||||
grpcData, err := proto.Marshal(&msgGRPC)
|
||||
if err != nil {
|
||||
logger.Err(err).Msgf("failed to protobuff sms message id %s", messageID)
|
||||
return
|
||||
} */
|
||||
|
||||
msg := &kafka_grpc.GRPC_AttendantPayload_MessageStatus{
|
||||
MessageStatus: &kafka_grpc.MessageStatus{
|
||||
MessageId: messageID,
|
||||
Status: kafka_grpc.EmumStatus(kafka_grpc.EmumStatus_value[string(status)]),
|
||||
},
|
||||
}
|
||||
kafkaMSG := kafka_grpc.GRPC_AttendantPayload{
|
||||
Handler: "sms_delivery_status_manifest",
|
||||
Data: msg,
|
||||
}
|
||||
|
||||
binaryPayload, _ := proto.Marshal(&kafkaMSG)
|
||||
err := rq.kafka.ProduceBinary(kafka.AttendantServiceGRPCKafka, "", binaryPayload, nil)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
logger.Err(err).Msgf("failed to produce kafka message for sms id %s", messageID)
|
||||
}
|
||||
}
|
||||
|
||||
func (rq *RunningQueue) sendSMS(msg SendSMS) (messageID string, err error) {
|
||||
in := SendSMSRequest{
|
||||
ICCID: msg.ICCID,
|
||||
MessageText: msg.Message,
|
||||
}
|
||||
|
||||
var smr *SendSMSResponse
|
||||
// Giving 3 tries for a gateway failure
|
||||
// Sometimes we fail to deliver a message to tmobile for some reason, so we have to retry sending the message
|
||||
for x := 0; x < 3; x++ {
|
||||
smr, err = rq.client.SendSMS(context.Background(), &in)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrBadGatewayCode) {
|
||||
logger.Warn().Err(err).Msgf("ICCID %s gateway failure sending sms", in.ICCID)
|
||||
time.Sleep(time.Millisecond * time.Duration(TMOBILE_MESSAGE_RETRY_MILLISECONDS)) // 300 milliseconds seems to work best for me
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, ErrAccessTokenExpired) {
|
||||
logger.Err(err).Msgf("Access token for sms expired, should never happen")
|
||||
}
|
||||
return "", errors.WithMessagef(err, "failed to send SMS to ICCID: %s", in.ICCID)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if smr == nil {
|
||||
return "", errors.WithMessagef(err, "failed to send SMS to ICCID: %s", in.ICCID)
|
||||
}
|
||||
|
||||
messageID = smr.SmsMessageID
|
||||
return
|
||||
}
|
||||
|
||||
// Continually read from the queue and check the status of the given message
|
||||
func (rq *RunningQueue) queueChecker(index int) {
|
||||
messageQueue := rq.messageQueues[index]
|
||||
for msg := range messageQueue {
|
||||
// This will only happen when we are not sending messages
|
||||
if !time.Now().After(msg.NextCheck) {
|
||||
time.Sleep(time.Until(msg.NextCheck))
|
||||
}
|
||||
logger.Debug().Msgf("Checking msgID %s for delivery status", msg.MessageID)
|
||||
go func(msg messageTimeout) {
|
||||
delivered, err := rq.checkIfMessageDelivered(msg.MessageID)
|
||||
if err == nil {
|
||||
if delivered {
|
||||
rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_DELIVERED)
|
||||
return
|
||||
}
|
||||
// Not delivered, but not an error
|
||||
if time.Now().After(msg.Timeout) {
|
||||
rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_TIMEOUT)
|
||||
return
|
||||
}
|
||||
// If we get here, we need to check on the message again
|
||||
msg.NextCheck = time.Now().Add(time.Millisecond * time.Duration(TMOBILE_MESSAGE_CHECK_MILLISECONDS))
|
||||
rq.addToQueue(msg)
|
||||
} else {
|
||||
rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_FAILED)
|
||||
}
|
||||
}(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (rq *RunningQueue) checkIfMessageDelivered(msgID string) (delivered bool, err error) {
|
||||
if !IsRealSMSID(msgID) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var out *SMSDetailsResponse
|
||||
for x := 0; x < 3; x++ {
|
||||
out, err = rq.client.Details(context.Background(), msgID)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrBadGatewayCode) {
|
||||
logger.Warn().Err(err).Msgf("MSG id %s gateway failure sending sms", msgID)
|
||||
time.Sleep(time.Millisecond * time.Duration(TMOBILE_MESSAGE_RETRY_MILLISECONDS))
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, ErrAccessTokenExpired) {
|
||||
logger.Err(err).Msgf("Access token for sms expired, should never happen")
|
||||
}
|
||||
|
||||
return false, errors.WithMessagef(err, "failed to check status of SMS: %s", msgID)
|
||||
}
|
||||
}
|
||||
|
||||
switch out.Status {
|
||||
case Pending, CancelPending:
|
||||
return false, nil
|
||||
case Delivered:
|
||||
return true, nil
|
||||
default:
|
||||
logger.Warn().Msgf("Fell through to default of sms status id: %s status: %s", msgID, out.Status)
|
||||
return false, errors.WithMessagef(
|
||||
ErrBadMsgStatus,
|
||||
"message with id %s failed with status %s",
|
||||
out.SmsMsgID,
|
||||
out.Status,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
type MessageStatus struct {
|
||||
MessageID string `json:"MessageID"`
|
||||
Status MessageStatusEnum `json:"Status"`
|
||||
}
|
||||
|
||||
func GRPCToTMessage(payload *kafka_grpc.GRPC_AttendantPayload) []byte {
|
||||
if payload.Data == nil {
|
||||
return nil
|
||||
}
|
||||
data := payload.Data.(*kafka_grpc.GRPC_AttendantPayload_MessageStatus)
|
||||
msg := MessageStatus{
|
||||
MessageID: data.MessageStatus.GetMessageId(),
|
||||
Status: MessageStatusEnum(data.MessageStatus.GetStatus().String()),
|
||||
}
|
||||
bytes, _ := json.Marshal(msg)
|
||||
return bytes
|
||||
}
|
||||
|
||||
type MessageStatusEnum string
|
||||
|
||||
const (
|
||||
MESSAGE_STATUS_DELIVERED MessageStatusEnum = "DELIVERED"
|
||||
MESSAGE_STATUS_TIMEOUT MessageStatusEnum = "TIMEOUT"
|
||||
MESSAGE_STATUS_FAILED MessageStatusEnum = "FAILED"
|
||||
)
|
||||
Reference in New Issue
Block a user