package tmobile import ( "context" "encoding/json" "time" "fiskerinc.com/modules/grpc/kafka_grpc" "fiskerinc.com/modules/kafka" "fiskerinc.com/modules/logger" "fiskerinc.com/modules/utils/envtool" "github.com/pkg/errors" "google.golang.org/protobuf/proto" ) var ( TMOBILE_MESSAGE_RETRY_MILLISECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_RETRY_MILLISECONDS", 300) TMOBILE_MESSAGE_RETRY_ATTEMPTS int = envtool.GetEnvInt("TMOBILE_MESSAGE_RETRY_ATTEMPTS", 3) // Number of times we will try to send a message before giving up TMOBILE_MESSAGE_TIMEOUT_SECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_TIMEOUT_SECONDS", 10) TMOBILE_MESSAGE_CHECK_MILLISECONDS int = envtool.GetEnvInt("TMOBILE_MESSAGE_CHECK_MILLISECONDS", 500) // About how often we want to check on a message // If we need to send even more messages at one time, I would suggest using an external message queue from either redis or maybe even kafka TMOBILE_MESSAGE_QUEUE_SIZE int = envtool.GetEnvInt("TMOBILE_MESSAGE_QUEUE_SIZE", 5000) // Number of messages we can wait on at one time TMOBILE_READ_THREADS int = envtool.GetEnvInt("TMOBILE_READ_THREADS", 3) // Number of goroutines that will be used to check on tmboile messages ) // When we send an sms, we want to send it off, and then check on it every second to see if it has been delivered yet. // After it has been delivered, we send a message in kafka to say that that message is done, and continue on // This is for messages that we don't care if we deliver the message right away. // Like if we are waking the car up for an update, we don't we can wait for the car to wake up nicely // for remote commands, we want the car to be awoken and we return right away. Can maybe be its own faster queue type RunningQueue struct { client TMobClienter messageQueues []chan messageTimeout // Need to lock and unlock this array toSendThread int // which thread we will try to send to kafka kafka.ProducerInterface } type messageTimeout struct { MessageID string Timeout time.Time NextCheck time.Time } func NewQueue(client TMobClienter, kafkaProducer kafka.ProducerInterface) (rq *RunningQueue) { rq = &RunningQueue{} rq.client = client rq.kafka = kafkaProducer for x := 0; x < TMOBILE_READ_THREADS; x++ { rq.messageQueues = append(rq.messageQueues, make(chan messageTimeout, TMOBILE_MESSAGE_QUEUE_SIZE)) go rq.queueChecker(x) } return } type SendSMS struct { Message string ICCID string // The target of our message CheckDelivery bool // Do we care if this message is delivered or not KafkaServiceTarget string // The kafka service that will get the sms delivery status } // try to send message, if we fail to even send the message, that can be notified back right away // then we thread out of here to wait for actual message delivery checks func (rq *RunningQueue) SendSMS(ctx context.Context, req *SendSMSRequest) (messageID string, err error) { msg := SendSMS{ Message: req.MessageText, ICCID: req.ICCID, CheckDelivery: req.await, KafkaServiceTarget: req.KafkaServiceTarget, } messageID, err = rq.sendSMS(msg) if err != nil { return } // If we don't want to await for the message delivery, we will skip checking on the message if !req.await { return } queueMSG := messageTimeout{ MessageID: messageID, Timeout: time.Now().Add(time.Second * time.Duration(TMOBILE_MESSAGE_TIMEOUT_SECONDS)), NextCheck: time.Now().Add(time.Millisecond * time.Duration(TMOBILE_MESSAGE_CHECK_MILLISECONDS)), } rq.addToQueue(queueMSG) return } func (rq *RunningQueue) addToQueue(msg messageTimeout) { rq.messageQueues[rq.toSendThread] <- msg // Lightweight round robin, due to the multithreadedness, we might not send one message to each queue, but I am fine with that rq.toSendThread = (rq.toSendThread + 1) % TMOBILE_READ_THREADS } // Send down the kafka line if this message has been successfully delivered or not // This should probably be moved outside of tmobile and into the sms service func (rq *RunningQueue) UpdateMessageStatusKafka(messageID string, status MessageStatusEnum) { // We want to use grpc, but the other attendant services would also need a conversion /* msgGRPC := kafka_grpc.SMSStatus{ MessageID: messageID, Status: 0, } //grpcCanSignal := cansignal.ToGrpc(batch) grpcData, err := proto.Marshal(&msgGRPC) if err != nil { logger.Err(err).Msgf("failed to protobuff sms message id %s", messageID) return } */ msg := &kafka_grpc.GRPC_AttendantPayload_MessageStatus{ MessageStatus: &kafka_grpc.MessageStatus{ MessageId: messageID, Status: kafka_grpc.EmumStatus(kafka_grpc.EmumStatus_value[string(status)]), }, } kafkaMSG := kafka_grpc.GRPC_AttendantPayload{ Handler: "sms_delivery_status_manifest", Data: msg, } binaryPayload, _ := proto.Marshal(&kafkaMSG) err := rq.kafka.ProduceBinary(kafka.AttendantServiceGRPCKafka, "", binaryPayload, nil) if err != nil { err = errors.WithStack(err) logger.Err(err).Msgf("failed to produce kafka message for sms id %s", messageID) } } func (rq *RunningQueue) sendSMS(msg SendSMS) (messageID string, err error) { in := SendSMSRequest{ ICCID: msg.ICCID, MessageText: msg.Message, } var smr *SendSMSResponse // Giving 3 tries for a gateway failure // Sometimes we fail to deliver a message to tmobile for some reason, so we have to retry sending the message for x := 0; x < 3; x++ { smr, err = rq.client.SendSMS(context.Background(), &in) if err != nil { if errors.Is(err, ErrBadGatewayCode) { logger.Warn().Err(err).Msgf("ICCID %s gateway failure sending sms", in.ICCID) time.Sleep(time.Millisecond * time.Duration(TMOBILE_MESSAGE_RETRY_MILLISECONDS)) // 300 milliseconds seems to work best for me continue } if errors.Is(err, ErrAccessTokenExpired) { logger.Err(err).Msgf("Access token for sms expired, should never happen") } return "", errors.WithMessagef(err, "failed to send SMS to ICCID: %s", in.ICCID) } break } if smr == nil { return "", errors.WithMessagef(err, "failed to send SMS to ICCID: %s", in.ICCID) } messageID = smr.SmsMessageID return } // Continually read from the queue and check the status of the given message func (rq *RunningQueue) queueChecker(index int) { messageQueue := rq.messageQueues[index] for msg := range messageQueue { // This will only happen when we are not sending messages if !time.Now().After(msg.NextCheck) { time.Sleep(time.Until(msg.NextCheck)) } logger.Debug().Msgf("Checking msgID %s for delivery status", msg.MessageID) go func(msg messageTimeout) { delivered, err := rq.checkIfMessageDelivered(msg.MessageID) if err == nil { if delivered { rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_DELIVERED) return } // Not delivered, but not an error if time.Now().After(msg.Timeout) { rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_TIMEOUT) return } // If we get here, we need to check on the message again msg.NextCheck = time.Now().Add(time.Millisecond * time.Duration(TMOBILE_MESSAGE_CHECK_MILLISECONDS)) rq.addToQueue(msg) } else { rq.UpdateMessageStatusKafka(msg.MessageID, MESSAGE_STATUS_FAILED) } }(msg) } } func (rq *RunningQueue) checkIfMessageDelivered(msgID string) (delivered bool, err error) { if !IsRealSMSID(msgID) { return true, nil } var out *SMSDetailsResponse for x := 0; x < 3; x++ { out, err = rq.client.Details(context.Background(), msgID) if err != nil { if errors.Is(err, ErrBadGatewayCode) { logger.Warn().Err(err).Msgf("MSG id %s gateway failure sending sms", msgID) time.Sleep(time.Millisecond * time.Duration(TMOBILE_MESSAGE_RETRY_MILLISECONDS)) continue } if errors.Is(err, ErrAccessTokenExpired) { logger.Err(err).Msgf("Access token for sms expired, should never happen") } return false, errors.WithMessagef(err, "failed to check status of SMS: %s", msgID) } } switch out.Status { case Pending, CancelPending: return false, nil case Delivered: return true, nil default: logger.Warn().Msgf("Fell through to default of sms status id: %s status: %s", msgID, out.Status) return false, errors.WithMessagef( ErrBadMsgStatus, "message with id %s failed with status %s", out.SmsMsgID, out.Status, ) } } type MessageStatus struct { MessageID string `json:"MessageID"` Status MessageStatusEnum `json:"Status"` } func GRPCToTMessage(payload *kafka_grpc.GRPC_AttendantPayload) []byte { if payload.Data == nil { return nil } data := payload.Data.(*kafka_grpc.GRPC_AttendantPayload_MessageStatus) msg := MessageStatus{ MessageID: data.MessageStatus.GetMessageId(), Status: MessageStatusEnum(data.MessageStatus.GetStatus().String()), } bytes, _ := json.Marshal(msg) return bytes } type MessageStatusEnum string const ( MESSAGE_STATUS_DELIVERED MessageStatusEnum = "DELIVERED" MESSAGE_STATUS_TIMEOUT MessageStatusEnum = "TIMEOUT" MESSAGE_STATUS_FAILED MessageStatusEnum = "FAILED" )