Add depot, attendant, jetfire, optimus, ota services with kustomize overlays

pkg/kafka/consumer.go

@@ -52,6 +52,8 @@ type ConsumerInterface interface {
 	ConsumeToChannel(topics []string, events chan *Message) error
 	ConsumeToChannelJson(topics []string, events chan common.EventRawJSON) error
 	ConsumePartitionsToChannel(partitions []TopicPartition, events chan *Message) error
+	ConsumeOrRebalancedCatch(topics []string, events chan *Message, rebalance chan struct{}) error
+	Subscribe(topics []string)
 	GetMetadata(topic string) (*Metadata, error)
 	Check(ctx context.Context) error
 	Stop()
@@ -275,6 +277,48 @@ func (c *Consumer) Stop() {
 	}
 }
 
+// Subscribe adds topics to consume (for compatibility with old API)
+func (c *Consumer) Subscribe(topics []string) {
+	c.client.AddConsumeTopics(topics...)
+}
+
+// ConsumeOrRebalancedCatch consumes messages and notifies on rebalance events
+func (c *Consumer) ConsumeOrRebalancedCatch(topics []string, events chan *Message, rebalance chan struct{}) error {
+	c.client.AddConsumeTopics(topics...)
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	c.running = true
+	for c.running {
+		fetches := c.client.PollFetches(c.ctx)
+		if errs := fetches.Errors(); len(errs) > 0 {
+			for _, e := range errs {
+				if e.Err == context.Canceled {
+					return nil
+				}
+				// Check for rebalance-related errors
+				if e.Err.Error() == "REBALANCE_IN_PROGRESS" || e.Err.Error() == "NOT_COORDINATOR" {
+					select {
+					case rebalance <- struct{}{}:
+					default:
+					}
+					return e.Err
+				}
+				logger.Error().Err(e.Err).Msgf("fetch error on %s", e.Topic)
+				c.setConnected(false)
+			}
+			continue
+		}
+
+		c.setConnected(true)
+		fetches.EachRecord(func(r *kgo.Record) {
+			events <- recordToMessage(r)
+		})
+	}
+
+	return nil
+}
+
 // Check verifies consumer connectivity
 func (c *Consumer) Check(ctx context.Context) error {
 	if !c.isConnected() {

pkg/loggerdataresp/data_error_resp.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/fiskerinc/cloud-services/pkg/logger"
 	"github.com/fiskerinc/cloud-services/pkg/utils"
-	cKafka "github.com/confluentinc/confluent-kafka-go/v2/kafka"
 	"github.com/go-pg/pg/v10"
 	"github.com/pkg/errors"
 )
@@ -106,10 +105,6 @@ func badDataErrorResp(w http.ResponseWriter, err error, defaultStatus int, errFu
 		return true
 	}
 
-	if handleKafkaErrors(err, errFunc, w) {
-		return true
-	}
-
 	logError(err, err.Error(), defaultStatus, errFunc, w)
 
 	return true
@@ -169,17 +164,6 @@ func handleSAPErrors(err error, errFunc respErr, w http.ResponseWriter) bool {
 	return false
 }
 
-func handleKafkaErrors(err error, errFunc respErr, w http.ResponseWriter) bool {
-	if kafkaErr, ok := err.(cKafka.Error); ok {
-		if kafkaErr.IsFatal() {
-			logError(kafkaErr, kafkaErr.Error(), http.StatusServiceUnavailable, errFunc, w)
-			return true
-		}
-	}
-
-	return false
-}
-
 func logError(err error, errMessage string, status int, errFunc respErr, w http.ResponseWriter) {
 	switch status {
 	case http.StatusServiceUnavailable, http.StatusInternalServerError: