Add depot, attendant, jetfire, optimus, ota services with kustomize overlays

pkg/kafka/consumer.go

@@ -52,6 +52,8 @@ type ConsumerInterface interface {
 	ConsumeToChannel(topics []string, events chan *Message) error
 	ConsumeToChannelJson(topics []string, events chan common.EventRawJSON) error
 	ConsumePartitionsToChannel(partitions []TopicPartition, events chan *Message) error
+	ConsumeOrRebalancedCatch(topics []string, events chan *Message, rebalance chan struct{}) error
+	Subscribe(topics []string)
 	GetMetadata(topic string) (*Metadata, error)
 	Check(ctx context.Context) error
 	Stop()
@@ -275,6 +277,48 @@ func (c *Consumer) Stop() {
 	}
 }
 
+// Subscribe adds topics to consume (for compatibility with old API)
+func (c *Consumer) Subscribe(topics []string) {
+	c.client.AddConsumeTopics(topics...)
+}
+
+// ConsumeOrRebalancedCatch consumes messages and notifies on rebalance events
+func (c *Consumer) ConsumeOrRebalancedCatch(topics []string, events chan *Message, rebalance chan struct{}) error {
+	c.client.AddConsumeTopics(topics...)
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	c.running = true
+	for c.running {
+		fetches := c.client.PollFetches(c.ctx)
+		if errs := fetches.Errors(); len(errs) > 0 {
+			for _, e := range errs {
+				if e.Err == context.Canceled {
+					return nil
+				}
+				// Check for rebalance-related errors
+				if e.Err.Error() == "REBALANCE_IN_PROGRESS" || e.Err.Error() == "NOT_COORDINATOR" {
+					select {
+					case rebalance <- struct{}{}:
+					default:
+					}
+					return e.Err
+				}
+				logger.Error().Err(e.Err).Msgf("fetch error on %s", e.Topic)
+				c.setConnected(false)
+			}
+			continue
+		}
+
+		c.setConnected(true)
+		fetches.EachRecord(func(r *kgo.Record) {
+			events <- recordToMessage(r)
+		})
+	}
+
+	return nil
+}
+
 // Check verifies consumer connectivity
 func (c *Consumer) Check(ctx context.Context) error {
 	if !c.isConnected() {