Initial cloud-services repo - gateway service + pkg modules

pkg/kafka/async_producer.go

@@ -0,0 +1,170 @@
+package kafka
+
+import (
+	"context"
+	"encoding/json"
+
+	"fiskerinc.com/modules/logger"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+	"github.com/pkg/errors"
+)
+
+type AsyncProducer struct {
+	producer *kafka.Producer
+}
+
+// NewProducer serves as factory method for producer to kafka
+func NewAsyncProducer(ctx context.Context) (ProducerInterface, error) {
+	var producer *kafka.Producer
+	configuration := loadKafkaProducerConfig()
+	producer, err := kafka.NewProducer(
+		&configuration,
+		// kafkaTrace.WithContext(ctx),
+	)
+	logger.Info().Msgf("NewAsyncProducer  hosts: %s", kafkaHosts)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return &AsyncProducer{producer: producer}, nil
+}
+
+// SetProducer sets the producer instance
+func (p *AsyncProducer) SetProducer(producer *kafka.Producer) {
+	p.producer = producer
+}
+
+// Len returns len of messages in queue.
+func (p *AsyncProducer) Len() int {
+	return p.producer.Len()
+}
+
+// Flush calls producer's Flush function.
+func (p *AsyncProducer) Flush(timeoutMs int) int {
+	return p.producer.Flush(timeoutMs)
+}
+
+// Produce sends a Kafka Message to Kafka
+func (p *AsyncProducer) Produce(topic string, key string, payload interface{}, headers map[string][]byte) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return p.ProduceBinary(topic, key, v, headers)
+}
+
+func (p *AsyncProducer) makeHeaders(headers map[string][]byte) []kafka.Header {
+	if headers == nil {
+		return nil
+	}
+
+	i := 0
+	result := make([]kafka.Header, len(headers))
+
+	for key, value := range headers {
+		result[i] = kafka.Header{
+			Key: key,
+		}
+		copy(result[i].Value, value)
+		i++
+	}
+
+	return result
+}
+
+func (p *AsyncProducer) ProduceBinary(topic string, key string, data []byte, headers map[string][]byte) error {
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          data,
+	}
+	if headers != nil {
+		km.Headers = p.makeHeaders(headers)
+	}
+
+	err := p.producer.Produce(&km, nil)
+	if err != nil {
+		// error handle inability to connect to kafka
+		return errors.WithStack(err)
+	}
+
+	return err
+}
+
+// ProduceToChannel sends a list of Kafka Messages to Kafka
+func (p *AsyncProducer) ProduceToChannel(topic string, key string, payload interface{}) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          v,
+	}
+
+	p.producer.ProduceChannel() <- &km
+	return nil
+}
+
+func (p *AsyncProducer) ProduceBinaryToChannel(topic string, key string, payload []byte) error {
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          payload,
+	}
+
+	p.producer.ProduceChannel() <- &km
+	return nil
+}
+
+// ProduceSignalBatch sends a specified batch of CAN signals
+//
+//	NOTE: does NOT produce proper JSON, Value is custom for Clickhouse
+func (p *AsyncProducer) ProduceSignalBatch(topic string, key string, payload interface{}) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          v[1 : len(v)-1],
+	}
+
+	err = p.producer.Produce(&km, nil)
+	if err != nil {
+		// error handle inability to connect to kafka
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+// ReadEvents listens to producer's events which are emitted as responses to producing events
+func (p *AsyncProducer) ReadEvents() {
+	for e := range p.producer.Events() {
+		switch m := e.(type) {
+		case *kafka.Message:
+			if m.TopicPartition.Error != nil {
+				logger.Error().
+					Err(m.TopicPartition.Error).
+					Send()
+			}
+		case kafka.Error:
+			logger.Error().
+				Str("err", m.String()).
+				Send()
+		}
+	}
+}
+
+// Close the Kafka Producer
+func (p *AsyncProducer) Close() {
+	p.producer.Flush(0)
+	p.producer.Close()
+	p.producer = nil
+}