Initial cloud-services repo - gateway service + pkg modules

pkg/kafka/admin.go

@@ -0,0 +1,55 @@
+package kafka
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"fiskerinc.com/modules/utils/envtool"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+	"github.com/pkg/errors"
+)
+
+// AdminClient interface
+type AdminClient interface {
+	CreateTopics(ctx context.Context, topics []kafka.TopicSpecification, options ...kafka.CreateTopicsAdminOption) ([]kafka.TopicResult, error)
+}
+
+// Global
+var (
+	Admin      AdminClient
+	kafkaHosts string = envtool.GetEnv("KAFKA_HOSTS", "localhost:9093")
+)
+
+func init() {
+	Admin, _ = kafka.NewAdminClient(&kafka.ConfigMap{"bootstrap.servers": kafkaHosts})
+}
+
+// EnsureTopicsExist checks Kafka for topic, if it doesn't exist it creates topic
+func EnsureTopicsExist(topics []string) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	s := fmt.Sprintf("%vs", envtool.GetEnvInt("KAFKA_TIMEOUT", 30))
+	maxDur, err := time.ParseDuration(s)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	ts := make([]kafka.TopicSpecification, len(topics))
+	for i, t := range topics {
+		ts[i] = kafka.TopicSpecification{
+			Topic:             t,
+			NumPartitions:     envtool.GetEnvInt("KAFKA_TOPIC_PARTITIONS", 1),
+			ReplicationFactor: envtool.GetEnvInt("KAFKA_TOPIC_REPLICATION_FACTOR", 1),
+		}
+	}
+
+	_, err = Admin.CreateTopics(ctx, ts, kafka.SetAdminOperationTimeout(maxDur))
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}

pkg/kafka/admin_test.go

@@ -0,0 +1,46 @@
+package kafka
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"fiskerinc.com/modules/kafka/mock"
+	"fiskerinc.com/modules/testhelper"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+)
+
+func TestAdmin(t *testing.T) {
+	a := mock.Admin{
+		MockFunc: func(ctx context.Context, topics []kafka.TopicSpecification, options ...kafka.CreateTopicsAdminOption) ([]kafka.TopicResult, error) {
+			return []kafka.TopicResult{{
+				Topic: "test",
+				Error: kafka.Error{},
+			}}, nil
+		},
+	}
+
+	Admin = &a
+
+	err := EnsureTopicsExist([]string{"test"})
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestAdmin", nil, err)
+	}
+}
+
+func TestAdminError(t *testing.T) {
+	testError := errors.New("Test error being thrown")
+	a := mock.Admin{
+		MockFunc: func(ctx context.Context, topics []kafka.TopicSpecification, options ...kafka.CreateTopicsAdminOption) ([]kafka.TopicResult, error) {
+			return []kafka.TopicResult{}, testError
+		},
+	}
+
+	Admin = &a
+
+	err := EnsureTopicsExist([]string{"test"})
+	if err.Error() != testError.Error() {
+		t.Errorf(testhelper.TestErrorTemplate, "TestAdmin", testError, err)
+	}
+}

pkg/kafka/async_producer.go

@@ -0,0 +1,170 @@
+package kafka
+
+import (
+	"context"
+	"encoding/json"
+
+	"fiskerinc.com/modules/logger"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+	"github.com/pkg/errors"
+)
+
+type AsyncProducer struct {
+	producer *kafka.Producer
+}
+
+// NewProducer serves as factory method for producer to kafka
+func NewAsyncProducer(ctx context.Context) (ProducerInterface, error) {
+	var producer *kafka.Producer
+	configuration := loadKafkaProducerConfig()
+	producer, err := kafka.NewProducer(
+		&configuration,
+		// kafkaTrace.WithContext(ctx),
+	)
+	logger.Info().Msgf("NewAsyncProducer  hosts: %s", kafkaHosts)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return &AsyncProducer{producer: producer}, nil
+}
+
+// SetProducer sets the producer instance
+func (p *AsyncProducer) SetProducer(producer *kafka.Producer) {
+	p.producer = producer
+}
+
+// Len returns len of messages in queue.
+func (p *AsyncProducer) Len() int {
+	return p.producer.Len()
+}
+
+// Flush calls producer's Flush function.
+func (p *AsyncProducer) Flush(timeoutMs int) int {
+	return p.producer.Flush(timeoutMs)
+}
+
+// Produce sends a Kafka Message to Kafka
+func (p *AsyncProducer) Produce(topic string, key string, payload interface{}, headers map[string][]byte) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return p.ProduceBinary(topic, key, v, headers)
+}
+
+func (p *AsyncProducer) makeHeaders(headers map[string][]byte) []kafka.Header {
+	if headers == nil {
+		return nil
+	}
+
+	i := 0
+	result := make([]kafka.Header, len(headers))
+
+	for key, value := range headers {
+		result[i] = kafka.Header{
+			Key: key,
+		}
+		copy(result[i].Value, value)
+		i++
+	}
+
+	return result
+}
+
+func (p *AsyncProducer) ProduceBinary(topic string, key string, data []byte, headers map[string][]byte) error {
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          data,
+	}
+	if headers != nil {
+		km.Headers = p.makeHeaders(headers)
+	}
+
+	err := p.producer.Produce(&km, nil)
+	if err != nil {
+		// error handle inability to connect to kafka
+		return errors.WithStack(err)
+	}
+
+	return err
+}
+
+// ProduceToChannel sends a list of Kafka Messages to Kafka
+func (p *AsyncProducer) ProduceToChannel(topic string, key string, payload interface{}) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          v,
+	}
+
+	p.producer.ProduceChannel() <- &km
+	return nil
+}
+
+func (p *AsyncProducer) ProduceBinaryToChannel(topic string, key string, payload []byte) error {
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          payload,
+	}
+
+	p.producer.ProduceChannel() <- &km
+	return nil
+}
+
+// ProduceSignalBatch sends a specified batch of CAN signals
+//
+//	NOTE: does NOT produce proper JSON, Value is custom for Clickhouse
+func (p *AsyncProducer) ProduceSignalBatch(topic string, key string, payload interface{}) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          v[1 : len(v)-1],
+	}
+
+	err = p.producer.Produce(&km, nil)
+	if err != nil {
+		// error handle inability to connect to kafka
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+// ReadEvents listens to producer's events which are emitted as responses to producing events
+func (p *AsyncProducer) ReadEvents() {
+	for e := range p.producer.Events() {
+		switch m := e.(type) {
+		case *kafka.Message:
+			if m.TopicPartition.Error != nil {
+				logger.Error().
+					Err(m.TopicPartition.Error).
+					Send()
+			}
+		case kafka.Error:
+			logger.Error().
+				Str("err", m.String()).
+				Send()
+		}
+	}
+}
+
+// Close the Kafka Producer
+func (p *AsyncProducer) Close() {
+	p.producer.Flush(0)
+	p.producer.Close()
+	p.producer = nil
+}

pkg/kafka/base_consumer.go

@@ -0,0 +1,211 @@
+package kafka
+
+import (
+	"context"
+	"sync"
+
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/utils/envtool"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+	"github.com/pkg/errors"
+)
+
+const KafkaTimeout = 30000 //ms
+// BaseConsumerInterface interface for Consumer utility
+//
+//	NOTE: DOES NOT AUTO COMMIT OFFSETS
+type BaseConsumerInterface interface {
+	Check(ctx context.Context) error
+	Commit(message *kafka.Message) ([]kafka.TopicPartition, error)
+	CommitOffsets(offsets []kafka.TopicPartition) ([]kafka.TopicPartition, error)
+	ConsumeToChannel(topics []string, events chan *kafka.Message) error
+	ConsumeOrRebalancedCatch(topics []string, events chan *kafka.Message, reb chan struct{}) error
+	LastOffsetConsumed(topic string, partition int32) (kafka.Offset, error)
+	Seek(topic string, offset kafka.Offset) error
+	Stop()
+	Subscribe(topics []string) error
+}
+
+// NoncommitalConsumer utility to produce messages
+type BaseConsumer struct {
+	consumer      *kafka.Consumer
+	running       bool
+	timeout       int
+	connected     bool
+	connectedLock sync.RWMutex
+	pollingError  error
+}
+
+// NewConsumer serves as factory method for consumer to kafka
+func NewBaseConsumer(serviceName string, minBufSize int, fetchMaxWaitTimems int, auto_commit bool) (BaseConsumerInterface, error) {
+	var consumer *kafka.Consumer
+	kafkaConfigMap := loadKafkaConsumerConfig(serviceName)
+	kafkaConfigMap.SetKey("enable.auto.commit", auto_commit)
+	if minBufSize != -1 {
+		kafkaConfigMap.SetKey("fetch.min.bytes", minBufSize)
+	}
+
+	consumer, err := kafka.NewConsumer(&kafkaConfigMap)
+	kafkaHosts := envtool.GetEnv("KAFKA_HOSTS", "localhost:9093")
+	logger.Info().Msgf("NewNoncommitalConsumer  hosts: %s", kafkaHosts)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return &BaseConsumer{
+		consumer: consumer,
+		timeout:  envtool.GetEnvInt("KAFKA_TIMEOUT", 10000),
+	}, nil
+}
+
+func (c *BaseConsumer) Subscribe(topics []string) error {
+	err := c.consumer.SubscribeTopics(topics, nil)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+// ConsumeToChannel runs a poll loop which waits for messages to consume
+func (c *BaseConsumer) ConsumeToChannel(topics []string, events chan *kafka.Message) error {
+	c.running = true
+
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	for c.running {
+		e := c.consumer.Poll(c.timeout)
+
+		switch msg := e.(type) {
+		case *kafka.Message:
+			events <- msg
+			msg = nil
+			c.setConnected(true)
+		case kafka.Error:
+			c.setConnected(false)
+			logger.Error().Msg(e.String())
+			c.pollingError = errors.WithStack(errors.New(e.String()))
+			return c.pollingError
+		}
+
+	}
+
+	return nil
+}
+
+// ConsumeOrRebalancedCatch runs a poll loop which waits for messages to consume
+func (c *BaseConsumer) ConsumeOrRebalancedCatch(topics []string, events chan *kafka.Message, reb chan struct{}) error {
+	c.running = true
+
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	c.pollingError = nil
+	for c.running {
+		e := c.consumer.Poll(c.timeout)
+		switch msg := e.(type) {
+		case *kafka.Message:
+			events <- msg
+			msg = nil
+			c.setConnected(true)
+
+		case kafka.AssignedPartitions:
+			reb <- struct{}{}
+			c.setConnected(true)
+
+		case kafka.Error:
+			c.setConnected(false)
+			c.pollingError = errors.WithStack(errors.New(e.String()))
+			logger.Error().Err(c.pollingError).Send()
+			return c.pollingError
+		}
+	}
+
+	return nil
+}
+
+// Seek provides a wrapper for private consumer method "Seek"
+func (c *BaseConsumer) Seek(topic string, offset kafka.Offset) error {
+	err := c.consumer.Seek(kafka.TopicPartition{
+		Topic:  &topic,
+		Offset: offset,
+	}, c.timeout)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+func (c *BaseConsumer) commitWithoutMessage() ([]kafka.TopicPartition, error) {
+	partitions, err := c.consumer.Commit()
+	if err != nil {
+		return partitions, errors.WithStack(err)
+	}
+	return partitions, nil
+}
+
+func (c *BaseConsumer) Commit(message *kafka.Message) ([]kafka.TopicPartition, error) {
+	if message == nil {
+		return c.commitWithoutMessage()
+	}
+	partitions, err := c.consumer.CommitMessage(message)
+	if err != nil {
+		return partitions, errors.WithStack(err)
+	}
+	return partitions, nil
+}
+
+func (c *BaseConsumer) LastOffsetConsumed(topic string, partition int32) (kafka.Offset, error) {
+	partitions, err := c.consumer.Position([]kafka.TopicPartition{{
+		Topic:     &topic,
+		Partition: partition,
+	}})
+	if err != nil {
+		return -1, errors.WithStack(err)
+	} else if len(partitions) != 1 {
+		return -1, errors.Errorf("no subscription for topic %s", topic)
+	}
+	return partitions[0].Offset, nil
+}
+
+// Stop stops the poll loop running
+func (c *BaseConsumer) Stop() {
+	if c.consumer != nil {
+		c.running = false
+		if err := c.consumer.Close(); err != nil {
+			logger.Warn().Err(err).Send()
+		}
+	}
+}
+
+// very basic implementation for health_check
+func (c *BaseConsumer) Check(ctx context.Context) error {
+	if c.pollingError != nil {
+		return c.pollingError
+	}
+
+	if !c.isConnected() {
+		return errors.New("consumer isn't connected to Kafka")
+	}
+
+	return nil
+}
+
+func (c *BaseConsumer) setConnected(cntd bool) {
+	c.connectedLock.Lock()
+	defer c.connectedLock.Unlock()
+
+	c.connected = cntd
+}
+
+func (c *BaseConsumer) isConnected() bool {
+	c.connectedLock.RLock()
+	defer c.connectedLock.RUnlock()
+
+	return c.connected
+}
+
+// passthrough function for CommitOffsets()
+func (c *BaseConsumer) CommitOffsets(offsets []kafka.TopicPartition) ([]kafka.TopicPartition, error) {
+	return c.consumer.CommitOffsets(offsets)
+}

pkg/kafka/base_consumer_test.go

@@ -0,0 +1,14 @@
+package kafka
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestNonCommitalConsumer(t *testing.T) {
+	_, err := NewBaseConsumer("test", -1, -1, true)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestConsumer", nil, err)
+	}
+}

pkg/kafka/compression_test.go

@@ -0,0 +1,175 @@
+package kafka_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"testing"
+
+	"github.com/golang/snappy"
+)
+
+const totalPayloadSize = 10
+
+func tryClose(w io.Writer) error {
+	if w, ok := w.(io.Closer); ok {
+		return w.Close()
+	}
+	return nil
+}
+
+func BenchmarkNoCompressionWrite(b *testing.B) {
+	w := io.Writer(io.Discard)
+	//payload := []byte("yes this is 25 bytes long")
+	payload := make([]byte, 10000)
+
+	b.ResetTimer()
+	for i := 0; i < 1024*1024*totalPayloadSize/4; i++ {
+		_, err := w.Write(payload)
+		if err != nil {
+			b.Error("error writing file")
+		}
+	}
+
+	err := tryClose(w)
+	if err != nil {
+		b.Error("error closing file")
+	}
+}
+
+func BenchmarkGZipCompressionWrite(b *testing.B) {
+	w := gzip.NewWriter(io.Discard)
+	//payload := []byte("yes this is 25 bytes long")
+	payload := make([]byte, 10000)
+
+	b.ResetTimer()
+	for i := 0; i < 1024*1024*totalPayloadSize/4; i++ {
+		_, err := w.Write(payload)
+		if err != nil {
+			b.Error("error writing file")
+		}
+	}
+
+	err := w.Close()
+	if err != nil {
+		b.Error("error closing file")
+	}
+}
+
+func BenchmarkSnappyCompressionWrite(b *testing.B) {
+	w := snappy.NewWriter(io.Discard)
+	//payload := []byte("yes this is 25 bytes long")
+	payload := make([]byte, 10000)
+
+	b.ResetTimer()
+	for i := 0; i < 1024*1024*totalPayloadSize/4; i++ {
+		_, err := w.Write(payload)
+		if err != nil {
+			b.Error("error writing file")
+		}
+	}
+
+	err := w.Close()
+	if err != nil {
+		b.Error("error closing file")
+	}
+}
+
+func BenchmarkNoCompressionRead(b *testing.B) {
+	var buf bytes.Buffer
+
+	w := io.Writer(&buf)
+	//payload := []byte("yes this is 25 bytes long")
+	payload := make([]byte, 10000)
+
+	for i := 0; i < 1024*1024*totalPayloadSize/4; i++ {
+		_, err := w.Write(payload)
+		if err != nil {
+			b.Error("error writing file")
+		}
+	}
+
+	err := tryClose(w)
+	if err != nil {
+		b.Error("error closing file")
+	}
+
+	//r, err := io.NewReader(&buf)
+	r := bytes.NewReader(payload)
+	if err != nil {
+		b.Error("error opening file")
+	}
+
+	b.ResetTimer()
+	if _, err := io.Copy(io.Discard, r); err != nil {
+		b.Error("error reading file")
+	}
+
+	// if err := r.Close(); err != nil {
+	// 	b.Error("error closing file")
+	// }
+}
+
+func BenchmarkGZipCompressionRead(b *testing.B) {
+	var buf bytes.Buffer
+
+	w := gzip.NewWriter(&buf)
+	//payload := []byte("yes this is 25 bytes long")
+	payload := make([]byte, 10000)
+
+	for i := 0; i < 1024*1024*totalPayloadSize/4; i++ {
+		_, err := w.Write(payload)
+		if err != nil {
+			b.Error("error writing file")
+		}
+	}
+
+	err := w.Close()
+	if err != nil {
+		b.Error("error closing file")
+	}
+
+	r, err := gzip.NewReader(&buf)
+	if err != nil {
+		b.Error("error opening file")
+	}
+
+	b.ResetTimer()
+	if _, err := io.Copy(io.Discard, r); err != nil {
+		b.Error("error reading file")
+	}
+
+	if err := r.Close(); err != nil {
+		b.Error("error closing file")
+	}
+}
+
+func BenchmarkSnappyCompressionRead(b *testing.B) {
+	var buf bytes.Buffer
+
+	w := snappy.NewWriter(&buf)
+	//payload := []byte("yes this is 25 bytes long")
+	payload := make([]byte, 10000)
+
+	for i := 0; i < 1024*1024*totalPayloadSize/4; i++ {
+		_, err := w.Write(payload)
+		if err != nil {
+			b.Error("error writing file")
+		}
+	}
+
+	err := w.Close()
+	if err != nil {
+		b.Error("error closing file")
+	}
+
+	r := snappy.NewReader(&buf)
+	if err != nil {
+		b.Error("error opening file")
+	}
+
+	b.ResetTimer()
+	if _, err := io.Copy(io.Discard, r); err != nil {
+		b.Error("error reading file")
+	}
+}

pkg/kafka/consumer.go

@@ -0,0 +1,242 @@
+package kafka
+
+import (
+	"context"
+	"sync"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/utils/envtool"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+	"github.com/pkg/errors"
+)
+
+// ConsumerInterface interface for Consumer utility
+type ConsumerInterface interface {
+	Consume(topics []string, handler func([]byte, []byte) error) error
+	ConsumeToChannel(topics []string, events chan *kafka.Message) error
+	ConsumeToChannelJson(topics []string, events chan common.EventRawJSON) error
+	ConsumePartitionsToChannel(partitions []kafka.TopicPartition, events chan *kafka.Message) error
+	GetMetadata(topic string) (*kafka.Metadata, error)
+	Check(ctx context.Context) error
+	Stop()
+}
+
+// Consumer utility to produce messages
+type Consumer struct {
+	consumer      *kafka.Consumer
+	running       bool
+	timeout       int
+	connected     bool
+	connectedLock sync.RWMutex
+}
+
+// NewConsumer serves as factory method for consumer to kafka
+func NewConsumer(serviceName string) (ConsumerInterface, error) {
+	var consumer *kafka.Consumer
+	config := loadKafkaConsumerConfig(serviceName)
+
+	consumer, err := kafka.NewConsumer(&config)
+	logger.Info().Msgf("NewConsumer  hosts: %s", envtool.GetEnv("KAFKA_HOSTS", "localhost:9093"))
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return &Consumer{
+		consumer: consumer,
+		timeout:  envtool.GetEnvInt("KAFKA_TIMEOUT", -1),
+	}, nil
+}
+
+// SetConsumer sets the consumer
+func (c *Consumer) SetConsumer(consumer *kafka.Consumer) {
+	c.consumer = consumer
+}
+
+// Consume runs a poll loop which waits for messages to consume
+func (c *Consumer) Consume(topics []string, handler func([]byte, []byte) error) error {
+	if err := c.consumer.SubscribeTopics(topics, nil); err != nil {
+		return errors.WithStack(err)
+	}
+
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	c.running = true
+	connected := true
+	for c.running {
+		e := c.consumer.Poll(c.timeout)
+		switch msg := e.(type) {
+		case *kafka.Message:
+			if !connected {
+				connected = true
+				c.setConnected(true)
+			}
+
+			if err := handler(msg.Key, msg.Value); err != nil {
+				logger.Warn().Err(err).Send()
+			}
+		case kafka.Error:
+			if connected {
+				connected = false
+				c.setConnected(false)
+			}
+
+			logger.Error().Msg(e.String())
+		}
+	}
+
+	return nil
+}
+
+// ConsumeToChannel runs a poll loop which waits for messages to consume
+func (c *Consumer) ConsumeToChannel(topics []string, events chan *kafka.Message) error {
+	err := c.consumer.SubscribeTopics(topics, nil)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	c.running = true
+	connected := true
+	for c.running {
+		e := c.consumer.Poll(c.timeout)
+		switch msg := e.(type) {
+		case *kafka.Message:
+			if !connected {
+				connected = true
+				c.setConnected(true)
+			}
+
+			events <- msg
+		case kafka.Error:
+			if connected {
+				connected = false
+				c.setConnected(false)
+			}
+
+			logger.Error().Msg(e.String())
+		}
+	}
+	return nil
+}
+
+func (c *Consumer) ConsumeToChannelJson(topics []string, events chan common.EventRawJSON) error {
+	err := c.consumer.SubscribeTopics(topics, nil)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	connected := true
+	c.running = true
+	for c.running {
+		e := c.consumer.Poll(c.timeout)
+		switch msg := e.(type) {
+		case *kafka.Message:
+			if !connected {
+				connected = true
+				c.setConnected(true)
+			}
+
+			events <- common.EventRawJSON{
+				Topic:   *msg.TopicPartition.Topic,
+				Key:     string(msg.Key),
+				Payload: msg.Value,
+			}
+		case kafka.Error:
+			if connected {
+				connected = false
+				c.setConnected(false)
+			}
+
+			logger.Error().Err(errors.New(e.String())).Send()
+		}
+	}
+
+	return nil
+}
+
+// ConsumePartitionsToChannel runs a poll loop which waits for messages to consume
+func (c *Consumer) ConsumePartitionsToChannel(partitions []kafka.TopicPartition, events chan *kafka.Message) error {
+	if len(partitions) == 0 {
+		return errors.WithStack(errors.New("no partitions provided"))
+	}
+	topic := "attendant_service"
+	m, err := c.consumer.GetMetadata(&topic, false, 200)
+	_ = m
+	err = c.consumer.Assign(partitions)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	c.setConnected(true)
+	defer c.setConnected(false)
+
+	c.running = true
+	connected := true
+	for c.running {
+		e := <-c.consumer.Events()
+		if e != nil {
+			logger.Error().Msg(e.String())
+		}
+		switch msg := e.(type) {
+		case *kafka.Message:
+			if !connected {
+				connected = true
+				c.setConnected(true)
+			}
+
+			events <- msg
+		case kafka.Error:
+			if connected {
+				connected = false
+				c.setConnected(false)
+			}
+
+			logger.Error().Msg(e.String())
+		}
+	}
+	return nil
+}
+
+func (c *Consumer) GetMetadata(topic string) (*kafka.Metadata, error) {
+	return c.consumer.GetMetadata(&topic, false, 200)
+}
+
+// Stop stops the poll loop running
+func (c *Consumer) Stop() {
+	if c.consumer != nil {
+		c.running = false
+		if err := c.consumer.Close(); err != nil {
+			logger.Warn().Err(err).Send()
+		}
+	}
+}
+
+func (c *Consumer) Check(ctx context.Context) error {
+	if !c.isConnected() {
+		return errors.New("consumer isn't connected to Kafka")
+	}
+
+	return nil
+}
+
+func (c *Consumer) setConnected(cntd bool) {
+	c.connectedLock.Lock()
+	defer c.connectedLock.Unlock()
+
+	c.connected = cntd
+}
+
+func (c *Consumer) isConnected() bool {
+	c.connectedLock.RLock()
+	defer c.connectedLock.RUnlock()
+
+	return c.connected
+}

pkg/kafka/consumer_test.go

@@ -0,0 +1,45 @@
+package kafka
+
+import (
+	"errors"
+	"testing"
+
+	"fiskerinc.com/modules/testhelper"
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+)
+
+func TestConsumer(t *testing.T) {
+	t.Skip()
+
+	c, err := kafka.NewConsumer(&kafka.ConfigMap{
+		"group.id":                 "gotest",
+		"socket.timeout.ms":        1000,
+		"session.timeout.ms":       1000,
+		"enable.auto.offset.store": false,
+	})
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestConsumer", nil, err)
+	}
+
+	consumer := &Consumer{consumer: c}
+	channel := make(chan *kafka.Message)
+
+	go consumer.ConsumeToChannel([]string{"gotest"}, channel)
+	consumer.Stop()
+}
+
+func TestConsumerError(t *testing.T) {
+	t.Skip()
+
+	c, err := kafka.NewConsumer(&kafka.ConfigMap{
+		"socket.timeout.ms":        1000,
+		"session.timeout.ms":       1000,
+		"enable.auto.offset.store": false,
+	})
+	if err == nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestConsumerError", errors.New("Required property group.id not set"), err)
+	}
+	if c != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestConsumerError", nil, c)
+	}
+}

pkg/kafka/kafka_config.go

@@ -0,0 +1,261 @@
+package kafka
+
+import (
+	"fiskerinc.com/modules/utils/envtool"
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+)
+
+// https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+// Load all the different settings we want for Kafka configuration into a kafka config object
+func loadKafkaConsumerConfig(serviceName string) (configuration kafka.ConfigMap) {
+	configuration = kafka.ConfigMap{
+		"group.id":          serviceName,
+		"bootstrap.servers": envtool.GetEnv("KAFKA_HOSTS", "localhost:9093"),
+		// "builtin.features":                        envtool.GetEnv("KAFKA_BUILTIN_FEATURES", kafka_builtin_features_default),  //Desc: Indicates the builtin features for this build of librdkafka. An application can either query this value or attempt to set it with its list of required features to check for library support. <br>*Type: CSV flags*
+		// "client.id":                               envtool.GetEnv("KAFKA_CLIENT_ID", "rdkafka"),                              //Desc: Client identifier. <br>*Type: string*
+		"metadata.broker.list": envtool.GetEnv("KAFKA_METADATA_BROKER_LIST", "localhost:9093"), //Desc: Initial list of brokers as a CSV list of broker host or host:port. The application may also use 'rd_kafka_brokers_add()' to add brokers during runtime. <br>*Type: string*
+		// "message.max.bytes":                       envtool.GetEnvInt("KAFKA_MESSAGE_MAX_BYTES", 1000000),                     //Desc: Maximum Kafka protocol request message size. Due to differing framing overhead between protocol versions the producer is unable to reliably enforce a strict max message limit at produce time and may exceed the maximum size by one message in protocol ProduceRequests, the broker will enforce the the topic's 'max.message.bytes' limit (see Apache Kafka documentation). <br>*Type: integer*   Range: 1000 .. 1000000000
+		// "message.copy.max.bytes":                  envtool.GetEnvInt("KAFKA_MESSAGE_COPY_MAX_BYTES", 65535),                  //Desc: Maximum size for message to be copied to buffer. Messages larger than this will be passed by reference (zero-copy) at the expense of larger iovecs. <br>*Type: integer*   Range: 0 .. 1000000000
+		// "receive.message.max.bytes":               envtool.GetEnvInt("KAFKA_RECEIVE_MESSAGE_MAX_BYTES", 100000000),           //Desc: Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hickups. This value must be at least 'fetch.max.bytes'  + 512 to allow for protocol overhead; the value is adjusted automatically unless the configuration property is explicitly set. <br>*Type: integer*   Range: 1000 .. 2147483647
+		// "max.in.flight.requests.per.connection":   envtool.GetEnvInt("KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION", 1000000), //Desc: Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. <br>*Type: integer*   Range: 1 .. 1000000
+		// "topic.metadata.refresh.interval.ms":      envtool.GetEnvInt("KAFKA_TOPIC_METADATA_REFRESH_INTERVAL_MS", 300000),     //Desc: Period of time in milliseconds at which topic and broker metadata is refreshed in order to proactively discover any new brokers, topics, partitions or partition leader changes. Use -1 to disable the intervalled refresh (not recommended). If there are no locally referenced topics (no topic objects created, no messages produced, no subscription or no assignment) then only the broker list will be refreshed every interval but no more often than every 10s. <br>*Type: integer*   Range: -1 .. 3600000
+		"metadata.max.age.ms": envtool.GetEnvInt("KAFKA_METADATA_MAX_AGE_MS", 900000), //Desc: Metadata cache max age. Defaults to topic.metadata.refresh.interval.ms * 3 <br>*Type: integer*   Range: 1 .. 86400000
+		// "topic.metadata.refresh.fast.interval.ms": envtool.GetEnvInt("KAFKA_TOPIC_METADATA_REFRESH_FAST_INTERVAL_MS", 100),   //Desc: When a topic loses its leader a new metadata request will be enqueued immediately and then with this initial interval, exponentially increasing upto 'retry.backoff.max.ms', until the topic metadata has been refreshed. If not set explicitly, it will be defaulted to 'retry.backoff.ms'. This is used to recover quickly from transitioning leader brokers. <br>*Type: integer*   Range: 1 .. 60000
+		// "topic.metadata.refresh.sparse":           envtool.GetEnvBool("KAFKA_TOPIC_METADATA_REFRESH_SPARSE", true),           //Desc: Sparse metadata requests (consumes less network bandwidth) <br>*Type: boolean*   Range: true, false
+		// "topic.metadata.propagation.max.ms":       envtool.GetEnvInt("KAFKA_TOPIC_METADATA_PROPAGATION_MAX_MS", 30000),       //Desc: Apache Kafka topic creation is asynchronous and it takes some time for a new topic to propagate throughout the cluster to all brokers. If a client requests topic metadata after manual topic creation but before the topic has been fully propagated to the broker the client is requesting metadata from, the topic will seem to be non-existent and the client will mark the topic as such, failing queued produced messages with 'ERR__UNKNOWN_TOPIC'. This setting delays marking a topic as non-existent until the configured propagation max time has passed. The maximum propagation time is calculated from the time the topic is first referenced in the client, e.g., on produce(). <br>*Type: integer*   Range: 0 .. 3600000
+		// "topic.blacklist":                         envtool.GetEnv("KAFKA_TOPIC_BLACKLIST", ""),                               //Desc: Topic blacklist, a comma-separated list of regular expressions for matching topic names that should be ignored in broker metadata information as if the topics did not exist. <br>*Type: pattern list*
+		// "debug":                                   envtool.GetEnv("KAFKA_DEBUG", ""),                                         //Desc: A comma-separated list of debug contexts to enable. Detailed Producer debugging: broker,topic,msg. Consumer: consumer,cgrp,topic,fetch <br>*Type: CSV flags*   Range: generic, broker, topic, metadata, feature, queue, msg, protocol, cgrp, security, fetch, interceptor, plugin, consumer, admin, eos, mock, assignor, conf, all
+		"socket.timeout.ms": envtool.GetEnvInt("KAFKA_SOCKET_TIMEOUT_MS", 60000), //Desc: Default timeout for network requests. Producer: ProduceRequests will use the lesser value of 'socket.timeout.ms' and remaining 'message.timeout.ms' for the first message in the batch. Consumer: FetchRequests will use 'fetch.wait.max.ms' + 'socket.timeout.ms'. Admin: Admin requests will use 'socket.timeout.ms' or explicitly set 'rd_kafka_AdminOptions_set_operation_timeout()' value. <br>*Type: integer*   Range: 10 .. 300000
+		// "socket.send.buffer.bytes":           envtool.GetEnvInt("KAFKA_SOCKET_SEND_BUFFER_BYTES", 0),               //Desc: Broker socket send buffer size. System default is used if 0. <br>*Type: integer*   Range: 0 .. 100000000
+		// "socket.receive.buffer.bytes":        envtool.GetEnvInt("KAFKA_SOCKET_RECEIVE_BUFFER_BYTES", 0),            //Desc: Broker socket receive buffer size. System default is used if 0. <br>*Type: integer*   Range: 0 .. 100000000
+		"socket.keepalive.enable": envtool.GetEnvBool("KAFKA_SOCKET_KEEPALIVE_ENABLE", true), //Desc: Enable TCP keep-alives (SO_KEEPALIVE) on broker sockets <br>*Type: boolean*   Range: true, false
+		// "socket.nagle.disable":               envtool.GetEnvBool("KAFKA_SOCKET_NAGLE_DISABLE", false),              //Desc: Disable the Nagle algorithm (TCP_NODELAY) on broker sockets. <br>*Type: boolean*   Range: true, false
+		// "socket.max.fails":                   envtool.GetEnvInt("KAFKA_SOCKET_MAX_FAILS", 1),                       //Desc: Disconnect from broker when this number of send failures (e.g., timed out requests) is reached. Disable with 0. WARNING: It is highly recommended to leave this setting at its default value of 1 to avoid the client and broker to become desynchronized in case of request timeouts. NOTE: The connection is automatically re-established. <br>*Type: integer*   Range: 0 .. 1000000
+		// "broker.address.ttl":                 envtool.GetEnvInt("KAFKA_BROKER_ADDRESS_TTL", 1000),                  //Desc: How long to cache the broker address resolving results (milliseconds). <br>*Type: integer*   Range: 0 .. 86400000
+		// "broker.address.family":              envtool.GetEnv("KAFKA_BROKER_ADDRESS_FAMILY", "any"),                 //Desc: Allowed broker IP address families: any, v4, v6 <br>*Type: enum value*   Range: any, v4, v6
+		// "socket.connection.setup.timeout.ms": envtool.GetEnvInt("KAFKA_SOCKET_CONNECTION_SETUP_TIMEOUT_MS", 30000), //Desc: Maximum time allowed for broker connection setup (TCP connection setup as well SSL and SASL handshake). If the connection to the broker is not fully functional after this the connection will be closed and retried. <br>*Type: integer*   Range: 1000 .. 2147483647
+		"connections.max.idle.ms": envtool.GetEnvInt("KAFKA_CONNECTIONS_MAX_IDLE_MS", 0), //Desc: Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info). <br>*Type: integer*   Range: 0 .. 2147483647
+		// "reconnect.backoff.ms":               envtool.GetEnvInt("KAFKA_RECONNECT_BACKOFF_MS", 100),                 //Desc: The initial time to wait before reconnecting to a broker after the connection has been closed. The time is increased exponentially until 'reconnect.backoff.max.ms' is reached. -25% to +50% jitter is applied to each reconnect backoff. A value of 0 disables the backoff and reconnects immediately. <br>*Type: integer*   Range: 0 .. 3600000
+		// "reconnect.backoff.max.ms":           envtool.GetEnvInt("KAFKA_RECONNECT_BACKOFF_MAX_MS", 10000),           //Desc: The maximum time to wait before reconnecting to a broker after the connection has been closed. <br>*Type: integer*   Range: 0 .. 3600000
+		"statistics.interval.ms": envtool.GetEnvInt("KAFKA_STATISTICS_INTERVAL_MS", 0), //Desc: librdkafka statistics emit interval. The application also needs to register a stats callback using 'rd_kafka_conf_set_stats_cb()'. The granularity is 1000ms. A value of 0 disables statistics. <br>*Type: integer*   Range: 0 .. 86400000
+		// "enabled_events":                     envtool.GetEnvInt("KAFKA_ENABLED_EVENTS", 0),                         //Desc: See 'rd_kafka_conf_set_events()' <br>*Type: integer*   Range: 0 .. 2147483647
+		// "error_cb":                                envtool.GetEnv("KAFKA_ERROR_CB", ""),                                                   //Desc: Error callback (set with rd_kafka_conf_set_error_cb()) <br>*Type: see dedicated API*
+		// "throttle_cb":                             envtool.GetEnv("KAFKA_THROTTLE_CB", ""),                                                //Desc: Throttle callback (set with rd_kafka_conf_set_throttle_cb()) <br>*Type: see dedicated API*
+		// "stats_cb":                                envtool.GetEnv("KAFKA_STATS_CB", ""),                                                   //Desc: Statistics callback (set with rd_kafka_conf_set_stats_cb()) <br>*Type: see dedicated API*
+		// "log_cb":                                  envtool.GetEnv("KAFKA_LOG_CB", ""),                                                     //Desc: Log callback (set with rd_kafka_conf_set_log_cb()) <br>*Type: see dedicated API*
+		"log_level": envtool.GetEnvInt("KAFKA_LOG_LEVEL", 6), //Desc: Logging level (syslog(3) levels) <br>*Type: integer*   Range: 0 .. 7
+		// "log.queue":            envtool.GetEnvBool("KAFKA_LOG_QUEUE", false),           //Desc: Disable spontaneous log_cb from internal librdkafka threads, instead enqueue log messages on queue set with 'rd_kafka_set_log_queue()' and serve log callbacks or events through the standard poll APIs. **NOTE**: Log messages will linger in a temporary queue until the log queue has been set. <br>*Type: boolean*   Range: true, false
+		// "log.thread.name":      envtool.GetEnvBool("KAFKA_LOG_THREAD_NAME", true),      //Desc: Print internal thread name in log messages (useful for debugging librdkafka internals) <br>*Type: boolean*   Range: true, false
+		// "enable.random.seed":   envtool.GetEnvBool("KAFKA_ENABLE_RANDOM_SEED", true),   //Desc: If enabled librdkafka will initialize the PRNG with srand(current_time.milliseconds) on the first invocation of rd_kafka_new() (required only if rand_r() is not available on your platform). If disabled the application must call srand() prior to calling rd_kafka_new(). <br>*Type: boolean*   Range: true, false
+		"log.connection.close": envtool.GetEnvBool("KAFKA_LOG_CONNECTION_CLOSE", true), //Desc: Log broker disconnects. It might be useful to turn this off when interacting with 0.9 brokers with an aggressive 'connections.max.idle.ms' value. <br>*Type: boolean*   Range: true, false
+		// "background_event_cb":                     envtool.GetEnv("KAFKA_BACKGROUND_EVENT_CB", ""),                                        //Desc: Background queue event callback (set with rd_kafka_conf_set_background_event_cb()) <br>*Type: see dedicated API*
+		// "socket_cb":                               envtool.GetEnv("KAFKA_SOCKET_CB", ""),                                                  //Desc: Socket creation callback to provide race-free CLOEXEC <br>*Type: see dedicated API*
+		// "connect_cb":                              envtool.GetEnv("KAFKA_CONNECT_CB", ""),                                                 //Desc: Socket connect callback <br>*Type: see dedicated API*
+		// "closesocket_cb":                          envtool.GetEnv("KAFKA_CLOSESOCKET_CB", ""),                                             //Desc: Socket close callback <br>*Type: see dedicated API*
+		// "open_cb":                                 envtool.GetEnv("KAFKA_OPEN_CB", ""),                                                    //Desc: File open callback to provide race-free CLOEXEC <br>*Type: see dedicated API*
+		// "resolve_cb":                              envtool.GetEnv("KAFKA_RESOLVE_CB", ""),                                                 //Desc: Address resolution callback (set with rd_kafka_conf_set_resolve_cb()). <br>*Type: see dedicated API*
+		// "opaque":                                  envtool.GetEnv("KAFKA_OPAQUE", ""),                                                     //Desc: Application opaque (set with rd_kafka_conf_set_opaque()) <br>*Type: see dedicated API*
+		// "default_topic_conf":                      envtool.GetEnv("KAFKA_DEFAULT_TOPIC_CONF", ""),                                         //Desc: Default topic configuration for automatically subscribed topics <br>*Type: see dedicated API*
+		// "internal.termination.signal":    envtool.GetEnvInt("KAFKA_INTERNAL_TERMINATION_SIGNAL", 0),        //Desc: Signal that librdkafka will use to quickly terminate on rd_kafka_destroy(). If this signal is not set then there will be a delay before rd_kafka_wait_destroyed() returns true as internal threads are timing out their system calls. If this signal is set however the delay will be minimal. The application should mask this signal as an internal signal handler is installed. <br>*Type: integer*   Range: 0 .. 128
+		"api.version.request": envtool.GetEnvBool("KAFKA_API_VERSION_REQUEST", true), //Desc: Request broker's supported API versions to adjust functionality to available protocol features. If set to false, or the ApiVersionRequest fails, the fallback version 'broker.version.fallback' will be used. **NOTE**: Depends on broker version >=0.10.0. If the request is not supported by (an older) broker the 'broker.version.fallback' fallback is used. <br>*Type: boolean*   Range: true, false
+		// "api.version.request.timeout.ms": envtool.GetEnvInt("KAFKA_API_VERSION_REQUEST_TIMEOUT_MS", 10000), //Desc: Timeout for broker API version requests. <br>*Type: integer*   Range: 1 .. 300000
+		"api.version.fallback.ms":  envtool.GetEnvInt("KAFKA_API_VERSION_FALLBACK_MS", 0),      //Desc: Dictates how long the 'broker.version.fallback' fallback is used in the case the ApiVersionRequest fails. **NOTE**: The ApiVersionRequest is only issued when a new connection to the broker is made (such as after an upgrade). <br>*Type: integer*   Range: 0 .. 604800000
+		"broker.version.fallback":  envtool.GetEnv("KAFKA_BROKER_VERSION_FALLBACK", "0.10.0"),  //Desc: Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see 'api.version.request') making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for 'api.version.fallback.ms'. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value >= 0.10, such as 0.10.2.1, enables ApiVersionRequests. <br>*Type: string*
+		"allow.auto.create.topics": envtool.GetEnvBool("KAFKA_ALLOW_AUTO_CREATE_TOPICS", true), //Desc: Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. The broker must also be configured with 'auto.create.topics.enable=true' for this configuration to take effect. Note: the default value (true) for the producer is different from the default value (false) for the consumer. Further, the consumer default value is different from the Java consumer (true), and this property is not supported by the Java producer. Requires broker version >= 0.11.0.0, for older broker versions only the broker configuration applies. <br>*Type: boolean*   Range: true, false
+		"security.protocol":        envtool.GetEnv("KAFKA_SECURITY_PROTOCOL", "plaintext"),     //Desc: Protocol used to communicate with brokers. <br>*Type: enum value*   Range: plaintext, ssl, sasl_plaintext, sasl_ssl
+		// "ssl.cipher.suites":              envtool.GetEnv("KAFKA_SSL_CIPHER_SUITES", ""),                    //Desc: A cipher suite is a named combination of authentication, encryption, MAC and key exchange algorithm used to negotiate the security settings for a network connection using TLS or SSL network protocol. See manual page for 'ciphers(1)' and 'SSL_CTX_set_cipher_list(3). <br>*Type: string*
+		// "ssl.curves.list":                envtool.GetEnv("KAFKA_SSL_CURVES_LIST", ""),                      //Desc: The supported-curves extension in the TLS ClientHello message specifies the curves (standard/named, or 'explicit' GF(2^k) or GF(p)) the client is willing to have the server use. See manual page for 'SSL_CTX_set1_curves_list(3)'. OpenSSL >= 1.0.2 required. <br>*Type: string*
+		// "ssl.sigalgs.list":               envtool.GetEnv("KAFKA_SSL_SIGALGS_LIST", ""),                     //Desc: The client uses the TLS ClientHello signature_algorithms extension to indicate to the server which signature/hash algorithm pairs may be used in digital signatures. See manual page for 'SSL_CTX_set1_sigalgs_list(3)'. OpenSSL >= 1.0.2 required. <br>*Type: string*
+		// "ssl.key.location":               envtool.GetEnv("KAFKA_SSL_KEY_LOCATION", ""),                     //Desc: Path to client's private key (PEM) used for authentication. <br>*Type: string*
+		// "ssl.key.password":               envtool.GetEnv("KAFKA_SSL_KEY_PASSWORD", ""),                     //Desc: Private key passphrase (for use with 'ssl.key.location' and 'set_ssl_cert()') <br>*Type: string*
+		// "ssl.key.pem":                    envtool.GetEnv("KAFKA_SSL_KEY_PEM", ""),                          //Desc: Client's private key string (PEM format) used for authentication. <br>*Type: string*
+		// "ssl_key":                                 envtool.GetEnv("KAFKA_SSL_KEY", ""),                                                    //Desc: Client's private key as set by rd_kafka_conf_set_ssl_cert() <br>*Type: see dedicated API*
+		// "ssl.certificate.location": envtool.GetEnv("KAFKA_SSL_CERTIFICATE_LOCATION", ""), //Desc: Path to client's public key (PEM) used for authentication. <br>*Type: string*
+		// "ssl.certificate.pem":      envtool.GetEnv("KAFKA_SSL_CERTIFICATE_PEM", ""),      //Desc: Client's public key string (PEM format) used for authentication. <br>*Type: string*
+		// "ssl_certificate":                         envtool.GetEnv("KAFKA_SSL_CERTIFICATE", ""),                                            //Desc: Client's public key as set by rd_kafka_conf_set_ssl_cert() <br>*Type: see dedicated API*
+		// "ssl.ca.location": envtool.GetEnv("KAFKA_SSL_CA_LOCATION", ""), //Desc: File or directory path to CA certificate(s) for verifying the broker's key. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to 'probe'. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or 'ssl.ca.location' is set to 'probe' a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see 'OPENSSLDIR' in 'openssl version -a'). <br>*Type: string*
+		// "ssl.ca.pem":      envtool.GetEnv("KAFKA_SSL_CA_PEM", ""),      //Desc: CA certificate string (PEM format) for verifying the broker's key. <br>*Type: string*
+		// "ssl_ca":                                  envtool.GetEnv("KAFKA_SSL_CA", ""),                                                     //Desc: CA certificate as set by rd_kafka_conf_set_ssl_cert() <br>*Type: see dedicated API*
+		// "ssl.ca.certificate.stores":               envtool.GetEnv("KAFKA_SSL_CA_CERTIFICATE_STORES", "Root"),                              //Desc: Comma-separated list of Windows Certificate stores to load CA certificates from. Certificates will be loaded in the same order as stores are specified. If no certificates can be loaded from any of the specified stores an error is logged and the OpenSSL library's default CA location is used instead. Store names are typically one or more of: MY, Root, Trust, CA. <br>*Type: string*
+		// "ssl.crl.location":      envtool.GetEnv("KAFKA_SSL_CRL_LOCATION", ""),      //Desc: Path to CRL for verifying broker's certificate validity. <br>*Type: string*
+		// "ssl.keystore.location": envtool.GetEnv("KAFKA_SSL_KEYSTORE_LOCATION", ""), //Desc: Path to client's keystore (PKCS#12) used for authentication. <br>*Type: string*
+		// "ssl.keystore.password": envtool.GetEnv("KAFKA_SSL_KEYSTORE_PASSWORD", ""), //Desc: Client's keystore (PKCS#12) password. <br>*Type: string*
+		// "ssl.providers":         envtool.GetEnv("KAFKA_SSL_PROVIDERS", ""),         //Desc: Comma-separated list of OpenSSL 3.0.x implementation providers. E.g., "default,legacy". <br>*Type: string*
+		// "ssl.engine.id":         envtool.GetEnv("KAFKA_SSL_ENGINE_ID", "dynamic"),  //Desc: OpenSSL engine id is the name used for loading engine. <br>*Type: string*
+		// "ssl_engine_callback_data":                envtool.GetEnv("KAFKA_SSL_ENGINE_CALLBACK_DATA", ""),                                   //Desc: OpenSSL engine callback data (set with rd_kafka_conf_set_engine_callback_data()). <br>*Type: see dedicated API*
+		"enable.ssl.certificate.verification": envtool.GetEnvBool("KAFKA_SSL_VERIFY", true), //Desc: Enable OpenSSL's builtin broker (server) certificate verification. This verification can be extended by the application by implementing a certificate_verify_cb. Type: boolean
+		// "ssl.endpoint.identification.algorithm": envtool.GetEnv("KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", "https"), //Desc: Endpoint identification algorithm to validate broker hostname using broker certificate. https - Server (broker) hostname verification as specified in RFC2818. none - No endpoint verification. OpenSSL >= 1.0.2 required. <br>*Type: enum value*   Range: none, https
+		// "ssl.certificate.verify_cb":               envtool.GetEnv("KAFKA_SSL_CERTIFICATE_VERIFY_CB", ""),                                  //Desc: Callback to verify the broker certificate chain. <br>*Type: see dedicated API*
+		"sasl.mechanisms": envtool.GetEnv("KAFKA_SASL_MECHANISMS", "GSSAPI"), //Desc: SASL mechanism to use for authentication. Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER. **NOTE**: Despite the name only one mechanism must be configured. <br>*Type: string*
+		// "sasl.kerberos.service.name":            envtool.GetEnv("KAFKA_SASL_KERBEROS_SERVICE_NAME", "kafka"),                            //Desc: Kerberos principal name that Kafka runs as, not including /hostname@REALM <br>*Type: string*
+		// "sasl.kerberos.principal":               envtool.GetEnv("KAFKA_SASL_KERBEROS_PRINCIPAL", "kafkaclient"),                         //Desc: This client's Kerberos principal name. (Not supported on Windows, will use the logon user's principal). <br>*Type: string*
+		// "sasl.kerberos.kinit.cmd":               envtool.GetEnv("KAFKA_SASL_KERBEROS_KINIT_CMD", kafka_sasl_kerberos_kinit_cmd_default), //Desc: Shell command to refresh or acquire the client's Kerberos ticket. This command is executed on client creation and every sasl.kerberos.min.time.before.relogin (0=disable). %{config.prop.name} is replaced by corresponding config object value. <br>*Type: string*
+		// "sasl.kerberos.keytab":                  envtool.GetEnv("KAFKA_SASL_KERBEROS_KEYTAB", ""),                                       //Desc: Path to Kerberos keytab file. This configuration property is only used as a variable in 'sasl.kerberos.kinit.cmd' as ' ... -t "%{sasl.kerberos.keytab}"'. <br>*Type: string*
+		// "sasl.kerberos.min.time.before.relogin": envtool.GetEnvInt("KAFKA_SASL_KERBEROS_MIN_TIME_BEFORE_RELOGIN", 60000),                //Desc: Minimum time in milliseconds between key refresh attempts. Disable automatic key refresh by setting this property to 0. <br>*Type: integer*   Range: 0 .. 86400000
+		"sasl.username": envtool.GetEnv("KAFKA_SASL_USERNAME", ""), //Desc: SASL username for use with the PLAIN and SASL-SCRAM-.. mechanisms <br>*Type: string*
+		"sasl.password": envtool.GetEnv("KAFKA_SASL_PASSWORD", ""), //Desc: SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism <br>*Type: string*
+		// "sasl.oauthbearer.config":               envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_CONFIG", ""),                                    //Desc: SASL/OAUTHBEARER configuration. The format is implementation-dependent and must be parsed accordingly. The default unsecured token implementation (see https://tools.ietf.org/html/rfc7515#appendix-A.5) recognizes space-separated name=value pairs with valid names including principalClaimName, principal, scopeClaimName, scope, and lifeSeconds. The default value for principalClaimName is "sub", the default value for scopeClaimName is "scope", and the default value for lifeSeconds is 3600. The scope value is CSV format with the default value being no/empty scope. For example: 'principalClaimName=azp principal=admin scopeClaimName=roles scope=role1,role2 lifeSeconds=600'. In addition, SASL extensions can be communicated to the broker via 'extension_NAME=value'. For example: 'principal=admin extension_traceId=123' <br>*Type: string*
+		// "enable.sasl.oauthbearer.unsecure.jwt": envtool.GetEnvBool("KAFKA_ENABLE_SASL_OAUTHBEARER_UNSECURE_JWT", false), //Desc: Enable the builtin unsecure JWT OAUTHBEARER token handler if no oauthbearer_refresh_cb has been set. This builtin handler should only be used for development or testing, and not in production. <br>*Type: boolean*   Range: true, false
+		// "oauthbearer_token_refresh_cb":            envtool.GetEnv("KAFKA_OAUTHBEARER_TOKEN_REFRESH_CB", ""),                               //Desc: SASL/OAUTHBEARER token refresh callback (set with rd_kafka_conf_set_oauthbearer_token_refresh_cb(), triggered by rd_kafka_poll(), et.al. This callback will be triggered when it is time to refresh the client's OAUTHBEARER token. Also see 'rd_kafka_conf_enable_sasl_queue()'. <br>*Type: see dedicated API*
+		// "sasl.oauthbearer.method":             envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_METHOD", "default"),      //Desc: Set to "default" or "oidc" to control which login method to be used. If set to "oidc", the following properties must also be be specified: 'sasl.oauthbearer.client.id', 'sasl.oauthbearer.client.secret', and 'sasl.oauthbearer.token.endpoint.url'. <br>*Type: enum value*   Range: default, oidc
+		// "sasl.oauthbearer.client.id":          envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_CLIENT_ID", ""),          //Desc: Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.client.secret":      envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET", ""),      //Desc: Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.scope":              envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_SCOPE", ""),              //Desc: Client use this to specify the scope of the access request to the broker. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.extensions":         envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_EXTENSIONS", ""),         //Desc: Allow additional information to be provided to the broker. Comma-separated list of key=value pairs. E.g., "supportFeatureX=true,organizationId=sales-emea".Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.token.endpoint.url": envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT_URL", ""), //Desc: OAuth/OIDC issuer token endpoint HTTP(S) URI used to retrieve token. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "plugin.library.paths": envtool.GetEnv("KAFKA_PLUGIN_LIBRARY_PATHS", ""), //Desc: List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically. <br>*Type: string*
+		// "interceptors":                            envtool.GetEnv("KAFKA_INTERCEPTORS", ""),                                               //Desc: Interceptors added through rd_kafka_conf_interceptor_add_..() and any configuration handled by interceptors. <br>*Type: see dedicated API*
+		// "group.instance.id":             envtool.GetEnv("KAFKA_GROUP_INSTANCE_ID", ""),                             //Desc: Enable static group membership. Static group members are able to leave and rejoin a group within the configured 'session.timeout.ms' without prompting a group rebalance. This should be used in combination with a larger 'session.timeout.ms' to avoid group rebalances caused by transient unavailability (e.g. process restarts). Requires broker version >= 2.3.0. <br>*Type: string*
+		"partition.assignment.strategy": envtool.GetEnv("KAFKA_PARTITION_ASSIGNMENT_STRATEGY", "range,roundrobin"), //Desc: The name of one or more partition assignment strategies. The elected group leader will use a strategy supported by all members of the group to assign partitions to group members. If there is more than one eligible strategy, preference is determined by the order of this list (strategies earlier in the list have higher priority). Cooperative and non-cooperative (eager) strategies must not be mixed. Available strategies: range, roundrobin, cooperative-sticky. <br>*Type: string*
+		"session.timeout.ms":            envtool.GetEnvInt("KAFKA_SESSION_TIMEOUT_MS", 45000),                      //Desc: Client group session and failure detection timeout. The consumer sends periodic heartbeats (heartbeat.interval.ms) to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. The allowed range is configured with the **broker** configuration properties 'group.min.session.timeout.ms' and 'group.max.session.timeout.ms'. Also see 'max.poll.interval.ms'. <br>*Type: integer*   Range: 1 .. 3600000
+		"heartbeat.interval.ms":         envtool.GetEnvInt("KAFKA_HEARTBEAT_INTERVAL_MS", 3000),                    //Desc: Group session keepalive heartbeat interval. <br>*Type: integer*   Range: 1 .. 3600000
+		// "group.protocol.type":           envtool.GetEnv("KAFKA_GROUP_PROTOCOL_TYPE", "consumer"),                   //Desc: Group protocol type. NOTE: Currently, the only supported group protocol type is 'consumer'. <br>*Type: string*
+		// "coordinator.query.interval.ms": envtool.GetEnvInt("KAFKA_COORDINATOR_QUERY_INTERVAL_MS", 600000),          //Desc: How often to query for the current client group coordinator. If the currently assigned coordinator is down the configured query interval will be divided by ten to more quickly recover in case of coordinator reassignment. <br>*Type: integer*   Range: 1 .. 3600000
+		"max.poll.interval.ms":     envtool.GetEnvInt("KAFKA_MAX_POLL_INTERVAL_MS", 300000),    //Desc: Maximum allowed time between calls to consume messages (e.g., rd_kafka_consumer_poll()) for high-level consumers. If this interval is exceeded the consumer is considered failed and the group will rebalance in order to reassign the partitions to another consumer group member. Warning: Offset commits may be not possible at this point. Note: It is recommended to set 'enable.auto.offset.store=false' for long-time processing applications and then explicitly store offsets (using offsets_store()) *after* message processing, to make sure offsets are not auto-committed prior to processing has finished. The interval is checked two times per second. See KIP-62 for more information. <br>*Type: integer*   Range: 1 .. 86400000
+		"enable.auto.commit":       envtool.GetEnvBool("KAFKA_ENABLE_AUTO_COMMIT", true),       //Desc: Automatically and periodically commit offsets in the background. Note: setting this to false does not prevent the consumer from fetching previously committed start offsets. To circumvent this behaviour set specific start offsets per partition in the call to assign(). <br>*Type: boolean*   Range: true, false
+		"auto.commit.interval.ms":  envtool.GetEnvInt("KAFKA_AUTO_COMMIT_INTERVAL_MS", 5000),   //Desc: The frequency in milliseconds that the consumer offsets are committed (written) to offset storage. (0 = disable). This setting is used by the high-level consumer. <br>*Type: integer*   Range: 0 .. 86400000
+		"enable.auto.offset.store": envtool.GetEnvBool("KAFKA_ENABLE_AUTO_OFFSET_STORE", true), //Desc: Automatically store offset of last message provided to application. The offset store is an in-memory store of the next offset to (auto-)commit for each partition. <br>*Type: boolean*   Range: true, false
+		// "queued.min.messages":           envtool.GetEnvInt("KAFKA_QUEUED_MIN_MESSAGES", 100000),                    //Desc: Minimum number of messages per topic+partition librdkafka tries to maintain in the local consumer queue. <br>*Type: integer*   Range: 1 .. 10000000
+		// "queued.max.messages.kbytes":    envtool.GetEnvInt("KAFKA_QUEUED_MAX_MESSAGES_KBYTES", 65536),              //Desc: Maximum number of kilobytes of queued pre-fetched messages in the local consumer queue. If using the high-level consumer this setting applies to the single consumer queue, regardless of the number of partitions. When using the legacy simple consumer or when separate partition queues are used this setting applies per partition. This value may be overshot by fetch.message.max.bytes. This property has higher priority than queued.min.messages. <br>*Type: integer*   Range: 1 .. 2097151
+		// "fetch.wait.max.ms":             envtool.GetEnvInt("KAFKA_FETCH_WAIT_MAX_MS", 500),                         //Desc: Maximum time the broker may wait to fill the Fetch response with fetch.min.bytes of messages. <br>*Type: integer*   Range: 0 .. 300000
+		// "fetch.queue.backoff.ms":        envtool.GetEnvInt("KAFKA_FETCH_QUEUE_BACKOFF_MS", 1000),                   //Desc: How long to postpone the next fetch request for a topic+partition in case the current fetch queue thresholds (queued.min.messages or queued.max.messages.kbytes) have been exceded. This property may need to be decreased if the queue thresholds are set low and the application is experiencing long (~1s) delays between messages. Low values may increase CPU utilization. <br>*Type: integer*   Range: 0 .. 300000
+		// "fetch.message.max.bytes":       envtool.GetEnvInt("KAFKA_FETCH_MESSAGE_MAX_BYTES", 1048576),               //Desc: Initial maximum number of bytes per topic+partition to request when fetching messages from the broker. If the client encounters a message larger than this value it will gradually try to increase it until the entire message can be fetched. <br>*Type: integer*   Range: 1 .. 1000000000
+		// "fetch.max.bytes":               envtool.GetEnvInt("KAFKA_FETCH_MAX_BYTES", 52428800),                      //Desc: Maximum amount of data the broker shall return for a Fetch request. Messages are fetched in batches by the consumer and if the first message batch in the first non-empty partition of the Fetch request is larger than this value, then the message batch will still be returned to ensure the consumer can make progress. The maximum message batch size accepted by the broker is defined via 'message.max.bytes' (broker config) or 'max.message.bytes' (broker topic config). 'fetch.max.bytes' is automatically adjusted upwards to be at least 'message.max.bytes' (consumer config). <br>*Type: integer*   Range: 0 .. 2147483135
+		"fetch.min.bytes": envtool.GetEnvInt("KAFKA_FETCH_MIN_BYTES", 1), //Desc: Minimum number of bytes the broker responds with. If fetch.wait.max.ms expires the accumulated data will be sent to the client regardless of this setting. <br>*Type: integer*   Range: 1 .. 100000000
+		// "fetch.error.backoff.ms":        envtool.GetEnvInt("KAFKA_FETCH_ERROR_BACKOFF_MS", 500),                    //Desc: How long to postpone the next fetch request for a topic+partition in case of a fetch error. <br>*Type: integer*   Range: 0 .. 300000
+		"isolation.level": envtool.GetEnv("KAFKA_ISOLATION_LEVEL", "read_committed"), //Desc: Controls how to read messages written transactionally: 'read_committed' - only return transactional messages which have been committed. 'read_uncommitted' - return all messages, even transactional messages which have been aborted. <br>*Type: enum value*   Range: read_uncommitted, read_committed
+		// "consume_cb":                              envtool.GetEnv("KAFKA_CONSUME_CB", ""),                                    //Desc: Message consume callback (set with rd_kafka_conf_set_consume_cb()) <br>*Type: see dedicated API*
+		// "rebalance_cb":                            envtool.GetEnv("KAFKA_REBALANCE_CB", ""),                                  //Desc: Called after consumer group has been rebalanced (set with rd_kafka_conf_set_rebalance_cb()) <br>*Type: see dedicated API*
+		// "offset_commit_cb":                        envtool.GetEnv("KAFKA_OFFSET_COMMIT_CB", ""),                              //Desc: Offset commit result propagation callback. (set with rd_kafka_conf_set_offset_commit_cb()) <br>*Type: see dedicated API*
+		// "enable.partition.eof": envtool.GetEnvBool("KAFKA_ENABLE_PARTITION_EOF", false),      //Desc: Emit RD_KAFKA_RESP_ERR__PARTITION_EOF event whenever the consumer reaches the end of a partition. <br>*Type: boolean*   Range: true, false
+		// "check.crcs":           envtool.GetEnvBool("KAFKA_CHECK_CRCS", false),                //Desc: Verify CRC32 of consumed messages, ensuring no on-the-wire or on-disk corruption to the messages occurred. This check comes at slightly increased CPU usage. <br>*Type: boolean*   Range: true, false
+		// "client.rack":          envtool.GetEnv("KAFKA_CLIENT_RACK", ""),                      //Desc: A rack identifier for this client. This can be any string value which indicates where this client is physically located. It corresponds with the broker config 'broker.rack'. <br>*Type: string*
+		// "client.dns.lookup":    envtool.GetEnv("KAFKA_CLIENT_DNS_LOOKUP", "use_all_dns_ips"), //Desc: Controls how the client uses DNS lookups. By default, when the lookup returns multiple IP addresses for a hostname, they will all be attempted for connection before the connection is considered failed. This applies to both bootstrap and advertised servers. If the value is set to 'resolve_canonical_bootstrap_servers_only', each entry will be resolved and expanded into a list of canonical names. NOTE: Default here is different from the Java client's default behavior, which connects only to the first IP address returned for a hostname.  <br>*Type: enum value*   Range: use_all_dns_ips, resolve_canonical_bootstrap_servers_only
+
+	}
+	return
+}
+
+// Shared by async producer and normal producer. No service calls both NewAsync and New
+func loadKafkaProducerConfig() (configuration kafka.ConfigMap) {
+	configuration = kafka.ConfigMap{
+		// "builtin.features":                        envtool.GetEnv("KAFKA_BUILTIN_FEATURES", kafka_builtin_features_default),  //Desc: Indicates the builtin features for this build of librdkafka. An application can either query this value or attempt to set it with its list of required features to check for library support. <br>*Type: CSV flags*
+		// "client.id":                               envtool.GetEnv("KAFKA_CLIENT_ID", "rdkafka"), //Desc: Client identifier. <br>*Type: string*
+		"bootstrap.servers":    envtool.GetEnv("KAFKA_HOSTS", "localhost:9093"),
+		"metadata.broker.list": envtool.GetEnv("KAFKA_METADATA_BROKER_LIST", ""), //Desc: Initial list of brokers as a CSV list of broker host or host:port. The application may also use 'rd_kafka_brokers_add()' to add brokers during runtime. <br>*Type: string*
+		// "message.max.bytes":                       envtool.GetEnvInt("KAFKA_MESSAGE_MAX_BYTES", 1000000),                     //Desc: Maximum Kafka protocol request message size. Due to differing framing overhead between protocol versions the producer is unable to reliably enforce a strict max message limit at produce time and may exceed the maximum size by one message in protocol ProduceRequests, the broker will enforce the the topic's 'max.message.bytes' limit (see Apache Kafka documentation). <br>*Type: integer*   Range: 1000 .. 1000000000
+		// "message.copy.max.bytes":                  envtool.GetEnvInt("KAFKA_MESSAGE_COPY_MAX_BYTES", 65535),                  //Desc: Maximum size for message to be copied to buffer. Messages larger than this will be passed by reference (zero-copy) at the expense of larger iovecs. <br>*Type: integer*   Range: 0 .. 1000000000
+		// "receive.message.max.bytes":               envtool.GetEnvInt("KAFKA_RECEIVE_MESSAGE_MAX_BYTES", 100000000),           //Desc: Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hickups. This value must be at least 'fetch.max.bytes'  + 512 to allow for protocol overhead; the value is adjusted automatically unless the configuration property is explicitly set. <br>*Type: integer*   Range: 1000 .. 2147483647
+		// "max.in.flight.requests.per.connection":   envtool.GetEnvInt("KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION", 1000000), //Desc: Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one. <br>*Type: integer*   Range: 1 .. 1000000
+		// "topic.metadata.refresh.interval.ms":      envtool.GetEnvInt("KAFKA_TOPIC_METADATA_REFRESH_INTERVAL_MS", 300000),     //Desc: Period of time in milliseconds at which topic and broker metadata is refreshed in order to proactively discover any new brokers, topics, partitions or partition leader changes. Use -1 to disable the intervalled refresh (not recommended). If there are no locally referenced topics (no topic objects created, no messages produced, no subscription or no assignment) then only the broker list will be refreshed every interval but no more often than every 10s. <br>*Type: integer*   Range: -1 .. 3600000
+		"metadata.max.age.ms": envtool.GetEnvInt("KAFKA_METADATA_MAX_AGE_MS", 900000), //Desc: Metadata cache max age. Defaults to topic.metadata.refresh.interval.ms * 3 <br>*Type: integer*   Range: 1 .. 86400000
+		// "topic.metadata.refresh.fast.interval.ms": envtool.GetEnvInt("KAFKA_TOPIC_METADATA_REFRESH_FAST_INTERVAL_MS", 100),   //Desc: When a topic loses its leader a new metadata request will be enqueued immediately and then with this initial interval, exponentially increasing upto 'retry.backoff.max.ms', until the topic metadata has been refreshed. If not set explicitly, it will be defaulted to 'retry.backoff.ms'. This is used to recover quickly from transitioning leader brokers. <br>*Type: integer*   Range: 1 .. 60000
+		// "topic.metadata.refresh.sparse":           envtool.GetEnvBool("KAFKA_TOPIC_METADATA_REFRESH_SPARSE", true),           //Desc: Sparse metadata requests (consumes less network bandwidth) <br>*Type: boolean*   Range: true, false
+		// "topic.metadata.propagation.max.ms":       envtool.GetEnvInt("KAFKA_TOPIC_METADATA_PROPAGATION_MAX_MS", 30000),       //Desc: Apache Kafka topic creation is asynchronous and it takes some time for a new topic to propagate throughout the cluster to all brokers. If a client requests topic metadata after manual topic creation but before the topic has been fully propagated to the broker the client is requesting metadata from, the topic will seem to be non-existent and the client will mark the topic as such, failing queued produced messages with 'ERR__UNKNOWN_TOPIC'. This setting delays marking a topic as non-existent until the configured propagation max time has passed. The maximum propagation time is calculated from the time the topic is first referenced in the client, e.g., on produce(). <br>*Type: integer*   Range: 0 .. 3600000
+		// "topic.blacklist":                         envtool.GetEnv("KAFKA_TOPIC_BLACKLIST", ""),                               //Desc: Topic blacklist, a comma-separated list of regular expressions for matching topic names that should be ignored in broker metadata information as if the topics did not exist. <br>*Type: pattern list*
+		// "debug":                                   envtool.GetEnv("KAFKA_DEBUG", "broker,topic,msg"),                         //Desc: A comma-separated list of debug contexts to enable. Detailed Producer debugging: broker,topic,msg. Consumer: consumer,cgrp,topic,fetch <br>*Type: CSV flags*   Range: generic, broker, topic, metadata, feature, queue, msg, protocol, cgrp, security, fetch, interceptor, plugin, consumer, admin, eos, mock, assignor, conf, all
+		"socket.timeout.ms": envtool.GetEnvInt("KAFKA_SOCKET_TIMEOUT_MS", 60000), //Desc: Default timeout for network requests. Producer: ProduceRequests will use the lesser value of 'socket.timeout.ms' and remaining 'message.timeout.ms' for the first message in the batch. Consumer: FetchRequests will use 'fetch.wait.max.ms' + 'socket.timeout.ms'. Admin: Admin requests will use 'socket.timeout.ms' or explicitly set 'rd_kafka_AdminOptions_set_operation_timeout()' value. <br>*Type: integer*   Range: 10 .. 300000
+		// "socket.send.buffer.bytes":           envtool.GetEnvInt("KAFKA_SOCKET_SEND_BUFFER_BYTES", 0),               //Desc: Broker socket send buffer size. System default is used if 0. <br>*Type: integer*   Range: 0 .. 100000000
+		// "socket.receive.buffer.bytes":        envtool.GetEnvInt("KAFKA_SOCKET_RECEIVE_BUFFER_BYTES", 0),            //Desc: Broker socket receive buffer size. System default is used if 0. <br>*Type: integer*   Range: 0 .. 100000000
+		"socket.keepalive.enable": envtool.GetEnvBool("KAFKA_SOCKET_KEEPALIVE_ENABLE", true), //Desc: Enable TCP keep-alives (SO_KEEPALIVE) on broker sockets <br>*Type: boolean*   Range: true, false
+		// "socket.nagle.disable":               envtool.GetEnvBool("KAFKA_SOCKET_NAGLE_DISABLE", false),              //Desc: Disable the Nagle algorithm (TCP_NODELAY) on broker sockets. <br>*Type: boolean*   Range: true, false
+		// "socket.max.fails":                   envtool.GetEnvInt("KAFKA_SOCKET_MAX_FAILS", 1),                       //Desc: Disconnect from broker when this number of send failures (e.g., timed out requests) is reached. Disable with 0. WARNING: It is highly recommended to leave this setting at its default value of 1 to avoid the client and broker to become desynchronized in case of request timeouts. NOTE: The connection is automatically re-established. <br>*Type: integer*   Range: 0 .. 1000000
+		// "broker.address.ttl":                 envtool.GetEnvInt("KAFKA_BROKER_ADDRESS_TTL", 1000),                  //Desc: How long to cache the broker address resolving results (milliseconds). <br>*Type: integer*   Range: 0 .. 86400000
+		// "broker.address.family":              envtool.GetEnv("KAFKA_BROKER_ADDRESS_FAMILY", "any"),                 //Desc: Allowed broker IP address families: any, v4, v6 <br>*Type: enum value*   Range: any, v4, v6
+		// "socket.connection.setup.timeout.ms": envtool.GetEnvInt("KAFKA_SOCKET_CONNECTION_SETUP_TIMEOUT_MS", 30000), //Desc: Maximum time allowed for broker connection setup (TCP connection setup as well SSL and SASL handshake). If the connection to the broker is not fully functional after this the connection will be closed and retried. <br>*Type: integer*   Range: 1000 .. 2147483647
+		"connections.max.idle.ms": envtool.GetEnvInt("KAFKA_CONNECTIONS_MAX_IDLE_MS", 0), //Desc: Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info). <br>*Type: integer*   Range: 0 .. 2147483647
+		// "reconnect.backoff.ms":               envtool.GetEnvInt("KAFKA_RECONNECT_BACKOFF_MS", 100),                 //Desc: The initial time to wait before reconnecting to a broker after the connection has been closed. The time is increased exponentially until 'reconnect.backoff.max.ms' is reached. -25% to +50% jitter is applied to each reconnect backoff. A value of 0 disables the backoff and reconnects immediately. <br>*Type: integer*   Range: 0 .. 3600000
+		// "reconnect.backoff.max.ms":           envtool.GetEnvInt("KAFKA_RECONNECT_BACKOFF_MAX_MS", 10000),           //Desc: The maximum time to wait before reconnecting to a broker after the connection has been closed. <br>*Type: integer*   Range: 0 .. 3600000
+		"statistics.interval.ms": envtool.GetEnvInt("KAFKA_STATISTICS_INTERVAL_MS", 0), //Desc: librdkafka statistics emit interval. The application also needs to register a stats callback using 'rd_kafka_conf_set_stats_cb()'. The granularity is 1000ms. A value of 0 disables statistics. <br>*Type: integer*   Range: 0 .. 86400000
+		// "enabled_events":                     envtool.GetEnvInt("KAFKA_ENABLED_EVENTS", 0),                         //Desc: See 'rd_kafka_conf_set_events()' <br>*Type: integer*   Range: 0 .. 2147483647
+		// "error_cb":                                envtool.GetEnv("KAFKA_ERROR_CB", ""),                                                   //Desc: Error callback (set with rd_kafka_conf_set_error_cb()) <br>*Type: see dedicated API*
+		// "throttle_cb":                             envtool.GetEnv("KAFKA_THROTTLE_CB", ""),                                                //Desc: Throttle callback (set with rd_kafka_conf_set_throttle_cb()) <br>*Type: see dedicated API*
+		// "stats_cb":                                envtool.GetEnv("KAFKA_STATS_CB", ""),                                                   //Desc: Statistics callback (set with rd_kafka_conf_set_stats_cb()) <br>*Type: see dedicated API*
+		// "log_cb":                                  envtool.GetEnv("KAFKA_LOG_CB", ""),                                                     //Desc: Log callback (set with rd_kafka_conf_set_log_cb()) <br>*Type: see dedicated API*
+		"log_level": envtool.GetEnvInt("KAFKA_LOG_LEVEL", 6), //Desc: Logging level (syslog(3) levels) <br>*Type: integer*   Range: 0 .. 7
+		// "log.queue":            envtool.GetEnvBool("KAFKA_LOG_QUEUE", false),           //Desc: Disable spontaneous log_cb from internal librdkafka threads, instead enqueue log messages on queue set with 'rd_kafka_set_log_queue()' and serve log callbacks or events through the standard poll APIs. **NOTE**: Log messages will linger in a temporary queue until the log queue has been set. <br>*Type: boolean*   Range: true, false
+		// "log.thread.name":      envtool.GetEnvBool("KAFKA_LOG_THREAD_NAME", true),      //Desc: Print internal thread name in log messages (useful for debugging librdkafka internals) <br>*Type: boolean*   Range: true, false
+		// "enable.random.seed":   envtool.GetEnvBool("KAFKA_ENABLE_RANDOM_SEED", true),   //Desc: If enabled librdkafka will initialize the PRNG with srand(current_time.milliseconds) on the first invocation of rd_kafka_new() (required only if rand_r() is not available on your platform). If disabled the application must call srand() prior to calling rd_kafka_new(). <br>*Type: boolean*   Range: true, false
+		"log.connection.close": envtool.GetEnvBool("KAFKA_LOG_CONNECTION_CLOSE", true), //Desc: Log broker disconnects. It might be useful to turn this off when interacting with 0.9 brokers with an aggressive 'connections.max.idle.ms' value. <br>*Type: boolean*   Range: true, false
+		// "background_event_cb":                     envtool.GetEnv("KAFKA_BACKGROUND_EVENT_CB", ""),                                        //Desc: Background queue event callback (set with rd_kafka_conf_set_background_event_cb()) <br>*Type: see dedicated API*
+		// "socket_cb":                               envtool.GetEnv("KAFKA_SOCKET_CB", ""),                                                  //Desc: Socket creation callback to provide race-free CLOEXEC <br>*Type: see dedicated API*
+		// "connect_cb":                              envtool.GetEnv("KAFKA_CONNECT_CB", ""),                                                 //Desc: Socket connect callback <br>*Type: see dedicated API*
+		// "closesocket_cb":                          envtool.GetEnv("KAFKA_CLOSESOCKET_CB", ""),                                             //Desc: Socket close callback <br>*Type: see dedicated API*
+		// "open_cb":                                 envtool.GetEnv("KAFKA_OPEN_CB", ""),                                                    //Desc: File open callback to provide race-free CLOEXEC <br>*Type: see dedicated API*
+		// "resolve_cb":                              envtool.GetEnv("KAFKA_RESOLVE_CB", ""),                                                 //Desc: Address resolution callback (set with rd_kafka_conf_set_resolve_cb()). <br>*Type: see dedicated API*
+		// "opaque":                                  envtool.GetEnv("KAFKA_OPAQUE", ""),                                                     //Desc: Application opaque (set with rd_kafka_conf_set_opaque()) <br>*Type: see dedicated API*
+		// "default_topic_conf":                      envtool.GetEnv("KAFKA_DEFAULT_TOPIC_CONF", ""),                                         //Desc: Default topic configuration for automatically subscribed topics <br>*Type: see dedicated API*
+		// "internal.termination.signal":    envtool.GetEnvInt("KAFKA_INTERNAL_TERMINATION_SIGNAL", 0),        //Desc: Signal that librdkafka will use to quickly terminate on rd_kafka_destroy(). If this signal is not set then there will be a delay before rd_kafka_wait_destroyed() returns true as internal threads are timing out their system calls. If this signal is set however the delay will be minimal. The application should mask this signal as an internal signal handler is installed. <br>*Type: integer*   Range: 0 .. 128
+		"api.version.request": envtool.GetEnvBool("KAFKA_API_VERSION_REQUEST", true), //Desc: Request broker's supported API versions to adjust functionality to available protocol features. If set to false, or the ApiVersionRequest fails, the fallback version 'broker.version.fallback' will be used. **NOTE**: Depends on broker version >=0.10.0. If the request is not supported by (an older) broker the 'broker.version.fallback' fallback is used. <br>*Type: boolean*   Range: true, false
+		// "api.version.request.timeout.ms": envtool.GetEnvInt("KAFKA_API_VERSION_REQUEST_TIMEOUT_MS", 10000), //Desc: Timeout for broker API version requests. <br>*Type: integer*   Range: 1 .. 300000
+		"api.version.fallback.ms":  envtool.GetEnvInt("KAFKA_API_VERSION_FALLBACK_MS", 0),      //Desc: Dictates how long the 'broker.version.fallback' fallback is used in the case the ApiVersionRequest fails. **NOTE**: The ApiVersionRequest is only issued when a new connection to the broker is made (such as after an upgrade). <br>*Type: integer*   Range: 0 .. 604800000
+		"broker.version.fallback":  envtool.GetEnv("KAFKA_BROKER_VERSION_FALLBACK", "0.10.0"),  //Desc: Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see 'api.version.request') making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for 'api.version.fallback.ms'. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value >= 0.10, such as 0.10.2.1, enables ApiVersionRequests. <br>*Type: string*
+		"allow.auto.create.topics": envtool.GetEnvBool("KAFKA_ALLOW_AUTO_CREATE_TOPICS", true), //Desc: Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. The broker must also be configured with 'auto.create.topics.enable=true' for this configuration to take effect. Note: the default value (true) for the producer is different from the default value (false) for the consumer. Further, the consumer default value is different from the Java consumer (true), and this property is not supported by the Java producer. Requires broker version >= 0.11.0.0, for older broker versions only the broker configuration applies. <br>*Type: boolean*   Range: true, false
+		"security.protocol":        envtool.GetEnv("KAFKA_SECURITY_PROTOCOL", "plaintext"),     //Desc: Protocol used to communicate with brokers. <br>*Type: enum value*   Range: plaintext, ssl, sasl_plaintext, sasl_ssl
+		// "ssl.cipher.suites":              envtool.GetEnv("KAFKA_SSL_CIPHER_SUITES", ""),                    //Desc: A cipher suite is a named combination of authentication, encryption, MAC and key exchange algorithm used to negotiate the security settings for a network connection using TLS or SSL network protocol. See manual page for 'ciphers(1)' and 'SSL_CTX_set_cipher_list(3). <br>*Type: string*
+		// "ssl.curves.list":                envtool.GetEnv("KAFKA_SSL_CURVES_LIST", ""),                      //Desc: The supported-curves extension in the TLS ClientHello message specifies the curves (standard/named, or 'explicit' GF(2^k) or GF(p)) the client is willing to have the server use. See manual page for 'SSL_CTX_set1_curves_list(3)'. OpenSSL >= 1.0.2 required. <br>*Type: string*
+		// "ssl.sigalgs.list":               envtool.GetEnv("KAFKA_SSL_SIGALGS_LIST", ""),                     //Desc: The client uses the TLS ClientHello signature_algorithms extension to indicate to the server which signature/hash algorithm pairs may be used in digital signatures. See manual page for 'SSL_CTX_set1_sigalgs_list(3)'. OpenSSL >= 1.0.2 required. <br>*Type: string*
+		// "ssl.key.location":               envtool.GetEnv("KAFKA_SSL_KEY_LOCATION", ""),                     //Desc: Path to client's private key (PEM) used for authentication. <br>*Type: string*
+		// "ssl.key.password":               envtool.GetEnv("KAFKA_SSL_KEY_PASSWORD", ""),                     //Desc: Private key passphrase (for use with 'ssl.key.location' and 'set_ssl_cert()') <br>*Type: string*
+		// "ssl.key.pem":                    envtool.GetEnv("KAFKA_SSL_KEY_PEM", ""),                          //Desc: Client's private key string (PEM format) used for authentication. <br>*Type: string*
+		// "ssl_key":                                 envtool.GetEnv("KAFKA_SSL_KEY", ""),                                                    //Desc: Client's private key as set by rd_kafka_conf_set_ssl_cert() <br>*Type: see dedicated API*
+		// "ssl.certificate.location": envtool.GetEnv("KAFKA_SSL_CERTIFICATE_LOCATION", ""), //Desc: Path to client's public key (PEM) used for authentication. <br>*Type: string*
+		// "ssl.certificate.pem":      envtool.GetEnv("KAFKA_SSL_CERTIFICATE_PEM", ""),      //Desc: Client's public key string (PEM format) used for authentication. <br>*Type: string*
+		// "ssl_certificate":                         envtool.GetEnv("KAFKA_SSL_CERTIFICATE", ""),                                            //Desc: Client's public key as set by rd_kafka_conf_set_ssl_cert() <br>*Type: see dedicated API*
+		// "ssl.ca.location": envtool.GetEnv("KAFKA_SSL_CA_LOCATION", ""), //Desc: File or directory path to CA certificate(s) for verifying the broker's key. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to 'probe'. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or 'ssl.ca.location' is set to 'probe' a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see 'OPENSSLDIR' in 'openssl version -a'). <br>*Type: string*
+		// "ssl.ca.pem":      envtool.GetEnv("KAFKA_SSL_CA_PEM", ""),      //Desc: CA certificate string (PEM format) for verifying the broker's key. <br>*Type: string*
+		// "ssl_ca":                                  envtool.GetEnv("KAFKA_SSL_CA", ""),                                                     //Desc: CA certificate as set by rd_kafka_conf_set_ssl_cert() <br>*Type: see dedicated API*
+		// "ssl.ca.certificate.stores":               envtool.GetEnv("KAFKA_SSL_CA_CERTIFICATE_STORES", "Root"),                              //Desc: Comma-separated list of Windows Certificate stores to load CA certificates from. Certificates will be loaded in the same order as stores are specified. If no certificates can be loaded from any of the specified stores an error is logged and the OpenSSL library's default CA location is used instead. Store names are typically one or more of: MY, Root, Trust, CA. <br>*Type: string*
+		// "ssl.crl.location":      envtool.GetEnv("KAFKA_SSL_CRL_LOCATION", ""),      //Desc: Path to CRL for verifying broker's certificate validity. <br>*Type: string*
+		// "ssl.keystore.location": envtool.GetEnv("KAFKA_SSL_KEYSTORE_LOCATION", ""), //Desc: Path to client's keystore (PKCS#12) used for authentication. <br>*Type: string*
+		// "ssl.keystore.password": envtool.GetEnv("KAFKA_SSL_KEYSTORE_PASSWORD", ""), //Desc: Client's keystore (PKCS#12) password. <br>*Type: string*
+		// "ssl.providers":         envtool.GetEnv("KAFKA_SSL_PROVIDERS", ""),         //Desc: Comma-separated list of OpenSSL 3.0.x implementation providers. E.g., "default,legacy". <br>*Type: string*
+		// "ssl.engine.id":         envtool.GetEnv("KAFKA_SSL_ENGINE_ID", "dynamic"),  //Desc: OpenSSL engine id is the name used for loading engine. <br>*Type: string*
+		// "ssl_engine_callback_data":                envtool.GetEnv("KAFKA_SSL_ENGINE_CALLBACK_DATA", ""),                                   //Desc: OpenSSL engine callback data (set with rd_kafka_conf_set_engine_callback_data()). <br>*Type: see dedicated API*
+		"enable.ssl.certificate.verification": envtool.GetEnvBool("KAFKA_SSL_VERIFY", true), //Desc: Enable OpenSSL's builtin broker (server) certificate verification. This verification can be extended by the application by implementing a certificate_verify_cb. Type: boolean
+		// "ssl.endpoint.identification.algorithm": envtool.GetEnv("KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM", "https"), //Desc: Endpoint identification algorithm to validate broker hostname using broker certificate. https - Server (broker) hostname verification as specified in RFC2818. none - No endpoint verification. OpenSSL >= 1.0.2 required. <br>*Type: enum value*   Range: none, https
+		// "ssl.certificate.verify_cb":               envtool.GetEnv("KAFKA_SSL_CERTIFICATE_VERIFY_CB", ""),                                  //Desc: Callback to verify the broker certificate chain. <br>*Type: see dedicated API*
+		"sasl.mechanisms": envtool.GetEnv("KAFKA_SASL_MECHANISMS", "GSSAPI"), //Desc: SASL mechanism to use for authentication. Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER. **NOTE**: Despite the name only one mechanism must be configured. <br>*Type: string*
+		// "sasl.kerberos.service.name":            envtool.GetEnv("KAFKA_SASL_KERBEROS_SERVICE_NAME", "kafka"),                            //Desc: Kerberos principal name that Kafka runs as, not including /hostname@REALM <br>*Type: string*
+		// "sasl.kerberos.principal":               envtool.GetEnv("KAFKA_SASL_KERBEROS_PRINCIPAL", "kafkaclient"),                         //Desc: This client's Kerberos principal name. (Not supported on Windows, will use the logon user's principal). <br>*Type: string*
+		// "sasl.kerberos.kinit.cmd":               envtool.GetEnv("KAFKA_SASL_KERBEROS_KINIT_CMD", kafka_sasl_kerberos_kinit_cmd_default), //Desc: Shell command to refresh or acquire the client's Kerberos ticket. This command is executed on client creation and every sasl.kerberos.min.time.before.relogin (0=disable). %{config.prop.name} is replaced by corresponding config object value. <br>*Type: string*
+		// "sasl.kerberos.keytab":                  envtool.GetEnv("KAFKA_SASL_KERBEROS_KEYTAB", ""),                                       //Desc: Path to Kerberos keytab file. This configuration property is only used as a variable in 'sasl.kerberos.kinit.cmd' as ' ... -t "%{sasl.kerberos.keytab}"'. <br>*Type: string*
+		// "sasl.kerberos.min.time.before.relogin": envtool.GetEnvInt("KAFKA_SASL_KERBEROS_MIN_TIME_BEFORE_RELOGIN", 60000),                //Desc: Minimum time in milliseconds between key refresh attempts. Disable automatic key refresh by setting this property to 0. <br>*Type: integer*   Range: 0 .. 86400000
+		"sasl.username": envtool.GetEnv("KAFKA_SASL_USERNAME", ""), //Desc: SASL username for use with the PLAIN and SASL-SCRAM-.. mechanisms <br>*Type: string*
+		"sasl.password": envtool.GetEnv("KAFKA_SASL_PASSWORD", ""), //Desc: SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism <br>*Type: string*
+		// "sasl.oauthbearer.config":               envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_CONFIG", ""),                                    //Desc: SASL/OAUTHBEARER configuration. The format is implementation-dependent and must be parsed accordingly. The default unsecured token implementation (see https://tools.ietf.org/html/rfc7515#appendix-A.5) recognizes space-separated name=value pairs with valid names including principalClaimName, principal, scopeClaimName, scope, and lifeSeconds. The default value for principalClaimName is "sub", the default value for scopeClaimName is "scope", and the default value for lifeSeconds is 3600. The scope value is CSV format with the default value being no/empty scope. For example: 'principalClaimName=azp principal=admin scopeClaimName=roles scope=role1,role2 lifeSeconds=600'. In addition, SASL extensions can be communicated to the broker via 'extension_NAME=value'. For example: 'principal=admin extension_traceId=123' <br>*Type: string*
+		// "enable.sasl.oauthbearer.unsecure.jwt":  envtool.GetEnvBool("KAFKA_ENABLE_SASL_OAUTHBEARER_UNSECURE_JWT", false),                //Desc: Enable the builtin unsecure JWT OAUTHBEARER token handler if no oauthbearer_refresh_cb has been set. This builtin handler should only be used for development or testing, and not in production. <br>*Type: boolean*   Range: true, false
+		// "oauthbearer_token_refresh_cb":          envtool.GetEnv("KAFKA_OAUTHBEARER_TOKEN_REFRESH_CB", ""),                               //Desc: SASL/OAUTHBEARER token refresh callback (set with rd_kafka_conf_set_oauthbearer_token_refresh_cb(), triggered by rd_kafka_poll(), et.al. This callback will be triggered when it is time to refresh the client's OAUTHBEARER token. Also see 'rd_kafka_conf_enable_sasl_queue()'. <br>*Type: see dedicated API*
+		// "sasl.oauthbearer.method":               envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_METHOD", "default"),                             //Desc: Set to "default" or "oidc" to control which login method to be used. If set to "oidc", the following properties must also be be specified: 'sasl.oauthbearer.client.id', 'sasl.oauthbearer.client.secret', and 'sasl.oauthbearer.token.endpoint.url'. <br>*Type: enum value*   Range: default, oidc
+		// "sasl.oauthbearer.client.id":            envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_CLIENT_ID", ""),                                 //Desc: Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.client.secret":        envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET", ""),                             //Desc: Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.scope":                envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_SCOPE", ""),                                     //Desc: Client use this to specify the scope of the access request to the broker. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.extensions":           envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_EXTENSIONS", ""),                                //Desc: Allow additional information to be provided to the broker. Comma-separated list of key=value pairs. E.g., "supportFeatureX=true,organizationId=sales-emea".Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "sasl.oauthbearer.token.endpoint.url":   envtool.GetEnv("KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT_URL", ""),                        //Desc: OAuth/OIDC issuer token endpoint HTTP(S) URI used to retrieve token. Only used when 'sasl.oauthbearer.method' is set to "oidc". <br>*Type: string*
+		// "plugin.library.paths": envtool.GetEnv("KAFKA_PLUGIN_LIBRARY_PATHS", ""), //Desc: List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically. <br>*Type: string*
+		// "interceptors":                            envtool.GetEnv("KAFKA_INTERCEPTORS", ""),                                               //Desc: Interceptors added through rd_kafka_conf_interceptor_add_..() and any configuration handled by interceptors. <br>*Type: see dedicated API*
+		// "client.rack":                  envtool.GetEnv("KAFKA_CLIENT_RACK", ""),                         //Desc: A rack identifier for this client. This can be any string value which indicates where this client is physically located. It corresponds with the broker config 'broker.rack'. <br>*Type: string*
+		// "transactional.id":             envtool.GetEnv("KAFKA_TRANSACTIONAL_ID", ""),                    //Desc: Enables the transactional producer. The transactional.id is used to identify the same transactional producer instance across process restarts. It allows the producer to guarantee that transactions corresponding to earlier instances of the same producer have been finalized prior to starting any new transactions, and that any zombie instances are fenced off. If no transactional.id is provided, then the producer is limited to idempotent delivery (if enable.idempotence is set). Requires broker version >= 0.11.0. <br>*Type: string*
+		// "transaction.timeout.ms":       envtool.GetEnvInt("KAFKA_TRANSACTION_TIMEOUT_MS", 60000),        //Desc: The maximum amount of time in milliseconds that the transaction coordinator will wait for a transaction status update from the producer before proactively aborting the ongoing transaction. If this value is larger than the 'transaction.max.timeout.ms' setting in the broker, the init_transactions() call will fail with ERR_INVALID_TRANSACTION_TIMEOUT. The transaction timeout automatically adjusts 'message.timeout.ms' and 'socket.timeout.ms', unless explicitly configured in which case they must not exceed the transaction timeout ('socket.timeout.ms' must be at least 100ms lower than 'transaction.timeout.ms'). This is also the default timeout value if no timeout (-1) is supplied to the transactional API methods. <br>*Type: integer*   Range: 1000 .. 2147483647
+		"enable.idempotence": envtool.GetEnvBool("KAFKA_ENABLE_IDEMPOTENCE", false), //Desc: When set to 'true', the producer will ensure that messages are successfully produced exactly once and in the original produce order. The following configuration properties are adjusted automatically (if not modified by the user) when idempotence is enabled: 'max.in.flight.requests.per.connection=5' (must be less than or equal to 5), 'retries=INT32_MAX' (must be greater than 0), 'acks=all', 'queuing.strategy=fifo'. Producer instantation will fail if user-supplied configuration is incompatible. <br>*Type: boolean*   Range: true, false
+		// "enable.gapless.guarantee":     envtool.GetEnvBool("KAFKA_ENABLE_GAPLESS_GUARANTEE", false),     //Desc: **EXPERIMENTAL**: subject to change or removal. When set to 'true', any error that could result in a gap in the produced message series when a batch of messages fails, will raise a fatal error (ERR__GAPLESS_GUARANTEE) and stop the producer. Messages failing due to 'message.timeout.ms' are not covered by this guarantee. Requires 'enable.idempotence=true'. <br>*Type: boolean*   Range: true, false
+		"queue.buffering.max.messages": envtool.GetEnvInt("KAFKA_QUEUE_BUFFERING_MAX_MESSAGES", 100000), //Desc: Maximum number of messages allowed on the producer queue. This queue is shared by all topics and partitions. A value of 0 disables this limit. <br>*Type: integer*   Range: 0 .. 2147483647
+		"queue.buffering.max.kbytes":   envtool.GetEnvInt("KAFKA_QUEUE_BUFFERING_MAX_KBYTES", 1048576),  //Desc: Maximum total message size sum allowed on the producer queue. This queue is shared by all topics and partitions. This property has higher priority than queue.buffering.max.messages. <br>*Type: integer*   Range: 1 .. 2147483647
+		"queue.buffering.max.ms":       envtool.GetEnvInt("KAFKA_QUEUE_BUFFERING_MAX_MS", 5),            //Desc: Delay in milliseconds to wait for messages in the producer queue to accumulate before constructing message batches (MessageSets) to transmit to brokers. A higher value allows larger and more effective (less overhead, improved compression) batches of messages to accumulate at the expense of increased message delivery latency. <br>*Type: float*   Range: 0 .. 900000
+		"message.send.max.retries":     envtool.GetEnvInt("KAFKA_MESSAGE_SEND_MAX_RETRIES", 2147483647), //Desc: How many times to retry sending a failing Message. **Note:** retrying may cause reordering unless 'enable.idempotence' is set to true. <br>*Type: integer*   Range: 0 .. 2147483647
+		// "retry.backoff.ms":                       envtool.GetEnvInt("KAFKA_RETRY_BACKOFF_MS", 100),                     //Desc: The backoff time in milliseconds before retrying a protocol request, this is the first backoff time, and will be backed off exponentially until number of retries is exhausted, and it's capped by retry.backoff.max.ms. <br>*Type: integer*   Range: 1 .. 300000
+		// "retry.backoff.max.ms":                   envtool.GetEnvInt("KAFKA_RETRY_BACKOFF_MAX_MS", 1000),                //Desc: The max backoff time in milliseconds before retrying a protocol request, this is the atmost backoff allowed for exponentially backed off requests. <br>*Type: integer*   Range: 1 .. 300000
+		// "queue.buffering.backpressure.threshold": envtool.GetEnvInt("KAFKA_QUEUE_BUFFERING_BACKPRESSURE_THRESHOLD", 1), //Desc: The threshold of outstanding not yet transmitted broker requests needed to backpressure the producer's message accumulator. If the number of not yet transmitted requests equals or exceeds this number, produce request creation that would have otherwise been triggered (for example, in accordance with linger.ms) will be delayed. A lower number yields larger and more effective batches. A higher value can improve latency when using compression on slow machines. <br>*Type: integer*   Range: 1 .. 1000000
+		"compression.codec":  envtool.GetEnv("KAFKA_COMPRESSION_CODEC", "none"),    //Desc: compression codec to use for compressing message sets. This is the default value for all topics, may be overridden by the topic configuration property 'compression.codec'.  <br>*Type: enum value*   Range: none, gzip, snappy, lz4, zstd
+		"batch.num.messages": envtool.GetEnvInt("KAFKA_BATCH_NUM_MESSAGES", 10000), //Desc: Maximum number of messages batched in one MessageSet. The total MessageSet size is also limited by batch.size and message.max.bytes. <br>*Type: integer*   Range: 1 .. 1000000
+		"batch.size":         envtool.GetEnvInt("KAFKA_BATCH_SIZE", 1000000),       //Desc: Maximum size (in bytes) of all messages batched in one MessageSet, including protocol framing overhead. This limit is applied after the first message has been added to the batch, regardless of the first message's size, this is to ensure that messages that exceed batch.size are produced. The total MessageSet size is also limited by batch.num.messages and message.max.bytes. <br>*Type: integer*   Range: 1 .. 2147483647
+		// "delivery.report.only.error":             envtool.GetEnvBool("KAFKA_DELIVERY_REPORT_ONLY_ERROR", false),        //Desc: Only provide delivery reports for failed messages. <br>*Type: boolean*   Range: true, false
+		// "dr_cb":                                   envtool.GetEnv("KAFKA_DR_CB", ""),                                                      //Desc: Delivery report callback (set with rd_kafka_conf_set_dr_cb()) <br>*Type: see dedicated API*
+		// "dr_msg_cb":                               envtool.GetEnv("KAFKA_DR_MSG_CB", ""),                                                  //Desc: Delivery report callback (set with rd_kafka_conf_set_dr_msg_cb()) <br>*Type: see dedicated API*
+		"sticky.partitioning.linger.ms": envtool.GetEnvInt("KAFKA_STICKY_PARTITIONING_LINGER_MS", 10), //Desc: Delay in milliseconds to wait to assign new sticky partitions for each topic. By default, set to double the time of linger.ms. To disable sticky behavior, set to 0. This behavior affects messages with the key NULL in all cases, and messages with key lengths of zero when the consistent_random partitioner is in use. These messages would otherwise be assigned randomly. A higher value allows for more effective batching of these messages. <br>*Type: integer*   Range: 0 .. 900000
+		// "client.dns.lookup":             envtool.GetEnv("KAFKA_CLIENT_DNS_LOOKUP", "use_all_dns_ips"), //Desc: Controls how the client uses DNS lookups. By default, when the lookup returns multiple IP addresses for a hostname, they will all be attempted for connection before the connection is considered failed. This applies to both bootstrap and advertised servers. If the value is set to 'resolve_canonical_bootstrap_servers_only', each entry will be resolved and expanded into a list of canonical names. NOTE: Default here is different from the Java client's default behavior, which connects only to the first IP address returned for a hostname.  <br>*Type: enum value*   Range: use_all_dns_ips, resolve_canonical_bootstrap_servers_only
+		"partitioner":        envtool.GetEnv("KAFKA_PARTITIONER", "consistent_random"), //Desc: artitioner: random - random distribution, consistent - CRC32 hash of key (Empty and NULL keys are mapped to single partition), consistent_random - CRC32 hash of key (Empty and NULL keys are randomly partitioned), murmur2 - Java Producer compatible Murmur2 hash of key (NULL keys are mapped to single partition), murmur2_random - Java Producer compatible Murmur2 hash of key (NULL keys are randomly partitioned. This is functionally equivalent to the default partitioner in the Java Producer.), fnv1a - FNV-1a hash of key (NULL keys are mapped to single partition), fnv1a_random - FNV-1a hash of key (NULL keys are randomly partitioned). Type: string
+		"request.timeout.ms": envtool.GetEnvInt("KAFKA_REQUEST_TIMEOUT_MS", 30000),     //Desc: The ack timeout of the producer request in milliseconds. This value is only enforced by the broker and relies on request.required.acks being != 0. Type: integer
+	}
+	return
+}

pkg/kafka/mock/admin.go

@@ -0,0 +1,17 @@
+package mock
+
+import (
+	"context"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+)
+
+// Admin is the mock admin
+type Admin struct {
+	MockFunc func(ctx context.Context, topics []kafka.TopicSpecification, options ...kafka.CreateTopicsAdminOption) ([]kafka.TopicResult, error)
+}
+
+// CreateTopics is the mock admin's version
+func (a *Admin) CreateTopics(ctx context.Context, topics []kafka.TopicSpecification, options ...kafka.CreateTopicsAdminOption) ([]kafka.TopicResult, error) {
+	return a.MockFunc(ctx, topics)
+}

pkg/kafka/mock/kafka_mock.go

@@ -0,0 +1,131 @@
+package mock
+
+import (
+	"fmt"
+	"sync"
+)
+
+// KafkaMock mock for Kafka utility
+type KafkaMock struct {
+	lock            sync.Mutex
+	err             error
+	produce         string
+	ProduceMessages map[string]map[string]interface{}
+}
+
+// Produce mocks produce method
+func (k *KafkaMock) Produce(topic string, key string, payload interface{}, headers map[string][]byte) error {
+	k.lock.Lock()
+	defer k.lock.Unlock()
+
+	if k.err != nil {
+		return k.err
+	}
+
+	k.produce = fmt.Sprintf("%s %v", topic, payload)
+	k.addMessage(topic, key, payload)
+
+	return nil
+}
+
+func (k *KafkaMock) ProduceBinary(topic string, key string, data []byte, header map[string][]byte) error {
+	if k.err != nil {
+		return k.err
+	}
+
+	k.produce = fmt.Sprintf("%s %v", topic, data)
+	k.addMessage(topic, key, data)
+
+	return nil
+}
+
+func (k *KafkaMock) ProduceBinaryToChannel(topic string, key string, payload []byte) error {
+	if k.err != nil {
+		return k.err
+	}
+
+	k.produce = fmt.Sprintf("%s %v", topic, payload)
+	k.addMessage(topic, key, payload)
+
+	return nil
+}
+
+/*
+func (k *KafkaMock) ProduceBatch(topic string, key string, payload [][]byte) error {
+	if k.err != nil {
+		return k.err
+	}
+
+	for _, item := range payload {
+		k.addMessage(topic, key, item)
+	}
+
+	return nil
+}*/
+
+func (k *KafkaMock) ProduceToChannel(topic string, key string, payload interface{}) error {
+	k.addMessage(topic, key, payload)
+
+	return nil
+}
+
+func (k *KafkaMock) ProduceSignalBatch(topic string, key string, payload interface{}) error {
+	if k.err != nil {
+		return k.err
+	}
+
+	k.produce = fmt.Sprintf("%s %v", topic, payload)
+	k.addMessage(topic, key, payload)
+
+	return nil
+}
+
+func (k *KafkaMock) ReadEvents() {
+	// nothing
+}
+
+// Close mocks close method
+func (k *KafkaMock) Close() {
+	// nothing
+}
+
+// SetError to set error for mock Kafka utility
+func (k *KafkaMock) SetError(err error) {
+	k.err = err
+}
+
+func (k *KafkaMock) addMessage(topic string, id string, payload interface{}) {
+	if k.ProduceMessages == nil {
+		k.ProduceMessages = map[string]map[string]interface{}{}
+	}
+
+	if k.ProduceMessages[topic] == nil {
+		k.ProduceMessages[topic] = map[string]interface{}{}
+	}
+
+	k.ProduceMessages[topic][id] = payload
+}
+
+func (k *KafkaMock) Reset() {
+	k.ProduceMessages = map[string]map[string]interface{}{}
+}
+
+func (k *KafkaMock) Len() int {
+	return 0
+}
+
+func (k *KafkaMock) Flush(timeoutMs int) int {
+	return 0
+}
+
+// GetProduce to set error for mock Kafka utility
+func (k *KafkaMock) GetProduce() string {
+	return k.produce
+}
+
+// GetKafkaMock returns mock Kafka utilty
+func GetKafkaMock(err error) *KafkaMock {
+	return &KafkaMock{
+		err: err,
+	}
+}

pkg/kafka/mock/kafka_test_case.go

@@ -0,0 +1,50 @@
+package mock
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"fiskerinc.com/modules/testhelper"
+)
+
+type KafkaTestCase struct {
+	ExpectedProduceMessages map[string]map[string]interface{}
+	MockError               error
+}
+
+func (tc *KafkaTestCase) Setup(mock *KafkaMock) {
+	mock.SetError(tc.MockError)
+}
+
+func (tc *KafkaTestCase) Validate(t *testing.T, name string, mock *KafkaMock) {
+	tc.validate(t, fmt.Sprintf("%s expected", name), tc.ExpectedProduceMessages, mock.ProduceMessages, true)
+	tc.validate(t, fmt.Sprintf("%s unexpected", name), mock.ProduceMessages, tc.ExpectedProduceMessages, false)
+}
+
+func (tc *KafkaTestCase) validate(t *testing.T, name string, control map[string]map[string]interface{}, source map[string]map[string]interface{}, compareValues bool) {
+	for topic, expected := range control {
+		if sent, hasTopic := source[topic]; hasTopic {
+			for id, payload := range expected {
+				if msg, hasID := sent[id]; hasID {
+					if compareValues {
+						tc.compare(t, name, payload, msg)
+					}
+				} else {
+					t.Errorf(testhelper.TestErrorTemplate, fmt.Sprintf("%s id", name), id, nil)
+				}
+			}
+		} else {
+			t.Errorf(testhelper.TestErrorTemplate, fmt.Sprintf("%s topic", name), topic, nil)
+		}
+	}
+}
+
+func (tc *KafkaTestCase) compare(t *testing.T, name string, control interface{}, source interface{}) {
+	data, err := json.Marshal(source)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, fmt.Sprintf("%s marshal payload", name), nil, err)
+	} else if control != string(data) {
+		t.Errorf(testhelper.TestErrorTemplate, fmt.Sprintf("%s message", name), control, string(data))
+	}
+}

pkg/kafka/mock/noncommital_consumer.go

@@ -0,0 +1,45 @@
+package mock
+
+import (
+	"context"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+)
+
+// NoncommitalConsumerMock mock for Kafka utility
+type NoncommitalConsumerMock struct{}
+
+func (c *NoncommitalConsumerMock) Subscribe(topics []string) error {
+	return nil
+}
+
+func (c *NoncommitalConsumerMock) ConsumeToChannel(topics []string, events chan *kafka.Message) error {
+	return nil
+}
+
+func (c *NoncommitalConsumerMock) ConsumeOrRebalancedCatch(topics []string, events chan *kafka.Message, reb chan struct{}) error {
+	return nil
+}
+
+func (c *NoncommitalConsumerMock) Seek(topic string, offset kafka.Offset) error {
+	return nil
+}
+
+func (c *NoncommitalConsumerMock) Commit(message *kafka.Message) ([]kafka.TopicPartition, error) {
+	return []kafka.TopicPartition{message.TopicPartition}, nil
+}
+
+func (c *NoncommitalConsumerMock) LastOffsetConsumed(topic string, partition int32) (kafka.Offset, error) {
+	return 1, nil
+}
+
+func (c *NoncommitalConsumerMock) Stop() {}
+
+func (c *NoncommitalConsumerMock) Check(ctx context.Context) error {
+	return nil
+}
+
+// passthrough function for CommitOffsets()
+func (c *NoncommitalConsumerMock) CommitOffsets(offsets []kafka.TopicPartition) ([]kafka.TopicPartition, error) {
+	return nil, nil
+}

pkg/kafka/producer.go

@@ -0,0 +1,192 @@
+package kafka
+
+import (
+	"context"
+	"encoding/json"
+
+	"fiskerinc.com/modules/logger"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+	"github.com/pkg/errors"
+)
+
+// ProducerInterface interface for Producer utilty
+type ProducerInterface interface {
+	Produce(topic string, key string, payload interface{}, headers map[string][]byte) error
+	ProduceBinary(topic string, key string, data []byte, headers map[string][]byte) error
+	ProduceBinaryToChannel(topic string, key string, payload []byte) error
+	ProduceToChannel(string, string, interface{}) error
+	ProduceSignalBatch(string, string, interface{}) error
+	ReadEvents()
+	Len() int
+	Flush(timeoutMs int) int
+	Close()
+}
+
+// Producer utility to produce messages
+type Producer struct {
+	producer *kafka.Producer
+}
+
+// NewProducer serves as factory method for producer to kafka
+func NewProducer(ctx context.Context) (ProducerInterface, error) {
+	var producer *kafka.Producer
+	configuration := loadKafkaProducerConfig()
+	producer, err := kafka.NewProducer(
+		&configuration,
+	)
+	logger.Info().Msgf("NewProducer  hosts: %s", kafkaHosts)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return &Producer{producer: producer}, nil
+}
+
+// SetProducer sets the producer instance
+func (p *Producer) SetProducer(producer *kafka.Producer) {
+	p.producer = producer
+}
+
+// Len returns len of messages in queue.
+func (p *Producer) Len() int {
+	return p.producer.Len()
+}
+
+// Flush calls producer's Flush function.
+func (p *Producer) Flush(timeoutMs int) int {
+	return p.producer.Flush(timeoutMs)
+}
+
+// Produce sends a Kafka Message to Kafka
+func (p *Producer) Produce(topic string, key string, payload interface{}, headers map[string][]byte) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return p.ProduceBinary(topic, key, v, headers)
+}
+
+func (p *Producer) makeHeaders(headers map[string][]byte) []kafka.Header {
+	if headers == nil {
+		return nil
+	}
+
+	i := 0
+	result := make([]kafka.Header, len(headers))
+
+	for key, value := range headers {
+		result[i].Key = key
+		result[i].Value = value
+		i++
+	}
+
+	return result
+}
+
+// Produce sends a Kafka Message to Kafka
+func (p *Producer) ProduceBinary(topic string, key string, data []byte, headers map[string][]byte) error {
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          data,
+	}
+
+	if headers != nil {
+		km.Headers = p.makeHeaders(headers)
+	}
+	deliveryChan := make(chan kafka.Event)
+
+	err := p.producer.Produce(&km, deliveryChan)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	for e := range deliveryChan {
+		switch ev := e.(type) {
+		case *kafka.Message:
+			// The message delivery report, indicating success or
+			// permanent failure after retries have been exhausted.
+			// Application level retries won't help since the client
+			// is already configured to do that.
+			m := ev
+			if m.TopicPartition.Error != nil {
+				logger.Error().Msgf("Delivery failed: %v\n", m.TopicPartition.Error)
+				return m.TopicPartition.Error
+			} else {
+				logger.Info().Msgf("Delivered message to topic %s [%d] at offset %v\n",
+					*m.TopicPartition.Topic, m.TopicPartition.Partition, m.TopicPartition.Offset)
+				return nil
+			}
+		default:
+			logger.Debug().Msgf("Ignored event: %s\n", ev)
+		}
+	}
+	return nil
+}
+
+// ProduceBatch is a stub for async producer
+func (p *Producer) ProduceBatch(topic string, key string, payload [][]byte) error {
+	// stub
+	return nil
+}
+
+// ProduceToChannel is a stub for async producer
+func (p *Producer) ProduceToChannel(topic string, key string, payload interface{}) error {
+	v, err := json.Marshal(payload)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          v,
+	}
+
+	p.producer.ProduceChannel() <- &km
+	return nil
+}
+
+// ProduceBinaryToChannel producing binary to a channel.
+func (p *Producer) ProduceBinaryToChannel(topic string, key string, payload []byte) error {
+	km := kafka.Message{
+		TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+		Key:            []byte(key),
+		Value:          payload,
+	}
+
+	p.producer.ProduceChannel() <- &km
+	return nil
+}
+
+// Produce sends a Kafka Message to Kafka
+func (p *Producer) ProduceSignalBatch(topic string, key string, payload interface{}) error {
+	// stub
+	return nil
+}
+
+// ReadEvents is a stub for async producer
+func (p *Producer) ReadEvents() {
+	for e := range p.producer.Events() {
+		switch ev := e.(type) {
+		case kafka.Error:
+			// Generic client instance-level errors, such as
+			// broker connection failures, authentication issues, etc.
+			//
+			// These errors should generally be considered informational
+			// as the underlying client will automatically try to
+			// recover from any errors encountered, the application
+			// does not need to take action on them.
+			logger.Error().Msgf("Error: %v\n", ev)
+		default:
+			logger.Debug().Msgf("Ignored event: %s\n", ev)
+		}
+	}
+}
+
+// Close the Kafka Producer
+func (p *Producer) Close() {
+	p.producer.Flush(0)
+	p.producer.Close()
+	p.producer = nil
+}

pkg/kafka/producer_test.go

@@ -0,0 +1,64 @@
+package kafka
+
+import (
+	"context"
+	"testing"
+
+	"fiskerinc.com/modules/testhelper"
+
+	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+)
+
+var producerConfig = &kafka.ConfigMap{
+	"socket.timeout.ms":  10,
+	"message.timeout.ms": 10,
+}
+
+func TestProducer(t *testing.T) {
+	p, err := kafka.NewProducer(producerConfig)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestProducer", nil, err)
+	}
+
+	producer := &Producer{producer: p}
+	err = producer.Produce("gotopic", "key", "test", nil)
+	if err == nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestProducer", "Local: Message timed out", err)
+	}
+}
+
+func TestProducerError(t *testing.T) {
+	p, err := kafka.NewProducer(producerConfig)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestProducer", nil, err)
+	}
+
+	producer := &Producer{producer: p}
+	err = producer.Produce("", "key", "test", nil)
+	if err == nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestProducer", "Local: Invalid argument of configuration", err)
+	}
+}
+
+func TestProduceBinary(t *testing.T) {
+	t.Skip()
+	ctx := context.Background()
+	producer, err := NewProducer(ctx)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	err = producer.ProduceBinary("topic", "key", []byte("This is a test"), nil)
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestProducer", "Local: Invalid argument of configuration", err)
+	}
+
+	err = producer.ProduceBinary("topic", "key", []byte("This is a test"), map[string][]byte{
+		"key1": []byte("val1"),
+		"key2": []byte("val2"),
+	})
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestProducer", "Local: Invalid argument of configuration", err)
+	}
+}

pkg/kafka/topics.go

@@ -0,0 +1,107 @@
+package kafka
+
+const AttendantService = "attendant_service"
+const AttendantServiceGRPCKafka = "attendant_service_grpc_kafka"
+const CargoService = "cargo_service"
+const DepotService = "depot_service"
+const DepotServiceGRPCKafka = "depot_service_grpc_kafka"
+const ValetService = "valet_service"
+const ValetServiceGRPCKafka = "valet_service_grpc_kafka"
+const LogService = "trexlog_service"
+const LogServiceGRPCKafka = "trexlog_service_grpc_kafka" // Stand in so that messages converted to grpc are sent to grpc service, and does not mess with old kafka messages
+const VehicleSignal = "vehicle_signal"
+const VehicleData = "vehicle_data"
+const OTAUpdateService = "ota_update"
+const SubscriptionService = "subscription"
+const RegexVehicleData = "^[A-Z0-9]{17}"
+
+// TRexHandlerTopics designates message topics
+// for handlers to destination service
+var TRexHandlerTopics = map[string]string{
+	"init":    DepotServiceGRPCKafka,
+	"del":     DepotServiceGRPCKafka,
+	"consent": DepotServiceGRPCKafka,
+
+	"car_state":           AttendantServiceGRPCKafka,
+	"car_update_status":   AttendantServiceGRPCKafka,
+	"car_update_download": AttendantServiceGRPCKafka,
+	"car_update_install":  AttendantServiceGRPCKafka,
+	"ecc_keys":            AttendantServiceGRPCKafka,
+	"get_filekeys":        AttendantServiceGRPCKafka,
+	"dtcs":                AttendantServiceGRPCKafka,
+
+	"charge_settings":    ValetServiceGRPCKafka,
+	"departure_schedule": ValetServiceGRPCKafka,
+	"charging_command":   ValetServiceGRPCKafka,
+
+	"error":      LogServiceGRPCKafka,
+	"trex_log":   LogServiceGRPCKafka,
+	"hmi_log":    LogServiceGRPCKafka,
+	"mobile_log": LogServiceGRPCKafka,
+
+	"canbus": VehicleData,
+}
+
+// HMIHandlerTopics designates message topics
+// for handlers to destination service
+var HMIHandlerTopics = map[string]string{
+	"init": DepotServiceGRPCKafka,
+	"del":  DepotServiceGRPCKafka,
+
+	"car_update_download": AttendantServiceGRPCKafka,
+	"car_update_install":  AttendantServiceGRPCKafka,
+	"car_update_status":   AttendantServiceGRPCKafka,
+	"get_filekeys":        AttendantServiceGRPCKafka,
+	"update_approve":      AttendantServiceGRPCKafka,
+
+	"consent":         ValetServiceGRPCKafka,
+	"ble_key":         ValetServiceGRPCKafka,
+	"map_history":     ValetServiceGRPCKafka,
+	"profiles":        ValetServiceGRPCKafka,
+	"profile_delete":  ValetServiceGRPCKafka,
+	"settings_update": ValetServiceGRPCKafka,
+	"user_pois":       ValetServiceGRPCKafka,
+}
+
+// MobileHandlerTopics designates message topics
+// for handlers to destination service
+var MobileHandlerTopics = map[string]string{
+	"init": DepotServiceGRPCKafka,
+	"del":  DepotServiceGRPCKafka,
+
+	"update_approve": AttendantServiceGRPCKafka,
+	"updates_get":    AttendantServiceGRPCKafka,
+
+	"remote_command":         ValetServiceGRPCKafka,
+	"car_locations":          ValetServiceGRPCKafka,
+	"charge_settings":        ValetServiceGRPCKafka,
+	"charge_settings_get":    ValetServiceGRPCKafka,
+	"digital_twin":           ValetServiceGRPCKafka,
+	"departure_schedule":     ValetServiceGRPCKafka,
+	"departure_schedule_get": ValetServiceGRPCKafka,
+	"map_destination":        ValetServiceGRPCKafka,
+	"map_route":              ValetServiceGRPCKafka,
+	"map_history_get":        ValetServiceGRPCKafka,
+	"map_history_add":        ValetServiceGRPCKafka,
+	"ota_install":            ValetServiceGRPCKafka,
+	"profiles":               ValetServiceGRPCKafka,
+	"settings_update":        ValetServiceGRPCKafka,
+	"store_inventory":        ValetServiceGRPCKafka,
+	"store_purchase":         ValetServiceGRPCKafka,
+	"user_pois_get":          ValetServiceGRPCKafka,
+	"user_poi_create":        ValetServiceGRPCKafka,
+	"user_poi_edit":          ValetServiceGRPCKafka,
+	"user_poi_delete":        ValetServiceGRPCKafka,
+	"wake_car":               ValetServiceGRPCKafka,
+	"manual_issue":           ValetServiceGRPCKafka,
+}
+
+// ServiceHandlerTopics designates message topics
+// for handlers to destination service. For routing
+// messages between cloud services
+var ServiceHandlerTopics = map[string]string{
+	"send_manifest":                AttendantServiceGRPCKafka,
+	"sms_delivery_status_manifest": AttendantServiceGRPCKafka,
+
+	"remote_command": ValetServiceGRPCKafka,
+}

pkg/kafka/unhandled_msg.go

@@ -0,0 +1,10 @@
+package kafka
+
+import (
+	"fiskerinc.com/modules/common"
+	"github.com/pkg/errors"
+)
+
+var ErrUnhandledMessage = func(device common.Device, id string, handler string, msg interface{}) error {
+	return errors.Errorf("unhandled message, device: %v, id: %s handler: %s, payload: %v", device, id, handler, msg)
+}

pkg/kafka/unhandled_msg_test.go

@@ -0,0 +1,27 @@
+package kafka_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/kafka"
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestErrUnhandledMessage(t *testing.T) {
+	id := "11111111111111111"
+	handler := "non_existent"
+	payload := []byte(`{"data":"payload"}`)
+	expectedTrexErr := `unhandled message, device: trex, id: 11111111111111111 handler: non_existent, payload: {"data":"payload"}`
+	expectedHMIErr := `unhandled message, device: hmi, id: 11111111111111111 handler: non_existent, payload: [123 34 100 97 116 97 34 58 34 112 97 121 108 111 97 100 34 125]`
+
+	err := kafka.ErrUnhandledMessage(common.TRex, id, handler, string(payload))
+	if err.Error() != expectedTrexErr {
+		t.Errorf(testhelper.TestErrorTemplate, "trex message", expectedTrexErr, err.Error())
+	}
+
+	err = kafka.ErrUnhandledMessage(common.HMI, id, handler, payload)
+	if err.Error() != expectedHMIErr {
+		t.Errorf(testhelper.TestErrorTemplate, "hmi message", expectedHMIErr, err.Error())
+	}
+}