Refactor kafka to pure Go (franz-go), fix DBC stubs, update Dockerfile
This commit is contained in:
Chris Rai
@@ -3,14 +3,14 @@ package kafka
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"fiskerinc.com/modules/logger"
|
||||
|
||||
"github.com/confluentinc/confluent-kafka-go/v2/kafka"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/twmb/franz-go/pkg/kgo"
|
||||
)
|
||||
|
||||
// ProducerInterface interface for Producer utilty
|
||||
// ProducerInterface interface for Producer utility
|
||||
type ProducerInterface interface {
|
||||
Produce(topic string, key string, payload interface{}, headers map[string][]byte) error
|
||||
ProduceBinary(topic string, key string, data []byte, headers map[string][]byte) error
|
||||
@@ -25,168 +25,179 @@ type ProducerInterface interface {
|
||||
|
||||
// Producer utility to produce messages
|
||||
type Producer struct {
|
||||
producer *kafka.Producer
|
||||
client *kgo.Client
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
pending int
|
||||
pendingMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewProducer serves as factory method for producer to kafka
|
||||
func NewProducer(ctx context.Context) (ProducerInterface, error) {
|
||||
var producer *kafka.Producer
|
||||
configuration := loadKafkaProducerConfig()
|
||||
producer, err := kafka.NewProducer(
|
||||
&configuration,
|
||||
cfg := LoadConfig()
|
||||
opts := buildClientOpts(cfg)
|
||||
|
||||
// Producer-specific options
|
||||
opts = append(opts,
|
||||
kgo.ProducerBatchCompression(kgo.NoCompression()),
|
||||
kgo.AllowAutoTopicCreation(),
|
||||
)
|
||||
logger.Info().Msgf("NewProducer hosts: %s", kafkaHosts)
|
||||
|
||||
client, err := kgo.NewClient(opts...)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
return &Producer{producer: producer}, nil
|
||||
|
||||
logger.Info().Msgf("NewProducer hosts: %v", cfg.Brokers)
|
||||
|
||||
pctx, cancel := context.WithCancel(ctx)
|
||||
return &Producer{
|
||||
client: client,
|
||||
ctx: pctx,
|
||||
cancel: cancel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SetProducer sets the producer instance
|
||||
func (p *Producer) SetProducer(producer *kafka.Producer) {
|
||||
p.producer = producer
|
||||
}
|
||||
|
||||
// Len returns len of messages in queue.
|
||||
// Len returns len of messages in queue (approximate)
|
||||
func (p *Producer) Len() int {
|
||||
return p.producer.Len()
|
||||
p.pendingMu.Lock()
|
||||
defer p.pendingMu.Unlock()
|
||||
return p.pending
|
||||
}
|
||||
|
||||
// Flush calls producer's Flush function.
|
||||
// Flush waits for all buffered records to be flushed
|
||||
func (p *Producer) Flush(timeoutMs int) int {
|
||||
return p.producer.Flush(timeoutMs)
|
||||
ctx := p.ctx
|
||||
if timeoutMs > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(p.ctx, durationMs(timeoutMs))
|
||||
defer cancel()
|
||||
}
|
||||
if err := p.client.Flush(ctx); err != nil {
|
||||
logger.Warn().Err(err).Msg("flush incomplete")
|
||||
return p.Len()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Produce sends a Kafka Message to Kafka
|
||||
// Produce sends a JSON-encoded message to Kafka
|
||||
func (p *Producer) Produce(topic string, key string, payload interface{}, headers map[string][]byte) error {
|
||||
v, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return p.ProduceBinary(topic, key, v, headers)
|
||||
}
|
||||
|
||||
func (p *Producer) makeHeaders(headers map[string][]byte) []kafka.Header {
|
||||
func makeHeaders(headers map[string][]byte) []kgo.RecordHeader {
|
||||
if headers == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
i := 0
|
||||
result := make([]kafka.Header, len(headers))
|
||||
|
||||
for key, value := range headers {
|
||||
result[i].Key = key
|
||||
result[i].Value = value
|
||||
i++
|
||||
result := make([]kgo.RecordHeader, 0, len(headers))
|
||||
for k, v := range headers {
|
||||
result = append(result, kgo.RecordHeader{Key: k, Value: v})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Produce sends a Kafka Message to Kafka
|
||||
// ProduceBinary sends a binary message to Kafka synchronously
|
||||
func (p *Producer) ProduceBinary(topic string, key string, data []byte, headers map[string][]byte) error {
|
||||
km := kafka.Message{
|
||||
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
|
||||
Key: []byte(key),
|
||||
Value: data,
|
||||
record := &kgo.Record{
|
||||
Topic: topic,
|
||||
Key: []byte(key),
|
||||
Value: data,
|
||||
Headers: makeHeaders(headers),
|
||||
}
|
||||
|
||||
if headers != nil {
|
||||
km.Headers = p.makeHeaders(headers)
|
||||
}
|
||||
deliveryChan := make(chan kafka.Event)
|
||||
p.pendingMu.Lock()
|
||||
p.pending++
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
err := p.producer.Produce(&km, deliveryChan)
|
||||
if err != nil {
|
||||
result := p.client.ProduceSync(p.ctx, record)
|
||||
|
||||
p.pendingMu.Lock()
|
||||
p.pending--
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
if err := result.FirstErr(); err != nil {
|
||||
logger.Error().Err(err).Msgf("Delivery failed to topic %s", topic)
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
for e := range deliveryChan {
|
||||
switch ev := e.(type) {
|
||||
case *kafka.Message:
|
||||
// The message delivery report, indicating success or
|
||||
// permanent failure after retries have been exhausted.
|
||||
// Application level retries won't help since the client
|
||||
// is already configured to do that.
|
||||
m := ev
|
||||
if m.TopicPartition.Error != nil {
|
||||
logger.Error().Msgf("Delivery failed: %v\n", m.TopicPartition.Error)
|
||||
return m.TopicPartition.Error
|
||||
} else {
|
||||
logger.Info().Msgf("Delivered message to topic %s [%d] at offset %v\n",
|
||||
*m.TopicPartition.Topic, m.TopicPartition.Partition, m.TopicPartition.Offset)
|
||||
return nil
|
||||
}
|
||||
default:
|
||||
logger.Debug().Msgf("Ignored event: %s\n", ev)
|
||||
}
|
||||
}
|
||||
|
||||
r := result[0].Record
|
||||
logger.Info().Msgf("Delivered message to topic %s [%d] at offset %d", r.Topic, r.Partition, r.Offset)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProduceBatch is a stub for async producer
|
||||
func (p *Producer) ProduceBatch(topic string, key string, payload [][]byte) error {
|
||||
// stub
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProduceToChannel is a stub for async producer
|
||||
// ProduceToChannel sends a message asynchronously
|
||||
func (p *Producer) ProduceToChannel(topic string, key string, payload interface{}) error {
|
||||
v, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
km := kafka.Message{
|
||||
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
|
||||
Key: []byte(key),
|
||||
Value: v,
|
||||
record := &kgo.Record{
|
||||
Topic: topic,
|
||||
Key: []byte(key),
|
||||
Value: v,
|
||||
}
|
||||
|
||||
p.producer.ProduceChannel() <- &km
|
||||
return nil
|
||||
}
|
||||
p.pendingMu.Lock()
|
||||
p.pending++
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
// ProduceBinaryToChannel producing binary to a channel.
|
||||
func (p *Producer) ProduceBinaryToChannel(topic string, key string, payload []byte) error {
|
||||
km := kafka.Message{
|
||||
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
|
||||
Key: []byte(key),
|
||||
Value: payload,
|
||||
}
|
||||
p.client.Produce(p.ctx, record, func(r *kgo.Record, err error) {
|
||||
p.pendingMu.Lock()
|
||||
p.pending--
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
p.producer.ProduceChannel() <- &km
|
||||
return nil
|
||||
}
|
||||
|
||||
// Produce sends a Kafka Message to Kafka
|
||||
func (p *Producer) ProduceSignalBatch(topic string, key string, payload interface{}) error {
|
||||
// stub
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadEvents is a stub for async producer
|
||||
func (p *Producer) ReadEvents() {
|
||||
for e := range p.producer.Events() {
|
||||
switch ev := e.(type) {
|
||||
case kafka.Error:
|
||||
// Generic client instance-level errors, such as
|
||||
// broker connection failures, authentication issues, etc.
|
||||
//
|
||||
// These errors should generally be considered informational
|
||||
// as the underlying client will automatically try to
|
||||
// recover from any errors encountered, the application
|
||||
// does not need to take action on them.
|
||||
logger.Error().Msgf("Error: %v\n", ev)
|
||||
default:
|
||||
logger.Debug().Msgf("Ignored event: %s\n", ev)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msgf("Async delivery failed to topic %s", topic)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProduceBinaryToChannel sends binary data asynchronously
|
||||
func (p *Producer) ProduceBinaryToChannel(topic string, key string, payload []byte) error {
|
||||
record := &kgo.Record{
|
||||
Topic: topic,
|
||||
Key: []byte(key),
|
||||
Value: payload,
|
||||
}
|
||||
|
||||
p.pendingMu.Lock()
|
||||
p.pending++
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
p.client.Produce(p.ctx, record, func(r *kgo.Record, err error) {
|
||||
p.pendingMu.Lock()
|
||||
p.pending--
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msgf("Async delivery failed to topic %s", topic)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProduceSignalBatch is a stub for batch signal production
|
||||
func (p *Producer) ProduceSignalBatch(topic string, key string, payload interface{}) error {
|
||||
// stub - implement if needed
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadEvents is a no-op for franz-go (callbacks handle events)
|
||||
func (p *Producer) ReadEvents() {
|
||||
// franz-go uses callbacks, this is kept for interface compatibility
|
||||
}
|
||||
|
||||
// Close the Kafka Producer
|
||||
func (p *Producer) Close() {
|
||||
p.producer.Flush(0)
|
||||
p.producer.Close()
|
||||
p.producer = nil
|
||||
p.Flush(5000)
|
||||
p.cancel()
|
||||
p.client.Close()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user