Refactor kafka to pure Go (franz-go), fix DBC stubs, update Dockerfile
This commit is contained in:
Chris Rai
@@ -3,168 +3,200 @@ package kafka
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"fiskerinc.com/modules/logger"
|
||||
|
||||
"github.com/confluentinc/confluent-kafka-go/v2/kafka"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/twmb/franz-go/pkg/kgo"
|
||||
)
|
||||
|
||||
// AsyncProducer is an async-first producer implementation
|
||||
type AsyncProducer struct {
|
||||
producer *kafka.Producer
|
||||
client *kgo.Client
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
pending int
|
||||
pendingMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewProducer serves as factory method for producer to kafka
|
||||
// NewAsyncProducer serves as factory method for async producer to kafka
|
||||
func NewAsyncProducer(ctx context.Context) (ProducerInterface, error) {
|
||||
var producer *kafka.Producer
|
||||
configuration := loadKafkaProducerConfig()
|
||||
producer, err := kafka.NewProducer(
|
||||
&configuration,
|
||||
// kafkaTrace.WithContext(ctx),
|
||||
cfg := LoadConfig()
|
||||
opts := buildClientOpts(cfg)
|
||||
|
||||
// Async producer options - optimized for throughput
|
||||
opts = append(opts,
|
||||
kgo.ProducerBatchCompression(kgo.NoCompression()),
|
||||
kgo.AllowAutoTopicCreation(),
|
||||
kgo.ProducerLinger(5), // 5ms linger for batching
|
||||
)
|
||||
logger.Info().Msgf("NewAsyncProducer hosts: %s", kafkaHosts)
|
||||
|
||||
client, err := kgo.NewClient(opts...)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
return &AsyncProducer{producer: producer}, nil
|
||||
|
||||
logger.Info().Msgf("NewAsyncProducer hosts: %v", cfg.Brokers)
|
||||
|
||||
pctx, cancel := context.WithCancel(ctx)
|
||||
return &AsyncProducer{
|
||||
client: client,
|
||||
ctx: pctx,
|
||||
cancel: cancel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SetProducer sets the producer instance
|
||||
func (p *AsyncProducer) SetProducer(producer *kafka.Producer) {
|
||||
p.producer = producer
|
||||
}
|
||||
|
||||
// Len returns len of messages in queue.
|
||||
// Len returns approximate pending message count
|
||||
func (p *AsyncProducer) Len() int {
|
||||
return p.producer.Len()
|
||||
p.pendingMu.Lock()
|
||||
defer p.pendingMu.Unlock()
|
||||
return p.pending
|
||||
}
|
||||
|
||||
// Flush calls producer's Flush function.
|
||||
// Flush waits for all buffered records to be flushed
|
||||
func (p *AsyncProducer) Flush(timeoutMs int) int {
|
||||
return p.producer.Flush(timeoutMs)
|
||||
ctx := p.ctx
|
||||
if timeoutMs > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(p.ctx, durationMs(timeoutMs))
|
||||
defer cancel()
|
||||
}
|
||||
if err := p.client.Flush(ctx); err != nil {
|
||||
logger.Warn().Err(err).Msg("flush incomplete")
|
||||
return p.Len()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Produce sends a Kafka Message to Kafka
|
||||
// Produce sends a JSON-encoded message asynchronously
|
||||
func (p *AsyncProducer) Produce(topic string, key string, payload interface{}, headers map[string][]byte) error {
|
||||
v, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return p.ProduceBinary(topic, key, v, headers)
|
||||
}
|
||||
|
||||
func (p *AsyncProducer) makeHeaders(headers map[string][]byte) []kafka.Header {
|
||||
if headers == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
i := 0
|
||||
result := make([]kafka.Header, len(headers))
|
||||
|
||||
for key, value := range headers {
|
||||
result[i] = kafka.Header{
|
||||
Key: key,
|
||||
}
|
||||
copy(result[i].Value, value)
|
||||
i++
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// ProduceBinary sends a binary message asynchronously
|
||||
func (p *AsyncProducer) ProduceBinary(topic string, key string, data []byte, headers map[string][]byte) error {
|
||||
km := kafka.Message{
|
||||
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
|
||||
Key: []byte(key),
|
||||
Value: data,
|
||||
}
|
||||
if headers != nil {
|
||||
km.Headers = p.makeHeaders(headers)
|
||||
record := &kgo.Record{
|
||||
Topic: topic,
|
||||
Key: []byte(key),
|
||||
Value: data,
|
||||
Headers: makeHeaders(headers),
|
||||
}
|
||||
|
||||
err := p.producer.Produce(&km, nil)
|
||||
if err != nil {
|
||||
// error handle inability to connect to kafka
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
p.pendingMu.Lock()
|
||||
p.pending++
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
return err
|
||||
p.client.Produce(p.ctx, record, func(r *kgo.Record, err error) {
|
||||
p.pendingMu.Lock()
|
||||
p.pending--
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msgf("Async delivery failed to topic %s", topic)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProduceToChannel sends a list of Kafka Messages to Kafka
|
||||
// ProduceToChannel sends a message asynchronously
|
||||
func (p *AsyncProducer) ProduceToChannel(topic string, key string, payload interface{}) error {
|
||||
v, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
km := kafka.Message{
|
||||
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
|
||||
Key: []byte(key),
|
||||
Value: v,
|
||||
record := &kgo.Record{
|
||||
Topic: topic,
|
||||
Key: []byte(key),
|
||||
Value: v,
|
||||
}
|
||||
|
||||
p.producer.ProduceChannel() <- &km
|
||||
p.pendingMu.Lock()
|
||||
p.pending++
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
p.client.Produce(p.ctx, record, func(r *kgo.Record, err error) {
|
||||
p.pendingMu.Lock()
|
||||
p.pending--
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msgf("Async delivery failed to topic %s", topic)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProduceBinaryToChannel sends binary data asynchronously
|
||||
func (p *AsyncProducer) ProduceBinaryToChannel(topic string, key string, payload []byte) error {
|
||||
km := kafka.Message{
|
||||
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
|
||||
Key: []byte(key),
|
||||
Value: payload,
|
||||
record := &kgo.Record{
|
||||
Topic: topic,
|
||||
Key: []byte(key),
|
||||
Value: payload,
|
||||
}
|
||||
|
||||
p.producer.ProduceChannel() <- &km
|
||||
p.pendingMu.Lock()
|
||||
p.pending++
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
p.client.Produce(p.ctx, record, func(r *kgo.Record, err error) {
|
||||
p.pendingMu.Lock()
|
||||
p.pending--
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msgf("Async delivery failed to topic %s", topic)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProduceSignalBatch sends a specified batch of CAN signals
|
||||
//
|
||||
// NOTE: does NOT produce proper JSON, Value is custom for Clickhouse
|
||||
// ProduceSignalBatch sends a batch of CAN signals (custom format for Clickhouse)
|
||||
func (p *AsyncProducer) ProduceSignalBatch(topic string, key string, payload interface{}) error {
|
||||
v, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
km := kafka.Message{
|
||||
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
|
||||
Key: []byte(key),
|
||||
Value: v[1 : len(v)-1],
|
||||
// Strip JSON array brackets for Clickhouse format
|
||||
record := &kgo.Record{
|
||||
Topic: topic,
|
||||
Key: []byte(key),
|
||||
Value: v[1 : len(v)-1],
|
||||
}
|
||||
|
||||
err = p.producer.Produce(&km, nil)
|
||||
if err != nil {
|
||||
// error handle inability to connect to kafka
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
p.pendingMu.Lock()
|
||||
p.pending++
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
p.client.Produce(p.ctx, record, func(r *kgo.Record, err error) {
|
||||
p.pendingMu.Lock()
|
||||
p.pending--
|
||||
p.pendingMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msgf("Async delivery failed to topic %s", topic)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadEvents listens to producer's events which are emitted as responses to producing events
|
||||
// ReadEvents listens to producer events (callbacks handle this in franz-go)
|
||||
func (p *AsyncProducer) ReadEvents() {
|
||||
for e := range p.producer.Events() {
|
||||
switch m := e.(type) {
|
||||
case *kafka.Message:
|
||||
if m.TopicPartition.Error != nil {
|
||||
logger.Error().
|
||||
Err(m.TopicPartition.Error).
|
||||
Send()
|
||||
}
|
||||
case kafka.Error:
|
||||
logger.Error().
|
||||
Str("err", m.String()).
|
||||
Send()
|
||||
}
|
||||
}
|
||||
// franz-go uses callbacks, kept for interface compatibility
|
||||
}
|
||||
|
||||
// Close the Kafka Producer
|
||||
func (p *AsyncProducer) Close() {
|
||||
p.producer.Flush(0)
|
||||
p.producer.Close()
|
||||
p.producer = nil
|
||||
p.Flush(5000)
|
||||
p.cancel()
|
||||
p.client.Close()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user