Refactor kafka to pure Go (franz-go), fix DBC stubs, update Dockerfile

pkg/kafka/admin.go

@@ -2,54 +2,52 @@ package kafka
 
 import (
 	"context"
-	"fmt"
 	"time"
 
-	"fiskerinc.com/modules/utils/envtool"
-
-	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
+	"github.com/fiskerinc/cloud-services/pkg/logger"
+	"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
 	"github.com/pkg/errors"
+	"github.com/twmb/franz-go/pkg/kadm"
+	"github.com/twmb/franz-go/pkg/kgo"
 )
 
-// AdminClient interface
-type AdminClient interface {
-	CreateTopics(ctx context.Context, topics []kafka.TopicSpecification, options ...kafka.CreateTopicsAdminOption) ([]kafka.TopicResult, error)
-}
-
-// Global
-var (
-	Admin      AdminClient
-	kafkaHosts string = envtool.GetEnv("KAFKA_HOSTS", "localhost:9093")
-)
+// Global admin client
+var adminClient *kadm.Client
 
 func init() {
-	Admin, _ = kafka.NewAdminClient(&kafka.ConfigMap{"bootstrap.servers": kafkaHosts})
+	cfg := LoadConfig()
+	client, err := kgo.NewClient(kgo.SeedBrokers(cfg.Brokers...))
+	if err != nil {
+		logger.Warn().Err(err).Msg("failed to create kafka admin client")
+		return
+	}
+	adminClient = kadm.NewClient(client)
 }
 
-// EnsureTopicsExist checks Kafka for topic, if it doesn't exist it creates topic
+// EnsureTopicsExist checks Kafka for topics, creates them if they don't exist
 func EnsureTopicsExist(topics []string) error {
-	ctx, cancel := context.WithCancel(context.Background())
+	if adminClient == nil {
+		return errors.New("kafka admin client not initialized")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(),
+		time.Duration(envtool.GetEnvInt("KAFKA_TIMEOUT", 30))*time.Second)
 	defer cancel()
 
-	s := fmt.Sprintf("%vs", envtool.GetEnvInt("KAFKA_TIMEOUT", 30))
-	maxDur, err := time.ParseDuration(s)
+	partitions := int32(envtool.GetEnvInt("KAFKA_TOPIC_PARTITIONS", 1))
+	replicationFactor := int16(envtool.GetEnvInt("KAFKA_TOPIC_REPLICATION_FACTOR", 1))
+
+	responses, err := adminClient.CreateTopics(ctx, partitions, replicationFactor, nil, topics...)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
-	ts := make([]kafka.TopicSpecification, len(topics))
-	for i, t := range topics {
-		ts[i] = kafka.TopicSpecification{
-			Topic:             t,
-			NumPartitions:     envtool.GetEnvInt("KAFKA_TOPIC_PARTITIONS", 1),
-			ReplicationFactor: envtool.GetEnvInt("KAFKA_TOPIC_REPLICATION_FACTOR", 1),
+	// Check for errors (topic already exists is not an error)
+	for _, r := range responses {
+		if r.Err != nil && r.Err.Error() != "TOPIC_ALREADY_EXISTS" {
+			logger.Warn().Err(r.Err).Msgf("failed to create topic %s", r.Topic)
 		}
 	}
 
-	_, err = Admin.CreateTopics(ctx, ts, kafka.SetAdminOperationTimeout(maxDur))
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
 	return nil
 }