Very basic publishing of Message to Apache Kafka Topic Fails

[Tobi696]

Here's my code:

package main

import (
	"log"

	"github.com/Shopify/sarama"
	"github.com/ThreeDotsLabs/watermill"
	"github.com/ThreeDotsLabs/watermill-kafka/v2/pkg/kafka"
	"github.com/ThreeDotsLabs/watermill/message"
)

func main() {
	saramaSubscriberConfig := kafka.DefaultSaramaSubscriberConfig()
	// equivalent of auto.offset.reset: earliest
	saramaSubscriberConfig.Consumer.Offsets.Initial = sarama.OffsetOldest

	publisher, err := kafka.NewPublisher(
		kafka.PublisherConfig{
			Brokers:   []string{"localhost:9092"},
			Marshaler: kafka.DefaultMarshaler{},
		},
		watermill.NewStdLogger(false, false),
	)
	if err != nil {
		log.Fatal(err)
	}

	if err := publisher.Publish("example.topic", message.NewMessage(watermill.NewUUID(), []byte("example message"))); err != nil {
		log.Fatal(err)
	}
}

When I execute it, I get following error: cannot produce message f5bbade7-01f9-4b8c-8763-6c4feefe9dd8: dial tcp: lookup b394238ea53a: no such host

I am using following docker-compose.yml for my Kafka Server:

version: "2"

services:
  kafka:
    image: docker.io/bitnami/kafka:3.4
    ports:
      - "9092:9092"
    environment:
      - ALLOW_PLAINTEXT_LISTENER=yes

[Tobi696]

I tried the same with RabbitMQ, it works there.

[luabagg]

Hi. This seems to be an error with your Kafka setup. You can see a complete example here, but it uses the Confluent Docker image.

Try adding the following configuration to your Docker Compose:

    ports:
      - "9092:9092"
      - "9094:9094"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_LISTENERS: INTERNAL://:9092,OUTSIDE://:9094
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://localhost:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT

And the broker as "localhost:9094" or "kafka:9092" if running in the same container. You need to add the Zookeeper service as well.