inits.cloud

Apache Kafka Message Broker

Sets up Apache Kafka for distributed streaming and message processing

Script Author

Rowan de Haas's avatar
Rowan de Haas
Script Author

Script Details

Created 10 months ago
Size 4 KB

Tags

Script Content

Raw
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#cloud-config
package_update: true
package_upgrade: true

packages:
  - openjdk-11-jdk
  - wget

users:
  - name: kafka
    system: true
    shell: /bin/false
    home: /opt/kafka

write_files:
  - path: /opt/kafka/config/server.properties
    content: |
      broker.id=1
      listeners=PLAINTEXT://localhost:9092
      advertised.listeners=PLAINTEXT://localhost:9092
      num.network.threads=3
      num.io.threads=8
      socket.send.buffer.bytes=102400
      socket.receive.buffer.bytes=102400
      socket.request.max.bytes=104857600
      log.dirs=/opt/kafka/logs
      num.partitions=1
      num.recovery.threads.per.data.dir=1
      offsets.topic.replication.factor=1
      transaction.state.log.replication.factor=1
      transaction.state.log.min.isr=1
      log.retention.hours=168
      log.segment.bytes=1073741824
      log.retention.check.interval.ms=300000
      zookeeper.connect=localhost:2181
      zookeeper.connection.timeout.ms=18000
      group.initial.rebalance.delay.ms=0

  - path: /opt/kafka/config/zookeeper.properties
    content: |
      dataDir=/opt/kafka/zookeeper-data
      clientPort=2181
      maxClientCnxns=0
      admin.enableServer=false
      tickTime=2000
      initLimit=10
      syncLimit=5

  - path: /etc/systemd/system/zookeeper.service
    content: |
      [Unit]
      Description=Apache Zookeeper
      After=network.target

      [Service]
      Type=simple
      User=kafka
      Group=kafka
      ExecStart=/opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties
      ExecStop=/opt/kafka/bin/zookeeper-server-stop.sh
      Restart=on-failure

      [Install]
      WantedBy=multi-user.target

  - path: /etc/systemd/system/kafka.service
    content: |
      [Unit]
      Description=Apache Kafka
      After=zookeeper.service
      Requires=zookeeper.service

      [Service]
      Type=simple
      User=kafka
      Group=kafka
      ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
      ExecStop=/opt/kafka/bin/kafka-server-stop.sh
      Restart=on-failure

      [Install]
      WantedBy=multi-user.target

runcmd:
  # Download and install Kafka
  - cd /opt
  - wget https://downloads.apache.org/kafka/2.13-3.5.0/kafka_2.13-3.5.0.tgz
  - tar -xzf kafka_2.13-3.5.0.tgz
  - mv kafka_2.13-3.5.0 kafka
  - chown -R kafka:kafka /opt/kafka
  - mkdir -p /opt/kafka/logs /opt/kafka/zookeeper-data
  - chown -R kafka:kafka /opt/kafka/logs /opt/kafka/zookeeper-data
  
  # Set JAVA_HOME
  - echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64' >> /etc/environment
  - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
  
  # Enable and start services
  - systemctl enable zookeeper
  - systemctl start zookeeper
  - sleep 10
  - systemctl enable kafka
  - systemctl start kafka
  
  # Configure firewall
  - ufw allow 9092/tcp
  - ufw allow 2181/tcp
  
  # Create test topic
  - sleep 15
  - sudo -u kafka /opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1

How to Use This Script

Cloud Provider Examples

Amazon EC2

aws ec2 run-instances
  --image-id ami-12345678
  --instance-type t3.micro
  --user-data file://script.yaml

DigitalOcean

doctl compute droplet create
  --image ubuntu-22-04-x64
  --size s-1vcpu-1gb
  --user-data-file script.yaml
  my-droplet

Google Cloud

gcloud compute instances create
  my-instance
  --metadata-from-file
  user-data=script.yaml