inits.cloud

Apache Kafka Message Broker

Sets up Apache Kafka for distributed streaming and message processing

Script Author

Rowan de Haas's avatar
Rowan de Haas
Script Author

Script Details

Created 5 months ago
Size 4 KB

Tags

Script Content

Raw
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#cloud-config
package_update: true
package_upgrade: true

packages:
  - openjdk-11-jdk
  - wget

users:
  - name: kafka
    system: true
    shell: /bin/false
    home: /opt/kafka

write_files:
  - path: /opt/kafka/config/server.properties
    content: |
      broker.id=1
      listeners=PLAINTEXT://localhost:9092
      advertised.listeners=PLAINTEXT://localhost:9092
      num.network.threads=3
      num.io.threads=8
      socket.send.buffer.bytes=102400
      socket.receive.buffer.bytes=102400
      socket.request.max.bytes=104857600
      log.dirs=/opt/kafka/logs
      num.partitions=1
      num.recovery.threads.per.data.dir=1
      offsets.topic.replication.factor=1
      transaction.state.log.replication.factor=1
      transaction.state.log.min.isr=1
      log.retention.hours=168
      log.segment.bytes=1073741824
      log.retention.check.interval.ms=300000
      zookeeper.connect=localhost:2181
      zookeeper.connection.timeout.ms=18000
      group.initial.rebalance.delay.ms=0

  - path: /opt/kafka/config/zookeeper.properties
    content: |
      dataDir=/opt/kafka/zookeeper-data
      clientPort=2181
      maxClientCnxns=0
      admin.enableServer=false
      tickTime=2000
      initLimit=10
      syncLimit=5

  - path: /etc/systemd/system/zookeeper.service
    content: |
      [Unit]
      Description=Apache Zookeeper
      After=network.target

      [Service]
      Type=simple
      User=kafka
      Group=kafka
      ExecStart=/opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties
      ExecStop=/opt/kafka/bin/zookeeper-server-stop.sh
      Restart=on-failure

      [Install]
      WantedBy=multi-user.target

  - path: /etc/systemd/system/kafka.service
    content: |
      [Unit]
      Description=Apache Kafka
      After=zookeeper.service
      Requires=zookeeper.service

      [Service]
      Type=simple
      User=kafka
      Group=kafka
      ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
      ExecStop=/opt/kafka/bin/kafka-server-stop.sh
      Restart=on-failure

      [Install]
      WantedBy=multi-user.target

runcmd:
  # Download and install Kafka
  - cd /opt
  - wget https://downloads.apache.org/kafka/2.13-3.5.0/kafka_2.13-3.5.0.tgz
  - tar -xzf kafka_2.13-3.5.0.tgz
  - mv kafka_2.13-3.5.0 kafka
  - chown -R kafka:kafka /opt/kafka
  - mkdir -p /opt/kafka/logs /opt/kafka/zookeeper-data
  - chown -R kafka:kafka /opt/kafka/logs /opt/kafka/zookeeper-data
  
  # Set JAVA_HOME
  - echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64' >> /etc/environment
  - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
  
  # Enable and start services
  - systemctl enable zookeeper
  - systemctl start zookeeper
  - sleep 10
  - systemctl enable kafka
  - systemctl start kafka
  
  # Configure firewall
  - ufw allow 9092/tcp
  - ufw allow 2181/tcp
  
  # Create test topic
  - sleep 15
  - sudo -u kafka /opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1

How to Use This Script

Cloud Provider Examples

Amazon EC2

aws ec2 run-instances
  --image-id ami-12345678
  --instance-type t3.micro
  --user-data file://script.yaml

DigitalOcean

doctl compute droplet create
  --image ubuntu-22-04-x64
  --size s-1vcpu-1gb
  --user-data-file script.yaml
  my-droplet

Google Cloud

gcloud compute instances create
  my-instance
  --metadata-from-file
  user-data=script.yaml