From 066477a8de89ef55ecaf6bd899a4f6c00dc5c953 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 6 Mar 2020 13:39:49 +0100
Subject: [PATCH 1/2] Makes static (pre-generated) configuration files possible

Experimental, based on the assumption that things like broker.rack
and listener addresses can be set using --overide flags with ev expansion.

If we had the statefulset por-ordinal label we could set those things too,
but now we need to rely on the pod name (+ for zookeeper ID_OFFSET).
---
 kafka/10broker-config.yml        |  1 +
 kafka/50kafka.yml                |  6 +++++-
 zookeeper/10zookeeper-config.yml |  1 +
 zookeeper/50pzoo.yml             | 11 ++++++++++-
 zookeeper/51zoo.yml              | 10 +++++++++-
 5 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml
index 857e1f13..e5e8f0ea 100644
--- a/kafka/10broker-config.yml
+++ b/kafka/10broker-config.yml
@@ -41,6 +41,7 @@ data:
     }
     printf '%s\n' "${SEDS[@]}" | sed -f - /etc/kafka-configmap/server.properties > /etc/kafka/server.properties.tmp
     [ $? -eq 0 ] && mv /etc/kafka/server.properties.tmp /etc/kafka/server.properties
+    ln -s /etc/kafka/server.properties /etc/kafka/server.properties.$POD_NAME
 
   server.properties: |-
     ############################# Log Basics #############################
diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml
index e0b52999..75c9afd5 100644
--- a/kafka/50kafka.yml
+++ b/kafka/50kafka.yml
@@ -47,6 +47,10 @@ spec:
       - name: broker
         image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2
         env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
         - name: CLASSPATH
           value: /opt/kafka/libs/extensions/*
         - name: KAFKA_LOG4J_OPTS
@@ -62,7 +66,7 @@ spec:
           containerPort: 5555
         command:
         - ./bin/kafka-server-start.sh
-        - /etc/kafka/server.properties
+        - /etc/kafka/server.properties.$(POD_NAME)
         lifecycle:
           preStop:
             exec:
diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml
index e2bf8cba..25676d79 100644
--- a/zookeeper/10zookeeper-config.yml
+++ b/zookeeper/10zookeeper-config.yml
@@ -20,6 +20,7 @@ data:
       for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done
     }
     sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties
+    ln -s /etc/kafka/zookeeper.properties /etc/kafka/zookeeper.properties.$POD_NAME
 
   zookeeper.properties: |
     4lw.commands.whitelist=ruok
diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml
index c5f6d1f6..92b03983 100644
--- a/zookeeper/50pzoo.yml
+++ b/zookeeper/50pzoo.yml
@@ -25,6 +25,11 @@ spec:
       - name: init-config
         image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef
         command: ['/bin/bash', '/etc/kafka-configmap/init.sh']
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
         volumeMounts:
         - name: configmap
           mountPath: /etc/kafka-configmap
@@ -36,11 +41,15 @@ spec:
       - name: zookeeper
         image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2
         env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
         command:
         - ./bin/zookeeper-server-start.sh
-        - /etc/kafka/zookeeper.properties
+        - /etc/kafka/zookeeper.properties.$(POD_NAME)
         lifecycle:
           preStop:
             exec:
diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml
index 5aa8ec5e..adcc6e24 100644
--- a/zookeeper/51zoo.yml
+++ b/zookeeper/51zoo.yml
@@ -26,6 +26,10 @@ spec:
         image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef
         command: ['/bin/bash', '/etc/kafka-configmap/init.sh']
         env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
         - name: ID_OFFSET
           value: "4"
         volumeMounts:
@@ -39,11 +43,15 @@ spec:
       - name: zookeeper
         image: solsson/kafka:2.4.0@sha256:201a1c7fd378405b6b1bfd801127c8a530ed7a971282bfcee4ec731bc0c50ad2
         env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
         command:
         - ./bin/zookeeper-server-start.sh
-        - /etc/kafka/zookeeper.properties
+        - /etc/kafka/zookeeper.properties.$(POD_NAME)
         lifecycle:
           preStop:
             exec:

From bc2896a348fe6d2d0b23a7cb33435d7d0679dd20 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 21 Apr 2020 06:57:46 +0200
Subject: [PATCH 2/2] Unlike broker config zookeeper's needs to know the
 replica count

---
 zookeeper/10zookeeper-config.yml | 3 ++-
 zookeeper/50pzoo.yml             | 2 +-
 zookeeper/51zoo.yml              | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml
index 25676d79..3c795c19 100644
--- a/zookeeper/10zookeeper-config.yml
+++ b/zookeeper/10zookeeper-config.yml
@@ -14,13 +14,14 @@ data:
     export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET))
     echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid
     cp -Lur /etc/kafka-configmap/* /etc/kafka/
+    REPLICAS=$(( PZOO_REPLICAS + ZOO_REPLICAS ))
     [ ! -z "$PZOO_REPLICAS" ] && [ ! -z "$ZOO_REPLICAS" ] && {
       sed -i "s/^server\\./#server./" /etc/kafka/zookeeper.properties
       for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done
       for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done
     }
     sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties
-    ln -s /etc/kafka/zookeeper.properties /etc/kafka/zookeeper.properties.$POD_NAME
+    ln -s /etc/kafka/zookeeper.properties /etc/kafka/zookeeper.properties.scale-$REPLICAS.$POD_NAME
 
   zookeeper.properties: |
     4lw.commands.whitelist=ruok
diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml
index 92b03983..c03ff12b 100644
--- a/zookeeper/50pzoo.yml
+++ b/zookeeper/50pzoo.yml
@@ -49,7 +49,7 @@ spec:
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
         command:
         - ./bin/zookeeper-server-start.sh
-        - /etc/kafka/zookeeper.properties.$(POD_NAME)
+        - /etc/kafka/zookeeper.properties.scale-5.$(POD_NAME)
         lifecycle:
           preStop:
             exec:
diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml
index adcc6e24..2d07e7b0 100644
--- a/zookeeper/51zoo.yml
+++ b/zookeeper/51zoo.yml
@@ -51,7 +51,7 @@ spec:
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
         command:
         - ./bin/zookeeper-server-start.sh
-        - /etc/kafka/zookeeper.properties.$(POD_NAME)
+        - /etc/kafka/zookeeper.properties.scale-5.$(POD_NAME)
         lifecycle:
           preStop:
             exec: