Merge pull request #253 from Yolean/kustomize

Introduce a Kustomization concept

Author: solsson

README.md

@@ -25,6 +25,22 @@ That'll give you client "bootstrap" `bootstrap.kafka.svc.cluster.local:9092`.
 Our only dependency is `kubectl`. Not because we dislike Helm or Operators, but because we think plain manifests make it easier to collaborate.
 If you begin to rely on this kafka setup we recommend you fork, for example to edit [broker config](https://github.com/Yolean/kubernetes-kafka/blob/master/kafka/10broker-config.yml#L47).
 
+## Kustomize
+
+With the introduction of [app customization](https://kubectl.docs.kubernetes.io/pages/app_customization/introduction.html) in `kubectl` 1.14 there's an alternative to forks. We as a community can maintain a set of overlays.
+
+See the [variants](./variants) folder for different overlays. For example to scale to 1 kafka broker try `kubectl apply -k variants/scale-1/`.
+
+Currently `apply -k` replaces `apply -f ./zookeeper; apply -f ./kafka`.
+The original commands now result in `error: unable to decode "zookeeper/kustomization.yaml": Object 'Kind' is missing in ...`
+and though they still seem to work you can get around that with a v1.14+ kubectl using: `kubectl apply -k variants/as-is/`.
+
+### Maintaining your own kustomization
+
+`kubectl apply -k` takes a siungle overlay, meaning that you can't compose different overlays from this repo.
+You'll probably want to maintain your own variant.
+One option is to keep kubernets-kafka as a git submodule and edit the relative path from an example variant.
+
 ## Version history
 
 | tag    | k8s ≥ | highlights  |

kafka/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+- 10broker-config.yml
+- 20dns.yml
+- 30bootstrap-service.yml
+- 50kafka.yml

variants/as-is/kustomization.yaml

@@ -0,0 +1,3 @@
+bases:
+- ../../kafka
+- ../../zookeeper

variants/scale-1/kafka.yaml

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kafka
+  namespace: kafka
+spec:
+  replicas: 1

variants/scale-1/kustomization.yaml

@@ -0,0 +1,6 @@
+bases:
+- ../../kafka
+- ../../zookeeper
+patchesStrategicMerge:
+- kafka.yaml
+- zookeeper.yaml

variants/scale-1/zookeeper.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pzoo
+  namespace: kafka
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: pzoo
+  namespace: kafka
+spec:
+  replicas: 1
+  template:
+    spec:
+      initContainers:
+      - name: init-config
+        env:
+        - name: PZOO_REPLICAS
+          value: '1'
+        - name: ZOO_REPLICAS
+          value: '0'
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: zoo
+  namespace: kafka
+spec:
+  replicas: 0
+  template:
+    spec:
+      initContainers:
+      - name: init-config
+        env:
+        # There's no validation on these numbers adding up to a coherent zk config, so watch out
+        - name: PZOO_REPLICAS
+          value: '1'
+        - name: ZOO_REPLICAS
+          value: '0'
+        - name: ID_OFFSET
+          value: '2'

variants/scale-2/kafka.yaml

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kafka
+  namespace: kafka
+spec:
+  replicas: 2

variants/scale-2/kustomization.yaml

@@ -0,0 +1,6 @@
+bases:
+- ../../kafka
+- ../../zookeeper
+patchesStrategicMerge:
+- kafka.yaml
+- zookeeper.yaml

variants/scale-2/zookeeper.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pzoo
+  namespace: kafka
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: pzoo
+  namespace: kafka
+spec:
+  replicas: 2
+  template:
+    spec:
+      initContainers:
+      - name: init-config
+        env:
+        - name: PZOO_REPLICAS
+          value: '2'
+        - name: ZOO_REPLICAS
+          value: '1'
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: zoo
+  namespace: kafka
+spec:
+  replicas: 1
+  template:
+    spec:
+      initContainers:
+      - name: init-config
+        env:
+        # There's no validation on these numbers adding up to a coherent zk config, so watch out
+        - name: PZOO_REPLICAS
+          value: '2'
+        - name: ZOO_REPLICAS
+          value: '1'
+        - name: ID_OFFSET
+          value: '3'

zookeeper/10zookeeper-config.yml

@@ -14,9 +14,14 @@ data:
     export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET))
     echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid
     cp -Lur /etc/kafka-configmap/* /etc/kafka/
+    [ ! -z "$PZOO_REPLICAS" ] && [ ! -z "$ZOO_REPLICAS" ] && {
+      sed -i "s/^server\\./#server./" /etc/kafka/zookeeper.properties
+      for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done
+      for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done
+    }
     sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties
 
-  zookeeper.properties: |-
+  zookeeper.properties: |
     tickTime=2000
     dataDir=/var/lib/zookeeper/data
     dataLogDir=/var/lib/zookeeper/log