feat: add filebeat support

Fixes #4.

Author: Jason Schmidt

pulumi/aws/README.md

@@ -84,7 +84,9 @@ vpc - defines and installs the VPC and subnets to use with EKS
     └─kic-image-build - project that builds a new KIC image  
       └─kic-image-push - pushes KIC image built in previous step to ECR
         └─kic-helm-chart - deploys NGINX Ingress Controller to the EKS cluster 
-          └─demo-app - deploys a sample application to the cluster
+          └─logstore - deploys a logstore (elasticsearch) to the EKS cluster 
+            └─logagent - deploys a logging agent (filebeat) to the EKS cluster 
+              └─demo-app - deploys a sample application to the cluster
 ```
 
 ### VPC
@@ -129,8 +131,65 @@ Ingress Controller on the previously deployed EKS cluster. You may want
 to customize this project to allow for deploying different versions of
 KIC.
 
+A sample config-map is provided in the Pulumi deployment code; this 
+code will adjust the logging format to approximate the upstream 
+NGINX KIC project which will allow for easier injestion into log
+storage and processing systems. 
+
+### Log Store
+
+In the [`logstore`](./logstore) directory, you will find the Pulumi
+project reponsible for installing your log store. The current
+solution deploys 
+[Elasticsearch and Kibana](https://www.elastic.co/elastic-stack) 
+using the 
+[Bitnami Elasticsearch](https://bitnami.com/stack/elasticsearch/helm)
+chart. This solution can be swapped for other options as desired.
+This application is deployed to the `logstore` namespace. 
+
+#### Notes
+In order to access the Kibana dashboard via your web browser, you will
+need to setup port forwarding for the kibana pod. This can be accomplished
+using the `kubectl` command:
+
+```
+$ # Find the Kibana pod name
+$ kubectl get pods -n logstore
+NAME                                            READY   STATUS    RESTARTS   AGE
+elastic-coordinating-only-b76674c4c-d58rh       1/1     Running   0          61m
+elastic-coordinating-only-b76674c4c-sb6v7       1/1     Running   0          61m
+elastic-elasticsearch-data-0                    1/1     Running   0          61m
+elastic-elasticsearch-data-1                    1/1     Running   0          61m
+elastic-elasticsearch-ingest-589d4ddf4b-6djjz   1/1     Running   0          61m
+elastic-elasticsearch-ingest-589d4ddf4b-6mzmb   1/1     Running   0          61m
+elastic-elasticsearch-master-0                  1/1     Running   0          61m
+elastic-elasticsearch-master-1                  1/1     Running   0          61m
+elastic-kibana-d45db8647-ghhx2                  1/1     Running   0          61m
+$ # Setup the port forward
+$ kubectl port-forward elastic-kibana-d45db8647-ghhx2 5601:5601 -n logstore
+Forwarding from 127.0.0.1:5601 -> 5601
+Forwarding from [::1]:5601 -> 5601
+Handling connection for 5601
+````
+
+Additionally, you will need to load the saved object data for Kibana from 
+the provided [`kibana-data.ndjson`](./extras/kibana/kibana-data.ndjson) 
+which can be found in the `./extras/kibana` directory in this project. To
+accomplish this, go to "Stack Management -> Saved Objects" in the Kibana
+interface. 
+
+### Log Agent
+
+In the [`logagent`](./logagent) directory, you will find the Pulumi
+project reponsible for installing your log agent. The current solution
+deploys [`Filebeat`](https://www.elastic.co/beats/) which connects
+to the logstore deployed in the previous step. This solution can be
+swapped for other options as desired. This application is
+deployed to the `logagent` namespace.
+
+
 ### Demo Application
 
 A simple sample application is contained in the [`demo-app`](./demo-app)
 directory. This project shows off how one may deploy their application
-to a cluster that is using KIC.
+to a cluster that is using KIC.

pulumi/aws/destroy.sh

@@ -9,6 +9,12 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "${script_dir}/demo-app"
 pulumi --emoji destroy --yes
 
+cd "${script_dir}/logagent"
+pulumi --emoji destroy --yes
+
+cd "${script_dir}/logstore"
+pulumi --emoji destroy --yes
+
 cd "${script_dir}/kic-helm-chart"
 pulumi --emoji destroy --yes
 
@@ -25,4 +31,4 @@ cd "${script_dir}/eks"
 pulumi --emoji destroy --yes
 
 cd "${script_dir}/vpc"
-pulumi --emoji destroy --yes
+pulumi --emoji destroy --yes

pulumi/aws/extras/kibana/kibana-data.ndjson

@@ -27,9 +27,32 @@ def project_name_from_project_dir(dirname: str):
 
 def build_chart_values(repository: dict) -> helm.ChartOpts:
     values: Dict[str, Dict[str, typing.Any]] = {
-        'controller': {
-            'healthStatus': True
-        }
+      'controller': {
+          'healthStatus': True,
+          'appprotect': {
+              'enable': False
+          },
+          'config': {
+              'name': 'nginx-config',
+              'entries': {
+                  'log-format': '$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" $upstream_response_time $upstream_status \"$uri\" $request_length $request_time [$proxy_host] [] $upstream_addr $upstream_bytes_sent $upstream_response_time $upstream_status $request_id'
+              }
+          },
+          'service': {
+              'annotations': {
+                  'co.elastic.logs/module': 'nginx'
+              }
+          },
+          'pod': {
+              'annotations': {
+                  'co.elastic.logs/module': 'nginx'
+              }
+          }
+      },
+      'prometheus': {
+         'create': True,
+         'port': 9113
+       }
     }
 
     if 'repository_url' in repository and 'image_tag_alias' in repository:

pulumi/aws/kic-helm-chart/__main__.py

@@ -1,7 +1,7 @@
-name: filebeat
+name: logagent
 runtime:
   name: python
   options:
     virtualenv: ../venv
 config: config
-description: Deploys filebeat daemonset
+description: Deploys a log agent

pulumi/aws/filebeat/Pulumi.yaml renamed to pulumi/aws/logagent/Pulumi.yaml

@@ -38,23 +38,22 @@ def project_name_from_project_dir(dirname: str):
 k8s_provider = k8s.Provider(resource_name=f'ingress-setup-sample',
                             kubeconfig=kubeconfig)
 
-#ns = k8s.core.v1.Namespace(resource_name='elastic',
-#                           metadata={'name': 'elastic'},
-#                           opts=pulumi.ResourceOptions(provider=k8s_provider))
+ns = k8s.core.v1.Namespace(resource_name='logagent',
+                           metadata={'name': 'logagent'},
+                           opts=pulumi.ResourceOptions(provider=k8s_provider))
 
 chart_values = {
     "daemonset": {
         "enabled": True,
         "filebeatConfig": {
-            "filebeat.yml": "filebeat.autodiscover:\n  providers:\n    - type: kubernetes\n      hints.enabled: true\n      hints.default_config:\n        type: container\n        paths:\n          - /var/lib/docker/containers/${data.kubernetes.container.id}/*.log\noutput.elasticsearch:\n  host: '${NODE_NAME}'\n  hosts: 'elasticsearch-coordinating-only.elastic.svc.cluster.local:9200'\n"
+            "filebeat.yml": "filebeat.autodiscover:\n  providers:\n    - type: kubernetes\n      hints.enabled: true\n      hints.default_config:\n        type: container\n        paths:\n          - /var/lib/docker/containers/${data.kubernetes.container.id}/*.log\noutput.elasticsearch:\n  host: '${NODE_NAME}'\n  hosts: 'elastic-coordinating-only.logstore.svc.cluster.local:9200'\n"
         }
      }
   }
 
 chart_ops = helm.ChartOpts(
         chart='filebeat',
-#        namespace=ns.metadata.name,
-        namespace='elastic',
+        namespace=ns.metadata.name,
         repo=FILEBEAT_HELM_REPO_NAME,
         fetch_opts=FetchOpts(repo=FILEBEAT_HELM_REPO_URL),
         version='7.13.2',

pulumi/aws/filebeat/__main__.py renamed to pulumi/aws/logagent/__main__.py

@@ -1,7 +1,7 @@
-name: elastic
+name: logstore
 runtime:
   name: python
   options:
     virtualenv: ../venv
 config: config
-description: Sets up an Elasticsearch Cluster
+description: Deploys a logging store

pulumi/aws/filebeat/config/.gitignore renamed to pulumi/aws/logagent/config/.gitignore

@@ -38,8 +38,8 @@ def project_name_from_project_dir(dirname: str):
 k8s_provider = k8s.Provider(resource_name=f'ingress-setup-sample',
                             kubeconfig=kubeconfig)
 
-ns = k8s.core.v1.Namespace(resource_name='elastic',
-                           metadata={'name': 'elastic'},
+ns = k8s.core.v1.Namespace(resource_name='logstore',
+                           metadata={'name': 'logstore'},
                            opts=pulumi.ResourceOptions(provider=k8s_provider))
 
 chart_values = {

pulumi/aws/elastic/Pulumi.yaml renamed to pulumi/aws/logstore/Pulumi.yaml

@@ -0,0 +1,3 @@
+config:
+  aws:profile: f5
+  aws:region: us-west-2