diff --git a/README.md b/README.md
index 11f36e4..77d30c1 100644
--- a/README.md
+++ b/README.md
@@ -111,11 +111,15 @@ module "kafka" {
name = "app"
vpc_id = "vpc-XXXXXXXX"
zone_id = "Z14EN2YD427LRQ"
- security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
subnet_ids = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"]
kafka_version = "2.4.1"
number_of_broker_nodes = 2 # this has to be a multiple of the # of subnet_ids
broker_instance_type = "kafka.m5.large"
+
+ # security groups to put on the cluster itself
+ broker_node_security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
+ # security groups to give access to the cluster
+ security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
}
```
@@ -146,7 +150,7 @@ Available targets:
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | >= 0.13.0 |
+| [terraform](#requirement\_terraform) | >= 0.14.0 |
| [aws](#requirement\_aws) | >= 3.0 |
| [random](#requirement\_random) | >= 2.2 |
@@ -160,6 +164,7 @@ Available targets:
| Name | Source | Version |
|------|--------|---------|
+| [broker\_security\_group](#module\_broker\_security\_group) | cloudposse/security-group/aws | 0.4.2 |
| [hostname](#module\_hostname) | cloudposse/route53-cluster-hostname/aws | 0.12.2 |
| [this](#module\_this) | cloudposse/label/null | 0.25.0 |
@@ -172,10 +177,6 @@ Available targets:
| [aws_msk_cluster.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster) | resource |
| [aws_msk_configuration.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration) | resource |
| [aws_msk_scram_secret_association.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_scram_secret_association) | resource |
-| [aws_security_group.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
-| [aws_security_group_rule.egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_cidr_blocks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_security_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
## Inputs
@@ -185,6 +186,7 @@ Available targets:
| [allowed\_cidr\_blocks](#input\_allowed\_cidr\_blocks) | List of CIDR blocks to be allowed to connect to the cluster | `list(string)` | `[]` | no |
| [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no |
| [broker\_instance\_type](#input\_broker\_instance\_type) | The instance type to use for the Kafka brokers | `string` | n/a | yes |
+| [broker\_node\_security\_groups](#input\_broker\_node\_security\_groups) | List of broker node security group IDs to be associated with the elastic network interfaces to control who can communicate with the cluster | `list(string)` | `[]` | no |
| [broker\_volume\_size](#input\_broker\_volume\_size) | The size in GiB of the EBS volume for the data drive on each broker node | `number` | `1000` | no |
| [certificate\_authority\_arns](#input\_certificate\_authority\_arns) | List of ACM Certificate Authority Amazon Resource Names (ARNs) to be used for TLS client authentication | `list(string)` | `[]` | no |
| [client\_broker](#input\_client\_broker) | Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT` | `string` | `"TLS"` | no |
diff --git a/README.yaml b/README.yaml
index baec01b..dff461b 100644
--- a/README.yaml
+++ b/README.yaml
@@ -86,11 +86,15 @@ usage: |-
name = "app"
vpc_id = "vpc-XXXXXXXX"
zone_id = "Z14EN2YD427LRQ"
- security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
subnet_ids = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"]
kafka_version = "2.4.1"
number_of_broker_nodes = 2 # this has to be a multiple of the # of subnet_ids
broker_instance_type = "kafka.m5.large"
+
+ # security groups to put on the cluster itself
+ broker_node_security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
+ # security groups to give access to the cluster
+ security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
}
```
diff --git a/docs/terraform.md b/docs/terraform.md
index 5c1f538..89724d6 100644
--- a/docs/terraform.md
+++ b/docs/terraform.md
@@ -3,7 +3,7 @@
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | >= 0.13.0 |
+| [terraform](#requirement\_terraform) | >= 0.14.0 |
| [aws](#requirement\_aws) | >= 3.0 |
| [random](#requirement\_random) | >= 2.2 |
@@ -17,6 +17,7 @@
| Name | Source | Version |
|------|--------|---------|
+| [broker\_security\_group](#module\_broker\_security\_group) | cloudposse/security-group/aws | 0.4.2 |
| [hostname](#module\_hostname) | cloudposse/route53-cluster-hostname/aws | 0.12.2 |
| [this](#module\_this) | cloudposse/label/null | 0.25.0 |
@@ -29,10 +30,6 @@
| [aws_msk_cluster.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster) | resource |
| [aws_msk_configuration.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration) | resource |
| [aws_msk_scram_secret_association.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_scram_secret_association) | resource |
-| [aws_security_group.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
-| [aws_security_group_rule.egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_cidr_blocks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_security_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
## Inputs
@@ -42,6 +39,7 @@
| [allowed\_cidr\_blocks](#input\_allowed\_cidr\_blocks) | List of CIDR blocks to be allowed to connect to the cluster | `list(string)` | `[]` | no |
| [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,
in the order they appear in the list. New attributes are appended to the
end of the list. The elements of the list are joined by the `delimiter`
and treated as a single ID element. | `list(string)` | `[]` | no |
| [broker\_instance\_type](#input\_broker\_instance\_type) | The instance type to use for the Kafka brokers | `string` | n/a | yes |
+| [broker\_node\_security\_groups](#input\_broker\_node\_security\_groups) | List of broker node security group IDs to be associated with the elastic network interfaces to control who can communicate with the cluster | `list(string)` | `[]` | no |
| [broker\_volume\_size](#input\_broker\_volume\_size) | The size in GiB of the EBS volume for the data drive on each broker node | `number` | `1000` | no |
| [certificate\_authority\_arns](#input\_certificate\_authority\_arns) | List of ACM Certificate Authority Amazon Resource Names (ARNs) to be used for TLS client authentication | `list(string)` | `[]` | no |
| [client\_broker](#input\_client\_broker) | Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT` | `string` | `"TLS"` | no |
diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf
index 1d387de..c31c403 100644
--- a/examples/complete/versions.tf
+++ b/examples/complete/versions.tf
@@ -1,21 +1,14 @@
terraform {
- required_version = ">= 0.13.0"
+ required_version = ">= 0.14.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 3.0"
}
- template = {
- source = "hashicorp/template"
- version = ">= 2.0"
- }
- local = {
- source = "hashicorp/local"
- version = ">= 1.3"
- }
random = {
- source = "hashicorp/random"
+ source = "hashicorp/random"
+ version = ">= 2.2"
}
}
}
diff --git a/main.tf b/main.tf
index af26169..c519c46 100644
--- a/main.tf
+++ b/main.tf
@@ -12,47 +12,81 @@ locals {
bootstrap_brokers_combined_list = concat(local.bootstrap_brokers_list, local.bootstrap_brokers_tls_list, local.bootstrap_brokers_scram_list, local.bootstrap_brokers_iam_list)
# If var.storage_autoscaling_max_capacity is not set, don't autoscale past current size
broker_volume_size_max = coalesce(var.storage_autoscaling_max_capacity, var.broker_volume_size)
-}
-
-resource "aws_security_group" "default" {
- count = local.enabled ? 1 : 0
- vpc_id = var.vpc_id
- name = module.this.id
- description = "Allow inbound traffic from Security Groups and CIDRs. Allow all outbound traffic"
- tags = module.this.tags
-}
-resource "aws_security_group_rule" "ingress_security_groups" {
- count = local.enabled ? length(var.security_groups) : 0
- description = "Allow inbound traffic from Security Groups"
- type = "ingress"
- from_port = 0
- to_port = 65535
- protocol = "tcp"
- source_security_group_id = var.security_groups[count.index]
- security_group_id = join("", aws_security_group.default.*.id)
+ # var.client_broker types
+ plaintext = "PLAINTEXT"
+ tls_plaintext = "TLS_PLAINTEXT"
+ tls = "TLS"
+
+ # The following ports are not configurable. See: https://docs.aws.amazon.com/msk/latest/developerguide/client-access.html#port-info
+ protocols = {
+ plaintext = {
+ name = "plaintext"
+ # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers
+ enabled = contains([local.plaintext, local.tls_plaintext], var.client_broker)
+ port = 9092
+ }
+ tls = {
+ name = "TLS"
+ # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers_tls
+ enabled = contains([local.tls_plaintext, local.tls], var.client_broker)
+ port = 9094
+ }
+ sasl_scram = {
+ name = "SASL/SCRAM"
+ # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers_sasl_scram
+ enabled = var.client_sasl_scram_enabled && contains([local.tls_plaintext, local.tls], var.client_broker)
+ port = 9096
+ }
+ sasl_iam = {
+ name = "SASL/IAM"
+ # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers_sasl_iam
+ enabled = var.client_sasl_iam_enabled && contains([local.tls_plaintext, local.tls], var.client_broker)
+ port = 9098
+ }
+ # The following two protocols are always enabled.
+ # See: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#zookeeper_connect_string
+ # and https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#zookeeper_connect_string_tls
+ zookeeper_plaintext = {
+ name = "Zookeeper plaintext"
+ enabled = true
+ port = 2181
+ }
+ zookeeper_tls = {
+ name = "Zookeeper TLS"
+ enabled = true
+ port = 2182
+ }
+ }
}
-resource "aws_security_group_rule" "ingress_cidr_blocks" {
- count = local.enabled && length(var.allowed_cidr_blocks) > 0 ? 1 : 0
- description = "Allow inbound traffic from CIDR blocks"
- type = "ingress"
- from_port = 0
- to_port = 65535
- protocol = "tcp"
- cidr_blocks = var.allowed_cidr_blocks
- security_group_id = join("", aws_security_group.default.*.id)
-}
+module "broker_security_group" {
+ source = "cloudposse/security-group/aws"
+ version = "0.4.2"
+
+ attributes = ["broker"]
+
+ security_group_description = "Allow inbound MSK-related traffic from Security Groups and CIDRs. Allow all outbound traffic"
+ allow_all_egress = true
+ rule_matrix = [
+ {
+ source_security_group_ids = var.security_groups
+ cidr_blocks = var.allowed_cidr_blocks
+ rules = [
+ for protocol_key, protocol in local.protocols : {
+ key = protocol_key
+ type = "ingress"
+ from_port = protocol.port
+ to_port = protocol.port
+ protocol = "tcp"
+ description = "Allow inbound ${protocol.name} traffic"
+ } if protocol.enabled
+ ]
+ }
+ ]
+ vpc_id = var.vpc_id
-resource "aws_security_group_rule" "egress" {
- count = local.enabled ? 1 : 0
- description = "Allow all egress traffic"
- type = "egress"
- from_port = 0
- to_port = 65535
- protocol = "tcp"
- cidr_blocks = ["0.0.0.0/0"]
- security_group_id = join("", aws_security_group.default.*.id)
+ context = module.this.context
}
resource "aws_msk_configuration" "config" {
@@ -65,6 +99,7 @@ resource "aws_msk_configuration" "config" {
}
resource "aws_msk_cluster" "default" {
+ #bridgecrew:skip=BC_AWS_LOGGING_18:Skipping `Amazon MSK cluster logging is not enabled` check since it can be enabled with cloudwatch_logs_enabled = true
count = local.enabled ? 1 : 0
cluster_name = module.this.id
kafka_version = var.kafka_version
@@ -75,7 +110,7 @@ resource "aws_msk_cluster" "default" {
instance_type = var.broker_instance_type
ebs_volume_size = var.broker_volume_size
client_subnets = var.subnet_ids
- security_groups = aws_security_group.default.*.id
+ security_groups = concat(var.broker_node_security_groups, [module.broker_security_group.id])
}
configuration_info {
diff --git a/outputs.tf b/outputs.tf
index 7cd21d6..af3113d 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -55,10 +55,10 @@ output "cluster_name" {
output "security_group_id" {
description = "The ID of the security group rule"
- value = join("", aws_security_group.default.*.id)
+ value = module.broker_security_group.id
}
output "security_group_name" {
description = "The name of the security group rule"
- value = join("", aws_security_group.default.*.name)
+ value = module.broker_security_group.name
}
diff --git a/variables.tf b/variables.tf
index de2eb75..2c90e10 100644
--- a/variables.tf
+++ b/variables.tf
@@ -41,6 +41,12 @@ variable "security_groups" {
description = "List of security group IDs to be allowed to connect to the cluster"
}
+variable "broker_node_security_groups" {
+ type = list(string)
+ default = []
+ description = "List of broker node security group IDs to be associated with the elastic network interfaces to control who can communicate with the cluster"
+}
+
variable "allowed_cidr_blocks" {
type = list(string)
default = []
diff --git a/versions.tf b/versions.tf
index 763f80f..c31c403 100644
--- a/versions.tf
+++ b/versions.tf
@@ -1,5 +1,5 @@
terraform {
- required_version = ">= 0.13.0"
+ required_version = ">= 0.14.0"
required_providers {
aws = {