From ad8fe0ccfc0824198d24c7f5fac251229eb14c43 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 14 Jan 2021 14:57:29 -0800 Subject: [PATCH 1/2] Support private networking on GCP --- cli/cmd/cluster_gcp.go | 5 ++ docs/clusters/gcp/install.md | 11 +++ manager/install.sh | 12 +-- manager/manifests/istio.yaml.j2 | 6 ++ pkg/types/clusterconfig/cluster_config_gcp.go | 75 ++++++++++++++----- pkg/types/clusterconfig/config_key.go | 2 + 6 files changed, 88 insertions(+), 23 deletions(-) diff --git a/cli/cmd/cluster_gcp.go b/cli/cmd/cluster_gcp.go index 93e0ad632b..9d1d66dc98 100644 --- a/cli/cmd/cluster_gcp.go +++ b/cli/cmd/cluster_gcp.go @@ -485,6 +485,11 @@ func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Cli if clusterConfig.Subnet != nil { gkeClusterConfig.Subnetwork = *clusterConfig.Subnet } + if clusterConfig.NodeVisibility == clusterconfig.PrivateSubnetVisibility { + gkeClusterConfig.PrivateClusterConfig = &containerpb.PrivateClusterConfig{ + EnablePrivateNodes: true, + } + } _, err := gcpClient.CreateCluster(&containerpb.CreateClusterRequest{ Parent: gkeClusterParent, diff --git a/docs/clusters/gcp/install.md b/docs/clusters/gcp/install.md index 6bcfed8153..aaf329cb99 100644 --- a/docs/clusters/gcp/install.md +++ b/docs/clusters/gcp/install.md @@ -45,6 +45,17 @@ max_instances: 5 # the name of the subnetwork in which to create your cluster # subnet: default + +# API load balancer scheme [internet-facing | internal] +api_load_balancer_scheme: internet-facing + +# operator load balancer scheme [internet-facing | internal] +# note: if using "internal", you must be within the cluster's VPC or configure VPC Peering to connect your CLI to your cluster operator +operator_load_balancer_scheme: internet-facing + +# node visibility [public (nodes will have public IPs) | private (nodes will not have public IPs)] +# note: if using "private", you will need to configure Cloud NAT in your VPC before creating your cluster +node_visibility: public ``` The docker images used by the Cortex cluster can also be overridden, although this is not common. They can be configured by adding any of these keys to your cluster configuration file (default values are shown): diff --git a/manager/install.sh b/manager/install.sh index e1629a3d0e..396f8071fc 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -706,12 +706,14 @@ function validate_cortex_gcp() { api_load_balancer_endpoint=$(kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\"ip\":\"\(.*\)\".*/\1/') fi - operator_endpoint_reachable="false" # don't cache this result - if ! curl --max-time 3 "${operator_endpoint}/verifycortex" >/dev/null 2>&1; then - success_cycles=0 - continue + if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then + operator_endpoint_reachable="false" # don't cache this result + if ! curl --max-time 3 "${operator_endpoint}/verifycortex" >/dev/null 2>&1; then + success_cycles=0 + continue + fi + operator_endpoint_reachable="true" fi - operator_endpoint_reachable="true" if [[ $success_cycles -lt 1 ]]; then ((success_cycles++)) diff --git a/manager/manifests/istio.yaml.j2 b/manager/manifests/istio.yaml.j2 index 50c8cad09a..ee6f84ead9 100644 --- a/manager/manifests/istio.yaml.j2 +++ b/manager/manifests/istio.yaml.j2 @@ -47,6 +47,9 @@ spec: {% if config.get('operator_load_balancer_scheme') == 'internal' %} service.beta.kubernetes.io/aws-load-balancer-internal: "true" {% endif %} + {% elif env['CORTEX_PROVIDER'] == "gcp" and config.get('operator_load_balancer_scheme') == 'internal' %} + serviceAnnotations: + cloud.google.com/load-balancer-type: "Internal" {% endif %} service: type: LoadBalancer @@ -106,6 +109,9 @@ spec: {% if config.get('ssl_certificate_arn', '') != '' %} service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "{{ config['ssl_certificate_arn'] }}" {% endif %} + {% elif env['CORTEX_PROVIDER'] == "gcp" and config.get('api_load_balancer_scheme') == 'internal' %} + serviceAnnotations: + cloud.google.com/load-balancer-type: "Internal" {% endif %} service: type: LoadBalancer diff --git a/pkg/types/clusterconfig/cluster_config_gcp.go b/pkg/types/clusterconfig/cluster_config_gcp.go index 63cbb26a27..c72c388661 100644 --- a/pkg/types/clusterconfig/cluster_config_gcp.go +++ b/pkg/types/clusterconfig/cluster_config_gcp.go @@ -33,24 +33,27 @@ import ( ) type GCPConfig struct { - Provider types.ProviderType `json:"provider" yaml:"provider"` - Project *string `json:"project" yaml:"project"` - Zone *string `json:"zone" yaml:"zone"` - InstanceType *string `json:"instance_type" yaml:"instance_type"` - AcceleratorType *string `json:"accelerator_type" yaml:"accelerator_type"` - Network *string `json:"network" yaml:"network"` - Subnet *string `json:"subnet" yaml:"subnet"` - MinInstances *int64 `json:"min_instances" yaml:"min_instances"` - MaxInstances *int64 `json:"max_instances" yaml:"max_instances"` - ClusterName string `json:"cluster_name" yaml:"cluster_name"` - Telemetry bool `json:"telemetry" yaml:"telemetry"` - ImageOperator string `json:"image_operator" yaml:"image_operator"` - ImageManager string `json:"image_manager" yaml:"image_manager"` - ImageDownloader string `json:"image_downloader" yaml:"image_downloader"` - ImageFluentBit string `json:"image_fluent_bit" yaml:"image_fluent_bit"` - ImageIstioProxy string `json:"image_istio_proxy" yaml:"image_istio_proxy"` - ImageIstioPilot string `json:"image_istio_pilot" yaml:"image_istio_pilot"` - ImageGooglePause string `json:"image_google_pause" yaml:"image_google_pause"` + Provider types.ProviderType `json:"provider" yaml:"provider"` + Project *string `json:"project" yaml:"project"` + Zone *string `json:"zone" yaml:"zone"` + InstanceType *string `json:"instance_type" yaml:"instance_type"` + AcceleratorType *string `json:"accelerator_type" yaml:"accelerator_type"` + Network *string `json:"network" yaml:"network"` + Subnet *string `json:"subnet" yaml:"subnet"` + NodeVisibility SubnetVisibility `json:"node_visibility" yaml:"node_visibility"` + APILoadBalancerScheme LoadBalancerScheme `json:"api_load_balancer_scheme" yaml:"api_load_balancer_scheme"` + OperatorLoadBalancerScheme LoadBalancerScheme `json:"operator_load_balancer_scheme" yaml:"operator_load_balancer_scheme"` + MinInstances *int64 `json:"min_instances" yaml:"min_instances"` + MaxInstances *int64 `json:"max_instances" yaml:"max_instances"` + ClusterName string `json:"cluster_name" yaml:"cluster_name"` + Telemetry bool `json:"telemetry" yaml:"telemetry"` + ImageOperator string `json:"image_operator" yaml:"image_operator"` + ImageManager string `json:"image_manager" yaml:"image_manager"` + ImageDownloader string `json:"image_downloader" yaml:"image_downloader"` + ImageFluentBit string `json:"image_fluent_bit" yaml:"image_fluent_bit"` + ImageIstioProxy string `json:"image_istio_proxy" yaml:"image_istio_proxy"` + ImageIstioPilot string `json:"image_istio_pilot" yaml:"image_istio_pilot"` + ImageGooglePause string `json:"image_google_pause" yaml:"image_google_pause"` } type InternalGCPConfig struct { @@ -148,6 +151,36 @@ var UserGCPValidation = &cr.StructValidation{ AllowExplicitNull: true, }, }, + { + StructField: "NodeVisibility", + StringValidation: &cr.StringValidation{ + AllowedValues: SubnetVisibilityStrings(), + Default: PublicSubnetVisibility.String(), + }, + Parser: func(str string) (interface{}, error) { + return SubnetVisibilityFromString(str), nil + }, + }, + { + StructField: "APILoadBalancerScheme", + StringValidation: &cr.StringValidation{ + AllowedValues: LoadBalancerSchemeStrings(), + Default: InternetFacingLoadBalancerScheme.String(), + }, + Parser: func(str string) (interface{}, error) { + return LoadBalancerSchemeFromString(str), nil + }, + }, + { + StructField: "OperatorLoadBalancerScheme", + StringValidation: &cr.StringValidation{ + AllowedValues: LoadBalancerSchemeStrings(), + Default: InternetFacingLoadBalancerScheme.String(), + }, + Parser: func(str string) (interface{}, error) { + return LoadBalancerSchemeFromString(str), nil + }, + }, { StructField: "MinInstances", Int64PtrValidation: &cr.Int64PtrValidation{ @@ -501,6 +534,9 @@ func (cc *GCPConfig) UserTable() table.KeyValuePairs { if cc.Subnet != nil { items.Add(SubnetUserKey, *cc.Subnet) } + items.Add(NodeVisibilityUserKey, cc.NodeVisibility) + items.Add(APILoadBalancerSchemeUserKey, cc.APILoadBalancerScheme) + items.Add(OperatorLoadBalancerSchemeUserKey, cc.OperatorLoadBalancerScheme) items.Add(TelemetryUserKey, cc.Telemetry) items.Add(ImageOperatorUserKey, cc.ImageOperator) items.Add(ImageManagerUserKey, cc.ImageManager) @@ -536,6 +572,9 @@ func (cc *GCPConfig) TelemetryEvent() map[string]interface{} { if cc.Subnet != nil { event["subnet._is_defined"] = true } + event["node_visibility"] = cc.NodeVisibility + event["api_load_balancer_scheme"] = cc.APILoadBalancerScheme + event["operator_load_balancer_scheme"] = cc.OperatorLoadBalancerScheme if cc.MinInstances != nil { event["min_instances._is_defined"] = true event["min_instances"] = *cc.MinInstances diff --git a/pkg/types/clusterconfig/config_key.go b/pkg/types/clusterconfig/config_key.go index 4cf8f5de74..b4057bc576 100644 --- a/pkg/types/clusterconfig/config_key.go +++ b/pkg/types/clusterconfig/config_key.go @@ -47,6 +47,7 @@ const ( SSLCertificateARNKey = "ssl_certificate_arn" BucketKey = "bucket" SubnetVisibilityKey = "subnet_visibility" + NodeVisibilityKey = "node_visibility" NATGatewayKey = "nat_gateway" APILoadBalancerSchemeKey = "api_load_balancer_scheme" OperatorLoadBalancerSchemeKey = "operator_load_balancer_scheme" @@ -98,6 +99,7 @@ const ( InstancePoolsUserKey = "spot instance pools" OnDemandBackupUserKey = "on demand backup" SubnetVisibilityUserKey = "subnet visibility" + NodeVisibilityUserKey = "node visibility" NATGatewayUserKey = "nat gateway" APILoadBalancerSchemeUserKey = "api load balancer scheme" OperatorLoadBalancerSchemeUserKey = "operator load balancer scheme" From 5835f95e38a23f4f8251002d2a348ed28704ba71 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Thu, 14 Jan 2021 15:11:41 -0800 Subject: [PATCH 2/2] Remove node visibility configuration --- cli/cmd/cluster_gcp.go | 5 ----- docs/clusters/gcp/install.md | 4 ---- pkg/types/clusterconfig/cluster_config_gcp.go | 13 ------------- pkg/types/clusterconfig/config_key.go | 2 -- 4 files changed, 24 deletions(-) diff --git a/cli/cmd/cluster_gcp.go b/cli/cmd/cluster_gcp.go index 9d1d66dc98..93e0ad632b 100644 --- a/cli/cmd/cluster_gcp.go +++ b/cli/cmd/cluster_gcp.go @@ -485,11 +485,6 @@ func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Cli if clusterConfig.Subnet != nil { gkeClusterConfig.Subnetwork = *clusterConfig.Subnet } - if clusterConfig.NodeVisibility == clusterconfig.PrivateSubnetVisibility { - gkeClusterConfig.PrivateClusterConfig = &containerpb.PrivateClusterConfig{ - EnablePrivateNodes: true, - } - } _, err := gcpClient.CreateCluster(&containerpb.CreateClusterRequest{ Parent: gkeClusterParent, diff --git a/docs/clusters/gcp/install.md b/docs/clusters/gcp/install.md index aaf329cb99..26c85897e7 100644 --- a/docs/clusters/gcp/install.md +++ b/docs/clusters/gcp/install.md @@ -52,10 +52,6 @@ api_load_balancer_scheme: internet-facing # operator load balancer scheme [internet-facing | internal] # note: if using "internal", you must be within the cluster's VPC or configure VPC Peering to connect your CLI to your cluster operator operator_load_balancer_scheme: internet-facing - -# node visibility [public (nodes will have public IPs) | private (nodes will not have public IPs)] -# note: if using "private", you will need to configure Cloud NAT in your VPC before creating your cluster -node_visibility: public ``` The docker images used by the Cortex cluster can also be overridden, although this is not common. They can be configured by adding any of these keys to your cluster configuration file (default values are shown): diff --git a/pkg/types/clusterconfig/cluster_config_gcp.go b/pkg/types/clusterconfig/cluster_config_gcp.go index c72c388661..63c0ac2ddf 100644 --- a/pkg/types/clusterconfig/cluster_config_gcp.go +++ b/pkg/types/clusterconfig/cluster_config_gcp.go @@ -40,7 +40,6 @@ type GCPConfig struct { AcceleratorType *string `json:"accelerator_type" yaml:"accelerator_type"` Network *string `json:"network" yaml:"network"` Subnet *string `json:"subnet" yaml:"subnet"` - NodeVisibility SubnetVisibility `json:"node_visibility" yaml:"node_visibility"` APILoadBalancerScheme LoadBalancerScheme `json:"api_load_balancer_scheme" yaml:"api_load_balancer_scheme"` OperatorLoadBalancerScheme LoadBalancerScheme `json:"operator_load_balancer_scheme" yaml:"operator_load_balancer_scheme"` MinInstances *int64 `json:"min_instances" yaml:"min_instances"` @@ -151,16 +150,6 @@ var UserGCPValidation = &cr.StructValidation{ AllowExplicitNull: true, }, }, - { - StructField: "NodeVisibility", - StringValidation: &cr.StringValidation{ - AllowedValues: SubnetVisibilityStrings(), - Default: PublicSubnetVisibility.String(), - }, - Parser: func(str string) (interface{}, error) { - return SubnetVisibilityFromString(str), nil - }, - }, { StructField: "APILoadBalancerScheme", StringValidation: &cr.StringValidation{ @@ -534,7 +523,6 @@ func (cc *GCPConfig) UserTable() table.KeyValuePairs { if cc.Subnet != nil { items.Add(SubnetUserKey, *cc.Subnet) } - items.Add(NodeVisibilityUserKey, cc.NodeVisibility) items.Add(APILoadBalancerSchemeUserKey, cc.APILoadBalancerScheme) items.Add(OperatorLoadBalancerSchemeUserKey, cc.OperatorLoadBalancerScheme) items.Add(TelemetryUserKey, cc.Telemetry) @@ -572,7 +560,6 @@ func (cc *GCPConfig) TelemetryEvent() map[string]interface{} { if cc.Subnet != nil { event["subnet._is_defined"] = true } - event["node_visibility"] = cc.NodeVisibility event["api_load_balancer_scheme"] = cc.APILoadBalancerScheme event["operator_load_balancer_scheme"] = cc.OperatorLoadBalancerScheme if cc.MinInstances != nil { diff --git a/pkg/types/clusterconfig/config_key.go b/pkg/types/clusterconfig/config_key.go index b4057bc576..4cf8f5de74 100644 --- a/pkg/types/clusterconfig/config_key.go +++ b/pkg/types/clusterconfig/config_key.go @@ -47,7 +47,6 @@ const ( SSLCertificateARNKey = "ssl_certificate_arn" BucketKey = "bucket" SubnetVisibilityKey = "subnet_visibility" - NodeVisibilityKey = "node_visibility" NATGatewayKey = "nat_gateway" APILoadBalancerSchemeKey = "api_load_balancer_scheme" OperatorLoadBalancerSchemeKey = "operator_load_balancer_scheme" @@ -99,7 +98,6 @@ const ( InstancePoolsUserKey = "spot instance pools" OnDemandBackupUserKey = "on demand backup" SubnetVisibilityUserKey = "subnet visibility" - NodeVisibilityUserKey = "node visibility" NATGatewayUserKey = "nat gateway" APILoadBalancerSchemeUserKey = "api load balancer scheme" OperatorLoadBalancerSchemeUserKey = "operator load balancer scheme"