Add frozen field to cluster spec that will suspend changes

docs/api.md

@@ -69,6 +69,7 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to.
 | podMetadata | PodMetadata is for any Metadata that is unique to the pods, and does not belong on any other objects, such as Prometheus scrape tags | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#objectmeta-v1-meta) | false |
 | parallelPodManagement | ParallelPodManagement sets StatefulSets created by the operator to have Parallel pod management instead of OrderedReady. This is an EXPERIMENTAL flag and subject to deprecation in a future release. This has not been tested in production and users should not depend on it without validating it for their own use case. | bool | true |
 | serviceAccountName | To use a non-default service account, specify the name here otherwise the service account \"default\" will be used. This is useful for advanced use-cases such as pod security policies. The service account must exist. This operator will not create it. | string | false |
+| frozen | Frozen is used to stop the operator from taking any further actions on a cluster. This is useful when troubleshooting as it guarantees the operator won't make any changes to the cluster. | bool | false |
 
 [Back to TOC](#table-of-contents)
 

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/uber-go/tally v3.3.13+incompatible
 	github.com/ultraware/funlen v0.0.2 // indirect
 	github.com/urfave/cli v1.22.2 // indirect
+	go.uber.org/atomic v1.6.0
 	go.uber.org/zap v1.13.0
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f
 	golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 // indirect

pkg/apis/m3dboperator/v1alpha1/cluster.go

@@ -306,6 +306,12 @@ type ClusterSpec struct {
 	// use-cases such as pod security policies. The service account must exist.
 	// This operator will not create it.
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
+	// Frozen is used to stop the operator from taking any further actions on a
+	// cluster. This is useful when troubleshooting as it guarantees the operator
+	// won't make any changes to the cluster.
+	// +optional
+	Frozen bool `json:"frozen,omitempty"`
 }
 
 // ExternalCoordinatorConfig defines parameters for using an external

pkg/controller/controller.go

@@ -358,6 +358,11 @@ func (c *M3DBController) handleClusterUpdate(cluster *myspec.M3DBCluster) error
 
 	clusterLogger := c.logger.With(zap.String("cluster", cluster.Name))
 
+	if cluster.Spec.Frozen {
+		clusterLogger.Info("cluster is frozen so no changes will be made")
+		return nil
+	}
+
 	// https://v1-12.docs.kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/
 	//
 	// If deletion timestamp is zero (cluster hasn't been deleted), make sure our

pkg/controller/controller_test.go

@@ -57,6 +57,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/uber-go/tally"
+	"go.uber.org/atomic"
 	"go.uber.org/zap"
 )
 
@@ -784,3 +785,36 @@ func TestHandleUpdateClusterUpdatesStatefulSets(t *testing.T) {
 		})
 	}
 }
+
+func TestHandleUpdateClusterFrozen(t *testing.T) {
+
+	var (
+		clusterMeta = newMeta("cluster1", map[string]string{
+			"foo":                      "bar",
+			"operator.m3db.io/app":     "m3db",
+			"operator.m3db.io/cluster": "cluster1",
+		}, nil)
+		sets = []*metav1.ObjectMeta{
+			newMeta("cluster1-rep0", nil, nil),
+		}
+	)
+	cluster, deps := setupTestCluster(t, *clusterMeta, sets, 3)
+	defer deps.cleanup()
+	controller := deps.newController(t)
+
+	cluster.Spec.Frozen = true
+
+	count := atomic.NewInt64(0)
+	controller.kubeClient.(*kubefake.Clientset).PrependReactor(
+		"*", "*", func(action ktesting.Action) (bool, runtime.Object, error) {
+			count.Inc()
+			return false, nil, nil
+		})
+
+	for i := 0; i < 20; i++ {
+		err := controller.handleClusterUpdate(cluster)
+		require.NoError(t, err)
+	}
+
+	assert.Equal(t, int64(0), count.Load())
+}