From 6b26b615117e62853f0232f9b9752d1494e1f5dd Mon Sep 17 00:00:00 2001
From: Jerome Froelich <jeromefroelich@hotmail.com>
Date: Thu, 24 Sep 2020 19:21:35 -0400
Subject: [PATCH] [controller] Fix bug in
 TestHandleUpdateClusterCreatesStatefulSets

---
 go.sum                            |  1 -
 pkg/controller/controller.go      |  8 ++++++++
 pkg/controller/controller_test.go | 15 ++++++++-------
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/go.sum b/go.sum
index a9325391..b4ef31c7 100644
--- a/go.sum
+++ b/go.sum
@@ -839,7 +839,6 @@ k8s.io/apiextensions-apiserver v0.17.2 h1:cP579D2hSZNuO/rZj9XFRzwJNYb41DbNANJb6K
 k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs=
 k8s.io/apimachinery v0.17.2 h1:hwDQQFbdRlpnnsR64Asdi55GyCaIP/3WQpMmbNBeWr4=
 k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
-k8s.io/apimachinery v0.18.5 h1:Lh6tgsM9FMkC12K5T5QjRm7rDs6aQN5JHkA0JomULDM=
 k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo=
 k8s.io/client-go v0.17.2 h1:ndIfkfXEGrNhLIgkr0+qhRguSD3u6DCmonepn1O6NYc=
 k8s.io/client-go v0.17.2/go.mod h1:QAzRgsa0C2xl4/eVpeVAZMvikCn8Nm81yqVx3Kk9XYI=
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index f47d38c3..a0a1daba 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -450,6 +450,14 @@ func (c *M3DBController) handleClusterUpdate(cluster *myspec.M3DBCluster) error
 
 			_, err = c.kubeClient.AppsV1().StatefulSets(cluster.Namespace).Create(sts)
 			if err != nil {
+				if kerrors.IsAlreadyExists(err) {
+					// There may be a delay between when a StatefulSet is created and when it is
+					// returned in calls to List because of the caching that the k8s client does.
+					// So if we receive an error because the StatefulSet already exists that's not
+					// indicative of a real problem.
+					c.logger.Info("statefulset already exists", zap.String("name", name))
+					return nil
+				}
 				c.logger.Error(err.Error())
 				return err
 			}
diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go
index 3ba5ad72..d2deb091 100644
--- a/pkg/controller/controller_test.go
+++ b/pkg/controller/controller_test.go
@@ -529,22 +529,23 @@ func TestHandleUpdateClusterCreatesStatefulSets(t *testing.T) {
 
 			// Keep running cluster updates until no more stateful sets required
 			var expectedMu sync.Mutex
-			expectedSetsCreated := make(map[string]bool)
-			for _, name := range test.expCreateStatefulSets {
-				expectedSetsCreated[name] = false
-			}
+			expectedSetsCreated := make(map[string]struct{})
 
 			c.kubeClient.(*kubefake.Clientset).PrependReactor("create", "statefulsets", func(action ktesting.Action) (bool, runtime.Object, error) {
 				cluster := action.(kubetesting.CreateActionImpl).GetObject().(*appsv1.StatefulSet)
 				name := cluster.Name
 				expectedMu.Lock()
-				expectedSetsCreated[name] = true
+				expectedSetsCreated[name] = struct{}{}
 				expectedMu.Unlock()
-				return true, cluster, nil
+				// Return false to indicate that the next reactor in the chain should process this
+				// object as well.
+				return false, nil, nil
 			})
 
+			// Iterate through the expected stateful sets twice to make sure we see all stateful
+			// sets that we expect and also be able to catch any extra stateful sets that we don't.
 			var done bool
-			for i := 0; i < 5; i++ {
+			for i := 0; i < 2*len(test.expCreateStatefulSets); i++ {
 				err := c.handleClusterUpdate(cluster)
 				require.NoError(t, err)