Prevent duplicate Vault logins when reconciliations commence

* Vault logins are only made from the renewToken thread
* renewToken notifies main thread when first login attempt
  has been made by means of a buffered channel that is passed
  into it as an argument
* token lifecycle management only commences if the first Vault
  login attempt succeeds
* small delay added between Vault login attempts if there are
  problems renewing a token (e.g. if Vault server cannot be
  reached because of a temporary problem on the network)

Author: George Harley

internal/vault_reader.go

@@ -3,10 +3,10 @@ package internal
 import (
 	"errors"
 	"fmt"
+	rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/api/v1beta1"
 	"os"
 	"sync"
-
-	rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/api/v1beta1"
+	"time"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 
@@ -41,13 +41,19 @@ type VaultClient struct {
 	Reader SecretReader
 }
 
-var ReadServiceAccountTokenFunc = ReadServiceAccountToken
-var ReadVaultClientSecretFunc = ReadVaultClientSecret
-var LoginToVaultFunc = LoginToVault
+// Created - and exported from package - for testing purposes
+var (
+	ReadServiceAccountTokenFunc = ReadServiceAccountToken
+	ReadVaultClientSecretFunc   = ReadVaultClientSecret
+	LoginToVaultFunc            = LoginToVault
+	FirstLoginAttemptResultCh   = make(chan error, 1)
+)
 
-var createSecretStoreClientOnce sync.Once
-var SecretClient SecretStoreClient
-var SecretClientCreationError error
+var (
+	createSecretStoreClientOnce sync.Once
+	SecretClient                SecretStoreClient
+	SecretClientCreationError   error
+)
 
 func GetSecretStoreClient(vaultSpec *rabbitmqv1beta1.VaultSpec) (SecretStoreClient, error) {
 	createSecretStoreClientOnce.Do(InitializeClient(vaultSpec))
@@ -64,7 +70,8 @@ func InitializeClient(vaultSpec *rabbitmqv1beta1.VaultSpec) func() {
 			return
 		}
 
-		_, err = login(vaultClient, vaultSpec)
+		go renewToken(vaultClient, vaultSpec, FirstLoginAttemptResultCh)
+		err = <-FirstLoginAttemptResultCh
 		if err != nil {
 			SecretClientCreationError = fmt.Errorf("unable to login to Vault: %w", err)
 			return
@@ -73,8 +80,6 @@ func InitializeClient(vaultSpec *rabbitmqv1beta1.VaultSpec) func() {
 		SecretClient = VaultClient{
 			Reader: &VaultSecretReader{client: vaultClient},
 		}
-
-		go renewToken(vaultClient, vaultSpec)
 	}
 }
 
@@ -187,19 +192,34 @@ func login(vaultClient *vault.Client, vaultSpec *rabbitmqv1beta1.VaultSpec) (*va
 	return vaultSecret, nil
 }
 
-func renewToken(client *vault.Client, vaultSpec *rabbitmqv1beta1.VaultSpec) {
+func renewToken(client *vault.Client, vaultSpec *rabbitmqv1beta1.VaultSpec, initialLoginErrorCh chan<- error) {
 	logger := ctrl.LoggerFrom(nil)
+	sentFirstLoginAttemptErr := false
 
 	for {
 		vaultLoginResp, err := login(client, vaultSpec)
 		if err != nil {
 			logger.Error(err, "unable to authenticate to Vault server")
 		}
 
+		if !sentFirstLoginAttemptErr {
+			initialLoginErrorCh <- err
+			sentFirstLoginAttemptErr = true
+			if err != nil {
+				// Initial login attempt failed so fail fast and don't try to manage (non-existent) token lifecycle
+				logger.Info("Lifecycle management of Vault token will not be carried out")
+				return
+			}
+			logger.Info("Initiating lifecycle management of Vault token")
+		}
+
 		err = manageTokenLifecycle(client, vaultLoginResp)
 		if err != nil {
 			logger.Error(err, "unable to start managing the Vault token lifecycle")
 		}
+
+		// Reduce load on Vault server in a problem situation where repeated login attempts may be made
+		time.Sleep(2 * time.Second)
 	}
 }
 

internal/vault_reader_test.go

@@ -289,8 +289,8 @@ var _ = Describe("VaultReader", func() {
 
 	Describe("Initialize secret store client", func() {
 		var (
-			vaultSpec *rabbitmqv1beta1.VaultSpec
-			getSecretStoreClientTester func(vaultSpec *rabbitmqv1beta1.VaultSpec)(internal.SecretStoreClient, error)
+			vaultSpec                  *rabbitmqv1beta1.VaultSpec
+			getSecretStoreClientTester func(vaultSpec *rabbitmqv1beta1.VaultSpec) (internal.SecretStoreClient, error)
 		)
 
 		When("vault spec has no role value", func() {
@@ -319,6 +319,7 @@ var _ = Describe("VaultReader", func() {
 
 		When("service account token is not in the expected place", func() {
 			BeforeEach(func() {
+				internal.FirstLoginAttemptResultCh = make(chan error, 1)
 				internal.SecretClient = nil
 				internal.SecretClientCreationError = nil
 				vaultSpec = &rabbitmqv1beta1.VaultSpec{
@@ -346,6 +347,7 @@ var _ = Describe("VaultReader", func() {
 
 		When("unable to log into vault to obtain client secret", func() {
 			BeforeEach(func() {
+				internal.FirstLoginAttemptResultCh = make(chan error, 1)
 				internal.SecretClient = nil
 				internal.SecretClientCreationError = nil
 				vaultSpec = &rabbitmqv1beta1.VaultSpec{
@@ -384,6 +386,7 @@ var _ = Describe("VaultReader", func() {
 
 		When("client secret obtained from vault", func() {
 			BeforeEach(func() {
+				internal.FirstLoginAttemptResultCh = make(chan error, 1)
 				internal.SecretClient = nil
 				internal.SecretClientCreationError = nil
 				vaultSpec = &rabbitmqv1beta1.VaultSpec{