Add comprehensive HCP full backup/restore test suite with existing cluster support

This commit introduces a complete HCP (Hosted Control Plane) backup and restore testing framework
with support for both newly created and existing HostedCluster environments.

## Key Features Added:

### New Test Infrastructure
- Add `hcp_full_backup_restore_suite_test.go`: Complete test suite for full HCP backup/restore scenarios
- Support for two operational modes:
  - `create`: Creates new HostedCluster for testing (existing behavior)
  - `existing`: Uses pre-existing HostedCluster with data plane

### Enhanced Configuration Options
- Add Makefile variables for HCP test configuration:
  - `HC_BACKUP_RESTORE_MODE`: Controls test execution mode (create/existing)
  - `HC_NAME`: Specifies HostedCluster name for existing mode
  - `HC_KUBECONFIG`: Path to guest cluster kubeconfig for existing mode
- Pass HCP configuration parameters to e2e test execution

### Improved Test Architecture
- Refactor `runHCPBackupAndRestore()` function for unified handling of both modes
- Add guest cluster verification functions (`PreBackupVerifyGuest`, `PostRestoreVerifyGuest`)
- Separate log gathering and DPA resource cleanup into reusable functions
- Enhanced error handling and validation for both control plane and guest cluster

### Guest Cluster Testing
- Add support for kubeconfig-based guest cluster operations
- Implement pre/post backup verification for guest cluster resources
- Add namespace creation/validation tests for guest cluster functionality

### Library Enhancements
- Add `GetHostedCluster()` method to retrieve existing HostedCluster objects
- Add `ClientGuest` field to `HCHandler` for guest cluster operations
- Improve error message formatting in DPA helpers

### Documentation Updates
- Add comprehensive testing documentation for HCP scenarios
- Include examples for running tests against existing HostedControlPlane
- Document environment variable configuration options

### Build System Improvements
- Add conditional must-gather build based on `SKIP_MUST_GATHER` flag
- Enhanced e2e test parameter passing for HCP configurations

## Technical Implementation:

The implementation supports testing both scenarios where OADP needs to:
1. Create a new HostedCluster and test backup/restore (existing functionality)
2. Work with an existing HostedCluster that already has workloads and data plane

This enables comprehensive testing of HCP backup/restore functionality in realistic
production-like environments where clusters already exist and contain user workloads.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: mgencur

Makefile

@@ -64,6 +64,16 @@ TTL_DURATION ?= 1h
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.32 # Kubernetes version from OpenShift 4.19.x https://openshift-release.apps.ci.l2s4.p1.openshiftapps.com/#4-stable
 
+# HC_BACKUP_RESTORE_MODE is used to run HCP tests against existing HostedControlPlane.
+# Possible values are: create, existing.
+HC_BACKUP_RESTORE_MODE ?= create
+# HC_NAME is the name of the HostedCluster to use for HCP tests when HC_BACKUP_RESTORE_MODE is set to existing.
+# Otherwise, HC_NAME is ignored.
+HC_NAME ?= ""
+# HC_KUBECONFIG is the path to the kubeconfig file for the HostedCluster to use for HCP tests when HC_BACKUP_RESTORE_MODE is set to existing.
+# Otherwise, HC_KUBECONFIG is ignored.
+HC_KUBECONFIG ?= ""
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -769,6 +779,9 @@ test-e2e: test-e2e-setup install-ginkgo ## Run E2E tests against OADP operator i
 	-velero_instance_name=$(VELERO_INSTANCE_NAME) \
 	-artifact_dir=$(ARTIFACT_DIR) \
 	-kvm_emulation=$(KVM_EMULATION) \
+	-hc_backup_restore_mode=$(HC_BACKUP_RESTORE_MODE) \
+	-hc_name=$(HC_NAME) \
+	-hc_kubeconfig=$(HC_KUBECONFIG) \
 	-hco_upstream=$(HCO_UPSTREAM) \
         -skipMustGather=$(SKIP_MUST_GATHER) \
 	--ginkgo.vv \
@@ -792,7 +805,6 @@ test-e2e-cleanup: login-required
 	for restore_name in $(shell $(OC_CLI) get restore -n $(OADP_TEST_NAMESPACE) -o name);do $(OC_CLI) patch "$$restore_name" -n $(OADP_TEST_NAMESPACE) -p '{"metadata":{"finalizers":null}}' --type=merge;done
 	rm -rf $(SETTINGS_TMP)
 
-
 .PHONY: update-non-admin-manifests
 update-non-admin-manifests: NON_ADMIN_CONTROLLER_IMG?=quay.io/konveyor/oadp-non-admin:latest
 update-non-admin-manifests: yq ## Update Non Admin Controller (NAC) manifests shipped with OADP, from NON_ADMIN_CONTROLLER_PATH
@@ -816,4 +828,8 @@ endif
 
 .PHONY: build-must-gather
 build-must-gather: ## Build OADP Must-gather binary must-gather/oadp-must-gather
+ifeq ($(SKIP_MUST_GATHER),true)
+	echo "Skipping must-gather build"
+else
 	cd must-gather && go build -mod=mod -a -o oadp-must-gather cmd/main.go
+endif

docs/developer/testing/TESTING.md

@@ -99,6 +99,17 @@ You can also execute make test-e2e with a $GINKGO_ARGS variable set. Example:
 make test-e2e GINKGO_ARGS="--ginkgo.focus='MySQL application DATAMOVER'"
 ```
 
+### Run selected test for HCP against existing HostedControlPlane
+
+Set common env variables as mentioned above, then run:
+
+```bash
+HC_BACKUP_RESTORE_MODE=existing \
+HC_NAME=hc1 \
+HC_KUBECONFIG=/path/to/kubeconfig/for/hosted/cluster \
+make test-e2e GINKGO_ARGS="--ginkgo.focus='HCP full Backup and Restore tests'"
+```
+
 ### Run tests with custom images
 
 You can run tests with custom images by setting the following environment variables:

tests/e2e/backup_restore_suite_test.go

@@ -237,6 +237,7 @@ func runRestore(brCase BackupRestoreCase, backupName, restoreName string, nsRequ
 
 func getFailedTestLogs(oadpNamespace string, appNamespace string, installTime time.Time, report ginkgo.SpecReport) {
 	baseReportDir := artifact_dir + "/" + report.LeafNodeText
+	log.Println("Storing failed test logs in: ", baseReportDir)
 	err := os.MkdirAll(baseReportDir, 0755)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -255,12 +256,12 @@ func getFailedTestLogs(oadpNamespace string, appNamespace string, installTime ti
 
 func tearDownBackupAndRestore(brCase BackupRestoreCase, installTime time.Time, report ginkgo.SpecReport) {
 	log.Println("Post backup and restore state: ", report.State.String())
+	gatherLogs(brCase, installTime, report)
+	tearDownDPAResources(brCase)
+	deleteNamespace(brCase.Namespace)
+}
 
-	if report.Failed() {
-		knownFlake = lib.CheckIfFlakeOccurred(accumulatedTestLogs)
-		accumulatedTestLogs = nil
-		getFailedTestLogs(namespace, brCase.Namespace, installTime, report)
-	}
+func tearDownDPAResources(brCase BackupRestoreCase) {
 	if brCase.BackupRestoreType == lib.CSI || brCase.BackupRestoreType == lib.CSIDataMover {
 		log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name)
 		snapshotClassPath := fmt.Sprintf("./sample-applications/snapclass-csi/%s.yaml", provider)
@@ -270,10 +271,20 @@ func tearDownBackupAndRestore(brCase BackupRestoreCase, installTime time.Time, r
 
 	err := dpaCR.Delete()
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+}
+
+func gatherLogs(brCase BackupRestoreCase, installTime time.Time, report ginkgo.SpecReport) {
+	if report.Failed() {
+		knownFlake = lib.CheckIfFlakeOccurred(accumulatedTestLogs)
+		accumulatedTestLogs = nil
+		getFailedTestLogs(namespace, brCase.Namespace, installTime, report)
+	}
+}
 
-	err = lib.DeleteNamespace(kubernetesClientForSuiteRun, brCase.Namespace)
+func deleteNamespace(namespace string) {
+	err := lib.DeleteNamespace(kubernetesClientForSuiteRun, namespace)
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
-	gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, brCase.Namespace), time.Minute*5, time.Second*5).Should(gomega.BeTrue())
+	gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, namespace), time.Minute*5, time.Second*5).Should(gomega.BeTrue())
 }
 
 var _ = ginkgo.Describe("Backup and restore tests", ginkgo.Ordered, func() {

tests/e2e/e2e_suite_test.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -24,10 +25,11 @@ import (
 
 var (
 	// Common vars obtained from flags passed in ginkgo.
-	bslCredFile, namespace, instanceName, provider, vslCredFile, settings, artifact_dir string
-	flakeAttempts                                                                       int64
+	bslCredFile, namespace, instanceName, provider, vslCredFile, settings, artifact_dir, hcKubeconfig string
+	flakeAttempts                                                                                     int64
 
 	kubernetesClientForSuiteRun *kubernetes.Clientset
+	crClientForHC               client.Client
 	runTimeClientForSuiteRun    client.Client
 	dynamicClientForSuiteRun    dynamic.Interface
 
@@ -37,12 +39,15 @@ var (
 	vslSecretName                   string
 
 	kubeConfig          *rest.Config
+	kubeConfigForHC     *rest.Config
 	knownFlake          bool
 	accumulatedTestLogs []string
 
-	kvmEmulation   bool
-	useUpstreamHco bool
-	skipMustGather bool
+	kvmEmulation        bool
+	useUpstreamHco      bool
+	skipMustGather      bool
+	hcBackupRestoreMode string
+	hcName              string
 )
 
 func init() {
@@ -59,6 +64,9 @@ func init() {
 	flag.BoolVar(&kvmEmulation, "kvm_emulation", true, "Enable or disable KVM emulation for virtualization testing")
 	flag.BoolVar(&useUpstreamHco, "hco_upstream", false, "Force use of upstream virtualization operator")
 	flag.BoolVar(&skipMustGather, "skipMustGather", false, "avoid errors with local execution and cluster architecture")
+	flag.StringVar(&hcBackupRestoreMode, "hc_backup_restore_mode", string(HCModeCreate), "Type of HC test to run")
+	flag.StringVar(&hcName, "hc_name", "", "Name of the HostedCluster to use for HCP tests")
+	flag.StringVar(&hcKubeconfig, "hc_kubeconfig", "", "Path to kubeconfig file for HostedCluster")
 
 	// helps with launching debug sessions from IDE
 	if os.Getenv("E2E_USE_ENV_FLAGS") == "true" {
@@ -115,6 +123,17 @@ func init() {
 				log.Println("Error parsing SKIP_MUST_GATHER, must-gather will be enabled by default: ", err)
 			}
 		}
+		if os.Getenv("HC_BACKUP_RESTORE_MODE") != "" {
+			hcBackupRestoreMode = os.Getenv("HC_BACKUP_RESTORE_MODE")
+		} else {
+			hcBackupRestoreMode = string(HCModeCreate)
+		}
+		if os.Getenv("HC_NAME") != "" {
+			hcName = os.Getenv("HC_NAME")
+		}
+		if os.Getenv("HC_KUBECONFIG") != "" {
+			hcKubeconfig = os.Getenv("HC_KUBECONFIG")
+		}
 	}
 
 }
@@ -123,15 +142,27 @@ func TestOADPE2E(t *testing.T) {
 	flag.Parse()
 
 	var err error
+
 	kubeConfig = config.GetConfigOrDie()
 	kubeConfig.QPS = 50
 	kubeConfig.Burst = 100
 
 	gomega.RegisterFailHandler(ginkgo.Fail)
 
+	// Set up kubeConfigForHC if kubeconfig_hc flag is provided
+	if hcKubeconfig != "" {
+		kubeConfigForHC, err = clientcmd.BuildConfigFromFlags("", hcKubeconfig)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		kubeConfigForHC.QPS = kubeConfig.QPS
+		kubeConfigForHC.Burst = kubeConfig.Burst
+	}
+
 	kubernetesClientForSuiteRun, err = kubernetes.NewForConfig(kubeConfig)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
+	crClientForHC, err = client.New(kubeConfigForHC, client.Options{Scheme: lib.Scheme})
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
 	runTimeClientForSuiteRun, err = client.New(kubeConfig, client.Options{Scheme: lib.Scheme})
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -200,7 +231,6 @@ var _ = ginkgo.AfterSuite(func() {
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	err = lib.DeleteSecret(kubernetesClientForSuiteRun, namespace, bslSecretNameWithCarriageReturn)
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
-
 	log.Printf("Deleting DPA")
 	err = dpaCR.Delete()
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())

tests/e2e/hcp_backup_restore_suite_test.go

@@ -8,19 +8,29 @@ import (
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/openshift/oadp-operator/tests/e2e/lib"
 	libhcp "github.com/openshift/oadp-operator/tests/e2e/lib/hcp"
 )
 
-type HCPBackupRestoreCase struct {
-	BackupRestoreCase
-	Template string
-	Provider string
-}
+type HCBackupRestoreMode string
 
-func runHCPBackupAndRestore(brCase HCPBackupRestoreCase, updateLastBRcase func(brCase HCPBackupRestoreCase), h *libhcp.HCHandler) {
+const (
+	HCModeCreate   HCBackupRestoreMode = "create"   // Create new HostedCluster for test
+	HCModeExisting HCBackupRestoreMode = "existing" // Get existing HostedCluster
+	// TODO: Add HCModeExistingROSA for ROSA where DPA and some other resources are already installed
+)
+
+// runHCPBackupAndRestore is the unified function that handles both create and existing HC modes
+func runHCPBackupAndRestore(
+	brCase HCPBackupRestoreCase,
+	updateLastBRcase func(HCPBackupRestoreCase),
+	updateLastInstallTime func(),
+	h *libhcp.HCHandler,
+) {
 	updateLastBRcase(brCase)
+	updateLastInstallTime()
 
 	log.Printf("Preparing backup and restore")
 	backupName, restoreName := prepareBackupAndRestore(brCase.BackupRestoreCase, func() {})
@@ -29,19 +39,41 @@ func runHCPBackupAndRestore(brCase HCPBackupRestoreCase, updateLastBRcase func(b
 	gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to add HCP plugin to DPA: %v", err)
 	// TODO: move the wait for HC just after the DPA modification to allow reconciliation to go ahead without waiting for the HC to be created
 
-	//Wait for HCP plugin to be added
+	// Wait for HCP plugin to be added
 	gomega.Eventually(libhcp.IsHCPPluginAdded(h.Client, dpaCR.Namespace, dpaCR.Name), 3*time.Minute, 1*time.Second).Should(gomega.BeTrue())
 
-	// Create the HostedCluster for the test
 	h.HCPNamespace = libhcp.GetHCPNamespace(brCase.BackupRestoreCase.Name, libhcp.ClustersNamespace)
-	h.HostedCluster, err = h.DeployHCManifest(brCase.Template, brCase.Provider, brCase.BackupRestoreCase.Name)
-	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
+	// Unified HostedCluster setup
+	switch brCase.Mode {
+	case HCModeCreate:
+		// Create new HostedCluster for test
+		h.HostedCluster, err = h.DeployHCManifest(brCase.Template, brCase.Provider, brCase.BackupRestoreCase.Name)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	case HCModeExisting:
+		// Get existing HostedCluster
+		h.HostedCluster, err = h.GetHostedCluster(brCase.BackupRestoreCase.Name, libhcp.ClustersNamespace)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	default:
+		ginkgo.Fail(fmt.Sprintf("unknown HCP mode: %s", brCase.Mode))
+	}
+
+	// Pre-backup verification
 	if brCase.PreBackupVerify != nil {
-		err := brCase.PreBackupVerify(runTimeClientForSuiteRun, brCase.Namespace)
+		log.Printf("Validating HC pre-backup")
+		err := brCase.PreBackupVerify(runTimeClientForSuiteRun, "" /*unused*/)
 		gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run HCP pre-backup verification: %v", err)
 	}
 
+	if brCase.Mode == HCModeExisting {
+		// Pre-backup verification for guest cluster
+		if brCase.PreBackupVerifyGuest != nil {
+			log.Printf("Validating guest cluster pre-backup")
+			err := brCase.PreBackupVerifyGuest(crClientForHC, "" /*unused*/)
+			gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run pre-backup verification for guest cluster: %v", err)
+		}
+	}
+
 	// Backup HCP & HC
 	log.Printf("Backing up HC")
 	includedResources := libhcp.HCPIncludedResources
@@ -59,10 +91,32 @@ func runHCPBackupAndRestore(brCase HCPBackupRestoreCase, updateLastBRcase func(b
 	log.Printf("Restoring HC")
 	runHCPRestore(brCase.BackupRestoreCase, backupName, restoreName, nsRequiresResticDCWorkaround)
 
-	// Wait for HCP to be restored
-	log.Printf("Validating HC")
-	err = libhcp.ValidateHCP(libhcp.ValidateHCPTimeout, libhcp.Wait10Min, []string{}, h.HCPNamespace)(h.Client, libhcp.ClustersNamespace)
-	gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run HCP post-restore verification: %v", err)
+	// Unified post-restore verification
+	if brCase.PostRestoreVerify != nil {
+		log.Printf("Validating HC post-restore")
+		err = brCase.PostRestoreVerify(runTimeClientForSuiteRun, "" /*unused*/)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run HCP post-restore verification: %v", err)
+	}
+
+	if brCase.Mode == HCModeExisting {
+		// Post-restore verification for guest cluster
+		if brCase.PostRestoreVerifyGuest != nil {
+			log.Printf("Validating guest cluster post-restore")
+			err := brCase.PostRestoreVerifyGuest(crClientForHC, "" /*unused*/)
+			gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run post-restore verification for guest cluster: %v", err)
+		}
+	}
+}
+
+type VerificationFunctionGuest func(client.Client, string) error
+
+type HCPBackupRestoreCase struct {
+	BackupRestoreCase
+	Mode                   HCBackupRestoreMode
+	PreBackupVerifyGuest   VerificationFunctionGuest
+	PostRestoreVerifyGuest VerificationFunctionGuest
+	Template               string // Optional: only used when Mode == HCPModeCreate
+	Provider               string // Optional: only used when Mode == HCPModeCreate
 }
 
 var _ = ginkgo.Describe("HCP Backup and Restore tests", ginkgo.Ordered, func() {
@@ -78,8 +132,15 @@ var _ = ginkgo.Describe("HCP Backup and Restore tests", ginkgo.Ordered, func() {
 		lastBRCase = brCase
 	}
 
+	updateLastInstallTime := func() {
+		lastInstallTime = time.Now()
+	}
+
 	// Before All
 	var _ = ginkgo.BeforeAll(func() {
+		if hcBackupRestoreMode == string(HCModeExisting) {
+			ginkgo.Skip("Skipping HCP backup and restore test for existing HCP")
+		}
 		reqOperators := []libhcp.RequiredOperator{
 			{
 				Name:          libhcp.MCEName,
@@ -124,11 +185,12 @@ var _ = ginkgo.Describe("HCP Backup and Restore tests", ginkgo.Ordered, func() {
 			if ginkgo.CurrentSpecReport().NumAttempts > 1 && !knownFlake {
 				ginkgo.Fail("No known FLAKE found in a previous run, marking test as failed.")
 			}
-			runHCPBackupAndRestore(brCase, updateLastBRcase, h)
+			runHCPBackupAndRestore(brCase, updateLastBRcase, updateLastInstallTime, h)
 		},
 
 		// Test Cases
 		ginkgo.Entry("None HostedCluster backup and restore", ginkgo.Label("hcp"), HCPBackupRestoreCase{
+			Mode:     HCModeCreate,
 			Template: libhcp.HCPNoneManifest,
 			Provider: "None",
 			BackupRestoreCase: BackupRestoreCase{
@@ -142,6 +204,7 @@ var _ = ginkgo.Describe("HCP Backup and Restore tests", ginkgo.Ordered, func() {
 		}, nil),
 
 		ginkgo.Entry("Agent HostedCluster backup and restore", ginkgo.Label("hcp"), HCPBackupRestoreCase{
+			Mode:     HCModeCreate,
 			Template: libhcp.HCPAgentManifest,
 			Provider: "Agent",
 			BackupRestoreCase: BackupRestoreCase{