feat: add optional published resources

Signed-off-by: Karol Szwaj <[email protected]>

On-behalf-of: @SAP [email protected]

Author: cnvergence

internal/sync/syncer_related.go

@@ -35,13 +35,15 @@ import (
 
 func (s *ResourceSyncer) processRelatedResources(log *zap.SugaredLogger, stateStore ObjectStateStore, remote, local syncSide) (requeue bool, err error) {
 	for _, relatedResource := range s.pubRes.Spec.Related {
-		requeue, err := s.processRelatedResource(log.With("identifier", relatedResource.Identifier), stateStore, remote, local, relatedResource)
-		if err != nil {
-			return false, fmt.Errorf("failed to process related resource %s: %w", relatedResource.Identifier, err)
-		}
+		if !relatedResource.Optional {
+			requeue, err := s.processRelatedResource(log.With("identifier", relatedResource.Identifier), stateStore, remote, local, relatedResource)
+			if err != nil {
+				return false, fmt.Errorf("failed to process related resource %s: %w", relatedResource.Identifier, err)
+			}
 
-		if requeue {
-			return true, nil
+			if requeue {
+				return true, nil
+			}
 		}
 	}
 
@@ -70,28 +72,23 @@ func (s *ResourceSyncer) processRelatedResource(log *zap.SugaredLogger, stateSto
 		dest = local
 	}
 
-	// to find the source related object, we first need to determine its name/namespace
 	sourceKey, err := resolveResourceReference(source.object, relRes.Reference)
 	if err != nil {
 		return false, fmt.Errorf("failed to determine related object's source key: %w", err)
 	}
 
-	// find the source related object
 	sourceObj := &unstructured.Unstructured{}
-	sourceObj.SetAPIVersion("v1") // we only support ConfigMaps and Secrets, both are in core/v1
+	sourceObj.SetAPIVersion("v1")
 	sourceObj.SetKind(relRes.Kind)
 
 	err = source.client.Get(source.ctx, *sourceKey, sourceObj)
 	if err != nil {
-		// the source object doesn't exist yet, so we can just stop
 		if apierrors.IsNotFound(err) {
 			return false, nil
 		}
-
 		return false, fmt.Errorf("failed to get source object: %w", err)
 	}
 
-	// do the same to find the destination object
 	destKey, err := resolveResourceReference(dest.object, relRes.Reference)
 	if err != nil {
 		return false, fmt.Errorf("failed to determine related object's destination key: %w", err)
@@ -137,7 +134,6 @@ func (s *ResourceSyncer) processRelatedResource(log *zap.SugaredLogger, stateSto
 			dest := source.DeepCopy()
 			dest.SetName(destKey.Name)
 			dest.SetNamespace(destKey.Namespace)
-
 			return dest
 		},
 		// ConfigMaps and Secrets have no subresources
@@ -242,10 +238,7 @@ func resolveResourceLocator(jsonData string, loc syncagentv1alpha1.ResourceLocat
 			return "", fmt.Errorf("invalid pattern %q: %w", re.Pattern, err)
 		}
 
-		// this does apply some coalescing, like turning numbers into strings
-		strVal := gval.String()
-
-		return expr.ReplaceAllString(strVal, re.Replacement), nil
+		return expr.ReplaceAllString(gval.String(), re.Replacement), nil
 	}
 
 	return gval.String(), nil

internal/sync/syncer_test.go

@@ -1320,6 +1320,282 @@ func TestSyncerProcessingSingleResourceWithStatus(t *testing.T) {
 		})
 	}
 }
+func TestSyncerProcessingRelatedResources(t *testing.T) {
+	type testcase struct {
+		name                 string
+		remoteAPIGroup       string
+		localCRD             *apiextensionsv1.CustomResourceDefinition
+		pubRes               *syncagentv1alpha1.PublishedResource
+		remoteObject         *unstructured.Unstructured
+		localObject          *unstructured.Unstructured
+		existingState        string
+		performRequeues      bool
+		expectedRemoteObject *unstructured.Unstructured
+		expectedLocalObject  *unstructured.Unstructured
+		expectedState        string
+		customVerification   func(t *testing.T, requeue bool, processErr error, finalRemoteObject *unstructured.Unstructured, finalLocalObject *unstructured.Unstructured, testcase testcase)
+	}
+
+	clusterName := logicalcluster.Name("testcluster")
+
+	remoteThingPR := &syncagentv1alpha1.PublishedResource{
+		Spec: syncagentv1alpha1.PublishedResourceSpec{
+			Resource: syncagentv1alpha1.SourceResourceDescriptor{
+				APIGroup: dummyv1alpha1.GroupName,
+				Version:  dummyv1alpha1.GroupVersion,
+				Kind:     "Thing",
+			},
+			Projection: &syncagentv1alpha1.ResourceProjection{
+				Kind: "RemoteThing",
+			},
+			// include explicit naming rules to be independent of possible changes to the defaults
+			Naming: &syncagentv1alpha1.ResourceNaming{
+				Name: "$remoteClusterName-$remoteName", // Things are Cluster-scoped
+			},
+			Related: []syncagentv1alpha1.RelatedResourceSpec{
+				{
+					Identifier: "mandatory-secret",
+					Origin:     "service",
+					Kind:       "Thing",
+					Reference: syncagentv1alpha1.RelatedResourceReference{
+						Name: syncagentv1alpha1.ResourceLocator{
+							Path: "spec.otherTest.name",
+						},
+						Namespace: &syncagentv1alpha1.ResourceLocator{
+							Path: "spec.otherTest.namespace",
+						},
+					},
+					Optional: false,
+				},
+				{
+					Identifier: "optional-secret",
+					Origin:     "kcp",
+					Kind:       "Thing",
+					Reference: syncagentv1alpha1.RelatedResourceReference{
+						Name: syncagentv1alpha1.ResourceLocator{
+							Path: "spec.test.name",
+						},
+						Namespace: &syncagentv1alpha1.ResourceLocator{
+							Path: "spec.test.namespace",
+						},
+					},
+					Optional: true,
+				},
+			},
+		},
+	}
+
+	testcases := []testcase{
+		{
+			name:            "optional related resource does not exist",
+			remoteAPIGroup:  "remote.example.corp",
+			localCRD:        loadCRD("things"),
+			pubRes:          remoteThingPR,
+			performRequeues: true,
+
+			remoteObject: newUnstructured(&dummyv1alpha1.Thing{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "my-test-thing",
+				},
+				Spec: dummyv1alpha1.ThingSpec{
+					Username: "Colonel Mustard",
+				},
+			}, withGroupKind("remote.example.corp", "RemoteThing")),
+			localObject:   nil,
+			existingState: "",
+
+			expectedRemoteObject: newUnstructured(&dummyv1alpha1.Thing{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "my-test-thing",
+					Finalizers: []string{
+						deletionFinalizer,
+					},
+				},
+				Spec: dummyv1alpha1.ThingSpec{
+					Username: "Colonel Mustard",
+				},
+			}, withGroupKind("remote.example.corp", "RemoteThing")),
+			expectedLocalObject: newUnstructured(&dummyv1alpha1.Thing{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testcluster-my-test-thing",
+					Labels: map[string]string{
+						agentNameLabel:            "textor-the-doctor",
+						remoteObjectClusterLabel:  "testcluster",
+						remoteObjectNameHashLabel: "c346c8ceb5d104cc783d09b95e8ea7032c190948",
+					},
+					Annotations: map[string]string{
+						remoteObjectNameAnnotation: "my-test-thing",
+					},
+				},
+				Spec: dummyv1alpha1.ThingSpec{
+					Username: "Colonel Mustard",
+				},
+			}),
+			expectedState: `{"apiVersion":"remote.example.corp/v1alpha1","kind":"RemoteThing","metadata":{"name":"my-test-thing"},"spec":{"username":"Colonel Mustard"}}`,
+		},
+		{
+			name:            "mandatory related resource does not exist",
+			remoteAPIGroup:  "remote.example.corp",
+			localCRD:        loadCRD("things"),
+			pubRes:          remoteThingPR,
+			performRequeues: true,
+
+			remoteObject: newUnstructured(&dummyv1alpha1.Thing{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "my-test-thing",
+				},
+				Spec: dummyv1alpha1.ThingSpec{
+					Username: "Colonel Mustard",
+				},
+			}, withGroupKind("remote.example.corp", "RemoteThing")),
+			localObject:   nil,
+			existingState: "",
+
+			expectedRemoteObject: newUnstructured(&dummyv1alpha1.Thing{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "my-test-thing",
+					Finalizers: []string{
+						deletionFinalizer,
+					},
+				},
+				Spec: dummyv1alpha1.ThingSpec{
+					Username: "Colonel Mustard",
+				},
+			}, withGroupKind("remote.example.corp", "RemoteThing")),
+			expectedLocalObject: newUnstructured(&dummyv1alpha1.Thing{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testcluster-my-test-thing",
+					Labels: map[string]string{
+						agentNameLabel:            "textor-the-doctor",
+						remoteObjectClusterLabel:  "testcluster",
+						remoteObjectNameHashLabel: "c346c8ceb5d104cc783d09b95e8ea7032c190948",
+					},
+					Annotations: map[string]string{
+						remoteObjectNameAnnotation: "my-test-thing",
+					},
+				},
+				Spec: dummyv1alpha1.ThingSpec{
+					Username: "Colonel Mustard",
+				},
+			}),
+			expectedState: `{"apiVersion":"remote.example.corp/v1alpha1","kind":"RemoteThing","metadata":{"name":"my-test-thing"},"spec":{"username":"Colonel Mustard"}}`,
+		},
+	}
+
+	const stateNamespace = "kcp-system"
+
+	for _, testcase := range testcases {
+		t.Run(testcase.name, func(t *testing.T) {
+			localClient := buildFakeClient(testcase.localObject)
+			remoteClient := buildFakeClient(testcase.remoteObject)
+
+			syncer, err := NewResourceSyncer(
+				// zap.Must(zap.NewDevelopment()).Sugar(),
+				zap.NewNop().Sugar(),
+				localClient,
+				remoteClient,
+				testcase.pubRes,
+				testcase.localCRD,
+				testcase.remoteAPIGroup,
+				nil,
+				stateNamespace,
+				"textor-the-doctor",
+			)
+			if err != nil {
+				t.Fatalf("Failed to create syncer: %v", err)
+			}
+
+			localCtx := context.Background()
+			remoteCtx := kontext.WithCluster(localCtx, clusterName)
+			ctx := NewContext(localCtx, remoteCtx)
+
+			// setup a custom state backend that we can prime
+			var backend *kubernetesBackend
+			syncer.newObjectStateStore = func(primaryObject, stateCluster syncSide) ObjectStateStore {
+				// .Process() is called multiple times, but we want the state to persist between reconciles.
+				if backend == nil {
+					backend = newKubernetesBackend(stateNamespace, primaryObject, stateCluster)
+					if testcase.existingState != "" {
+						if err := backend.Put(testcase.remoteObject, clusterName, []byte(testcase.existingState)); err != nil {
+							t.Fatalf("Failed to prime state store: %v", err)
+						}
+					}
+				}
+
+				return &objectStateStore{
+					backend: backend,
+				}
+			}
+
+			var requeue bool
+
+			if testcase.performRequeues {
+				target := testcase.remoteObject.DeepCopy()
+
+				for i := 0; true; i++ {
+					if i > 20 {
+						t.Fatalf("Detected potential infinite loop, stopping after %d requeues.", i)
+					}
+
+					requeue, err = syncer.Process(ctx, target)
+					if err != nil {
+						break
+					}
+
+					if !requeue {
+						break
+					}
+
+					if err = remoteClient.Get(remoteCtx, ctrlruntimeclient.ObjectKeyFromObject(target), target); err != nil {
+						// it's possible for the processing to have deleted the remote object,
+						// so a NotFound is valid here
+						if apierrors.IsNotFound(err) {
+							break
+						}
+
+						t.Fatalf("Failed to get updated remote object: %v", err)
+					}
+				}
+			} else {
+				requeue, err = syncer.Process(ctx, testcase.remoteObject)
+			}
+
+			finalRemoteObject, getErr := getFinalObjectVersion(remoteCtx, remoteClient, testcase.remoteObject, testcase.expectedRemoteObject)
+			if getErr != nil {
+				t.Fatalf("Failed to get final remote object: %v", getErr)
+			}
+
+			finalLocalObject, getErr := getFinalObjectVersion(localCtx, localClient, testcase.localObject, testcase.expectedLocalObject)
+			if getErr != nil {
+				t.Fatalf("Failed to get final local object: %v", getErr)
+			}
+
+			if testcase.customVerification != nil {
+				testcase.customVerification(t, requeue, err, finalRemoteObject, finalLocalObject, testcase)
+			} else {
+				if err != nil {
+					t.Fatalf("Processing failed: %v", err)
+				}
+
+				assertObjectsEqual(t, "local", testcase.expectedLocalObject, finalLocalObject)
+				assertObjectsEqual(t, "remote", testcase.expectedRemoteObject, finalRemoteObject)
+
+				if testcase.expectedState != "" {
+					if backend == nil {
+						t.Fatal("Cannot check object state, state store was never instantiated.")
+					}
+
+					finalState, err := backend.Get(testcase.expectedRemoteObject, clusterName)
+					if err != nil {
+						t.Fatalf("Failed to get final state: %v", err)
+					} else if !bytes.Equal(finalState, []byte(testcase.expectedState)) {
+						t.Fatalf("States do not match:\n%s", diff.StringDiff(testcase.expectedState, string(finalState)))
+					}
+				}
+			}
+		})
+	}
+}
 
 func assertObjectsEqual(t *testing.T, kind string, expected, actual *unstructured.Unstructured) {
 	if expected == nil {

sdk/apis/syncagent/v1alpha1/published_resource.go

@@ -176,6 +176,9 @@ type RelatedResourceSpec struct {
 	// Mutation configures optional transformation rules for the related resource.
 	// Status mutations are only performed when the related resource originates in kcp.
 	Mutation *ResourceMutationSpec `json:"mutation,omitempty"`
+
+	// Optional indicates whether the related resource must be referenced.
+	Optional bool `json:"optional,omitempty"`
 }
 
 type RelatedResourceReference struct {