Fix 6323: allow compression of the configMap

Add the `--gzip-configmap=true` CLI parameter in order to compress the configMap, if it exeeds the max length.

Signed-off-by: Nahshon Unna-Tsameret <[email protected]>

Author: nunnatsa

changelog/fragments/support_configmap_compression.yaml

@@ -0,0 +1,19 @@
+# entries is a list of entries to include in
+# release notes and/or the migration guide
+entries:
+  - description: >
+      Allow compress the bundle content. Added a new cli flag `--gzip-configmap=true` to the `operator-sdk run bundle`.
+      This will create compressed configmaps. Use it when getting this error:
+      `... ConfigMap ... is invalid: []: Too long: must have at most 1048576 bytes`.
+      Fixes issue [#6323](https://github.com/operator-framework/operator-sdk/issues/6323)
+
+    # kind is one of:
+    # - addition
+    # - change
+    # - deprecation
+    # - removal
+    # - bugfix
+    kind: "bugfix"
+
+    # Is this a breaking change?
+    breaking: false

internal/olm/operator/config.go

@@ -37,6 +37,7 @@ type Configuration struct {
 	Client         client.Client
 	Scheme         *runtime.Scheme
 	Timeout        time.Duration
+	GzipCM         bool
 
 	overrides *clientcmd.ConfigOverrides
 }
@@ -62,6 +63,8 @@ func (c *Configuration) BindFlags(fs *pflag.FlagSet) {
 			"This value does not override the operator's service account")
 	fs.DurationVar(&c.Timeout, "timeout", 2*time.Minute,
 		"Duration to wait for the command to complete before failing")
+	fs.BoolVar(&c.GzipCM, "gzip-configmap", false, `If 'true', the configmap data will be compressed. `+
+		`Use if in case of getting the like "... ConfigMap ... is invalid: []: Too long: must have at most 1048576 bytes"`)
 }
 
 func (c *Configuration) Load() error {

internal/olm/operator/registry/fbcindex/configMapWriter.go

@@ -0,0 +1,225 @@
+package fbcindex
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"strings"
+)
+
+const (
+	yamlSeparator    = "\n---\n"
+	gzipSuffixLength = 13
+	maxGZIPLength    = maxConfigMapSize - gzipSuffixLength
+
+	ConfigMapEncodingAnnotationKey  = "olm.contentEncoding"
+	ConfigMapEncodingAnnotationGzip = "gzip+base64"
+)
+
+/*
+This file implements the actual building of the CM list. It uses the template method design pattern to implement both
+regular string VM, and compressed binary CM.
+
+The method itself is FBCRegistryPod.getConfigMaps. This file contains the actual implementation of the writing actions,
+used by the method.
+*/
+
+type configMapWriter interface {
+	reset()
+	newConfigMap(string) *corev1.ConfigMap
+	getFilePath() string
+	isEmpty() bool
+	exceedMaxLength(cmSize int, data string) (bool, error)
+	closeCM(cm *corev1.ConfigMap) error
+	addData(data string) error
+	continueAddData(data string) error
+	writeLastFragment(cm *corev1.ConfigMap) error
+}
+
+type defaultCMWriter struct {
+	cmName      string
+	namespace   string
+	dataBuilder strings.Builder
+}
+
+func newDefaultWriter(name, namespace string) *defaultCMWriter {
+	return &defaultCMWriter{
+		cmName:      name,
+		namespace:   namespace,
+		dataBuilder: strings.Builder{},
+	}
+}
+
+func (cmw *defaultCMWriter) reset() {
+	cmw.dataBuilder = strings.Builder{}
+}
+
+func (cmw *defaultCMWriter) newConfigMap(name string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: cmw.namespace,
+		},
+		Data: map[string]string{},
+	}
+}
+
+func (cmw *defaultCMWriter) isEmpty() bool {
+	return cmw.dataBuilder.Len() > 0
+}
+
+func (cmw *defaultCMWriter) exceedMaxLength(cmSize int, data string) (bool, error) {
+	return cmSize+cmw.dataBuilder.Len()+len(data) > maxConfigMapSize, nil
+}
+
+func (cmw *defaultCMWriter) closeCM(cm *corev1.ConfigMap) error {
+	cm.Data[defaultConfigMapKey] = cmw.dataBuilder.String()
+	cmw.dataBuilder = strings.Builder{}
+	return nil
+}
+
+func (cmw *defaultCMWriter) getFilePath() string {
+	return fmt.Sprintf("%s.yaml", defaultConfigMapKey)
+}
+
+func (cmw *defaultCMWriter) addData(data string) error {
+	cmw.dataBuilder.WriteString(data)
+	return nil
+}
+
+// continueAddData completes adding the data after starting adding it in exceedMaxLength
+func (cmw *defaultCMWriter) continueAddData(data string) error {
+	return cmw.addData(data)
+}
+
+func (cmw *defaultCMWriter) writeLastFragment(cm *corev1.ConfigMap) error {
+	cm.Data[defaultConfigMapKey] = cmw.dataBuilder.String()
+
+	return nil
+}
+
+type gzipCMWriter struct {
+	actualBuff   *bytes.Buffer
+	helperBuff   *bytes.Buffer
+	actualWriter *gzip.Writer
+	helperWriter *gzip.Writer
+	cmName       string
+	namespace    string
+}
+
+func newGZIPWriter(name, namespace string) *gzipCMWriter {
+	actualBuff := &bytes.Buffer{}
+	helperBuff := &bytes.Buffer{}
+
+	return &gzipCMWriter{
+		actualBuff:   actualBuff,
+		helperBuff:   helperBuff,
+		actualWriter: gzip.NewWriter(actualBuff),
+		helperWriter: gzip.NewWriter(helperBuff),
+		cmName:       name,
+		namespace:    namespace,
+	}
+}
+
+func (cmw *gzipCMWriter) reset() {
+	cmw.actualBuff.Reset()
+	cmw.actualWriter.Reset(cmw.actualBuff)
+	cmw.helperBuff.Reset()
+	cmw.helperWriter.Reset(cmw.helperBuff)
+}
+
+func (cmw *gzipCMWriter) newConfigMap(name string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: cmw.namespace,
+			Name:      name,
+			Annotations: map[string]string{
+				ConfigMapEncodingAnnotationKey: ConfigMapEncodingAnnotationGzip,
+			},
+		},
+		BinaryData: map[string][]byte{},
+	}
+}
+
+func (cmw *gzipCMWriter) getFilePath() string {
+	return fmt.Sprintf("%s.yaml.gz", defaultConfigMapKey)
+}
+
+func (cmw *gzipCMWriter) isEmpty() bool {
+	return cmw.actualBuff.Len() > 0
+}
+
+func (cmw *gzipCMWriter) exceedMaxLength(cmSize int, data string) (bool, error) {
+	_, err := cmw.helperWriter.Write([]byte(data))
+	if err != nil {
+		return false, err
+	}
+
+	err = cmw.helperWriter.Flush()
+	if err != nil {
+		return false, err
+	}
+
+	return cmSize+cmw.helperBuff.Len() > maxGZIPLength, nil
+}
+
+func (cmw *gzipCMWriter) closeCM(cm *corev1.ConfigMap) error {
+	err := cmw.actualWriter.Close()
+	if err != nil {
+		return err
+	}
+
+	err = cmw.actualWriter.Flush()
+	if err != nil {
+		return err
+	}
+
+	cm.BinaryData[defaultConfigMapKey] = make([]byte, cmw.actualBuff.Len())
+	copy(cm.BinaryData[defaultConfigMapKey], cmw.actualBuff.Bytes())
+
+	cmw.reset()
+
+	return nil
+}
+
+func (cmw *gzipCMWriter) addData(data string) error {
+	dataBytes := []byte(data)
+	_, err := cmw.helperWriter.Write(dataBytes)
+	if err != nil {
+		return err
+	}
+	_, err = cmw.actualWriter.Write(dataBytes)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// continueAddData completes adding the data after starting adding it in exceedMaxLength
+func (cmw *gzipCMWriter) continueAddData(data string) error {
+	_, err := cmw.actualWriter.Write([]byte(data))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cmw *gzipCMWriter) writeLastFragment(cm *corev1.ConfigMap) error {
+	err := cmw.actualWriter.Close()
+	if err != nil {
+		return err
+	}
+
+	cm.BinaryData[defaultConfigMapKey] = cmw.actualBuff.Bytes()
+	return nil
+}