Fix 6323: allow compression of the configMap

Add the `--gzip-configmap=true` CLI parameter in order to compress the configMap, if it exeeds the max length.

Signed-off-by: Nahshon Unna-Tsameret <[email protected]>

Author: nunnatsa

internal/olm/operator/config.go

@@ -37,6 +37,7 @@ type Configuration struct {
 	Client         client.Client
 	Scheme         *runtime.Scheme
 	Timeout        time.Duration
+	GzipCM         bool
 
 	overrides *clientcmd.ConfigOverrides
 }
@@ -62,6 +63,7 @@ func (c *Configuration) BindFlags(fs *pflag.FlagSet) {
 			"This value does not override the operator's service account")
 	fs.DurationVar(&c.Timeout, "timeout", 2*time.Minute,
 		"Duration to wait for the command to complete before failing")
+	fs.BoolVar(&c.GzipCM, "gzip-configmap", false, "If 'true', the configmap data will be compressed")
 }
 
 func (c *Configuration) Load() error {

internal/olm/operator/registry/fbcindex/configMapWriter.go

@@ -0,0 +1,282 @@
+package fbcindex
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"strings"
+	"text/template"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	yamlSeparator    = "\n---\n"
+	gzipSuffixLength = 13
+	maxGZIPLength    = maxConfigMapSize - gzipSuffixLength
+
+	ConfigMapEncodingAnnotationKey  = "olm.contentEncoding"
+	ConfigMapEncodingAnnotationGzip = "gzip+base64"
+)
+
+type configMapWriter interface {
+	newConfigMap() *corev1.ConfigMap
+	writeYaml([]string) ([]*corev1.ConfigMap, error)
+	getFilePath() string
+	getCommandTemplate() *template.Template
+}
+
+type defaultCMWriter struct {
+	cmName    string
+	namespace string
+	template  *template.Template
+}
+
+func newDefaultWriter(name, namespace string) *defaultCMWriter {
+	t := template.Must(template.New("cmd").Parse(fbcGzipCmdTemplate))
+	return &defaultCMWriter{
+		cmName:    name,
+		namespace: namespace,
+		template:  t,
+	}
+}
+
+// container entrypoint command for FBC type images.
+const fbcCmdTemplate = `opm serve {{ .FBCIndexRootDir}} -p {{ .GRPCPort }}`
+
+func (cmw defaultCMWriter) newConfigMap() *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: cmw.namespace,
+		},
+		Data: map[string]string{},
+	}
+}
+
+func (cmw defaultCMWriter) writeYaml(yamlDefs []string) ([]*corev1.ConfigMap, error) {
+	cm := cmw.newConfigMap()
+	cm.SetName(fmt.Sprintf("%s-partition-1", cmw.cmName))
+	configMaps := []*corev1.ConfigMap{cm}
+
+	partitionCount := 1
+
+	// for each chunk of yaml see if it can be added to the ConfigMap partition
+	for _, yamlDef := range yamlDefs {
+		yamlDef = strings.TrimSpace(yamlDef)
+		if len(strings.TrimSpace(yamlDef)) == 0 {
+			continue
+		}
+		// If the ConfigMap has data then lets attempt to add to it
+		if len(cm.Data) != 0 {
+			// Create a copy to use to verify that adding the data doesn't
+			// exceed the max ConfigMap size of 1 MiB.
+			tempCm := cm.DeepCopy()
+			tempCm.Data[defaultConfigMapKey] = tempCm.Data[defaultConfigMapKey] + yamlSeparator + yamlDef
+
+			// if it would be too large adding the data then partition it.
+			if tempCm.Size() >= maxConfigMapSize {
+				// Create a new ConfigMap
+				cm = cmw.newConfigMap()
+
+				// Set the ConfigMap name based on the partition it is
+				cm.SetName(fmt.Sprintf("%s-partition-%d", cmw.cmName, partitionCount+1))
+
+				// Since adding this data would have made the previous
+				// ConfigMap too large, add it to this new one.
+				// No chunk of YAML from the bundle should cause
+				// the ConfigMap size to exceed 1 MiB and if
+				// somehow it does then there is a problem with the
+				// YAML itself. We can't reasonably break it up smaller
+				// since it is a single object.
+				cm.Data[defaultConfigMapKey] = yamlDef
+
+				// Add the ConfigMap to the list of ConfigMaps
+				configMaps = append(configMaps, cm)
+
+			} else {
+				// if adding the data to the ConfigMap
+				// doesn't make the ConfigMap exceed the
+				// size limit then actually add it.
+				cm.Data = tempCm.Data
+			}
+		} else {
+			// If there is no data in the ConfigMap
+			// then this is the first pass. Since it is
+			// the first pass go ahead and add the data.
+			cm.Data[defaultConfigMapKey] = yamlDef
+		}
+	}
+
+	return configMaps, nil
+}
+
+func (cmw defaultCMWriter) getFilePath() string {
+	return fmt.Sprintf("%s.yaml", defaultConfigMapKey)
+}
+
+func (cmw defaultCMWriter) getCommandTemplate() *template.Template {
+	return cmw.template
+}
+
+type gzipCMWriter struct {
+	actualBuff   *bytes.Buffer
+	helperBuff   *bytes.Buffer
+	actualWriter *gzip.Writer
+	helperWriter *gzip.Writer
+	cmName       string
+	namespace    string
+	template     *template.Template
+}
+
+func newGZIPWriter(name, namespace string) *gzipCMWriter {
+	actualBuff := &bytes.Buffer{}
+	helperBuff := &bytes.Buffer{}
+	t := template.Must(template.New("cmd").Parse(fbcGzipCmdTemplate))
+
+	return &gzipCMWriter{
+		actualBuff:   actualBuff,
+		helperBuff:   helperBuff,
+		actualWriter: gzip.NewWriter(actualBuff),
+		helperWriter: gzip.NewWriter(helperBuff),
+		cmName:       name,
+		namespace:    namespace,
+		template:     t,
+	}
+}
+
+// container entrypoint command for gzipped FBC type images.
+const fbcGzipCmdTemplate = `` +
+	`serverDir=/var/tmp/{{ .FBCIndexRootDir }}/;` +
+	`mkdir ${serverDir};` +
+	`for dir in {{ .FBCIndexRootDir }}/*configmap-partition-*;` +
+	`do targetDir="/var/tmp/${dir}";` +
+	`mkdir ${targetDir};` +
+	`for f in ${dir}/*gz; ` +
+	`do cat $f | gzip -d -c > "/var/tmp/${f%.*}";` +
+	`done;` +
+	`done;` +
+	`opm serve ${serverDir} -p {{ .GRPCPort }}`
+
+func (cmw *gzipCMWriter) reset() {
+	cmw.actualBuff.Reset()
+	cmw.actualWriter.Reset(cmw.actualBuff)
+	cmw.helperBuff.Reset()
+	cmw.helperWriter.Reset(cmw.helperBuff)
+}
+
+func (cmw *gzipCMWriter) newConfigMap() *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: cmw.namespace,
+			Annotations: map[string]string{
+				ConfigMapEncodingAnnotationKey: ConfigMapEncodingAnnotationGzip,
+			},
+		},
+		BinaryData: map[string][]byte{},
+	}
+}
+
+func (cmw *gzipCMWriter) writeYaml(yamlDefs []string) ([]*corev1.ConfigMap, error) {
+	defer cmw.reset()
+
+	cm := cmw.newConfigMap()
+	cm.SetName(fmt.Sprintf("%s-partition-1", cmw.cmName))
+	configMaps := []*corev1.ConfigMap{cm}
+
+	partitionCount := 1
+
+	// for each chunk of yaml see if it can be added to the ConfigMap partition
+	for _, yamlDef := range yamlDefs {
+		yamlDef = strings.TrimSpace(yamlDef)
+		if len(yamlDef) == 0 {
+			continue
+		}
+
+		if cmw.actualBuff.Len() > 0 {
+			data := []byte(yamlSeparator + yamlDef)
+			_, err := cmw.helperWriter.Write(data)
+			if err != nil {
+				return nil, err
+			}
+
+			err = cmw.helperWriter.Flush()
+			if err != nil {
+				return nil, err
+			}
+
+			if cm.Size()+cmw.helperBuff.Len() > maxGZIPLength {
+				err = cmw.actualWriter.Close()
+				if err != nil {
+					return nil, err
+				}
+
+				err = cmw.actualWriter.Flush()
+				if err != nil {
+					return nil, err
+				}
+
+				cm.BinaryData[defaultConfigMapKey] = make([]byte, cmw.actualBuff.Len())
+				copy(cm.BinaryData[defaultConfigMapKey], cmw.actualBuff.Bytes())
+
+				cmw.reset()
+
+				partitionCount++
+				cm = cmw.newConfigMap()
+				cm.SetName(fmt.Sprintf("%s-partition-%d", cmw.cmName, partitionCount))
+				configMaps = append(configMaps, cm)
+
+				data = []byte(yamlDef)
+				_, err = cmw.helperWriter.Write(data)
+				if err != nil {
+					return nil, err
+				}
+				_, err = cmw.actualWriter.Write(data)
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				_, err = cmw.actualWriter.Write(data)
+				if err != nil {
+					return nil, err
+				}
+			}
+		} else {
+			data := []byte(yamlDef)
+			_, err := cmw.helperWriter.Write(data)
+			if err != nil {
+				return nil, err
+			}
+			_, err = cmw.actualWriter.Write(data)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// write the data of the last cm
+	err := cmw.actualWriter.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	cm.BinaryData[defaultConfigMapKey] = cmw.actualBuff.Bytes()
+
+	return configMaps, nil
+}
+
+func (cmw *gzipCMWriter) getFilePath() string {
+	return fmt.Sprintf("%s.yaml.gz", defaultConfigMapKey)
+}
+
+func (cmw *gzipCMWriter) getCommandTemplate() *template.Template {
+	return cmw.template
+}