Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions changelog/fragments/helm-operator-rollback.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# entries is a list of entries to include in
# release notes and/or the migration guide
entries:
- description: >
For Helm-based operators, whenever the operator encounters an
error during reconcilliation, it would attempt to rollback the
changes with the `--force` option. This behavior could have
undesired side effects in some scenario.

This change allows the users to change this behavior by adding the
annotation, `helm.sdk.operatorframework.io/rollback-force: false`
to the custom resource.

# kind is one of:
# - addition
# - change
# - deprecation
# - removal
# - bugfix
kind: "addition"

# Is this a breaking change?
breaking: false

# NOTE: ONLY USE `pull_request_override` WHEN ADDING THIS
# FILE FOR A PREVIOUSLY MERGED PULL_REQUEST!
#
# The generator auto-detects the PR number from the commit
# message in which this file was originally added.
#
# What is the pull request number (without the "#")?
# pull_request_override: 0

24 changes: 23 additions & 1 deletion internal/helm/controller/reconcile.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ const (
uninstallFinalizerLegacy = "uninstall-helm-release"

helmUpgradeForceAnnotation = "helm.sdk.operatorframework.io/upgrade-force"
helmRollbackForceAnnotation = "helm.sdk.operatorframework.io/rollback-force"
helmUninstallWaitAnnotation = "helm.sdk.operatorframework.io/uninstall-wait"
helmReconcilePeriodAnnotation = "helm.sdk.operatorframework.io/reconcile-period"
)
Expand Down Expand Up @@ -311,7 +312,28 @@ func (r HelmOperatorReconciler) Reconcile(ctx context.Context, request reconcile
"Chart value %q overridden to %q by operator's watches.yaml", k, v)
}
force := hasAnnotation(helmUpgradeForceAnnotation, o)
previousRelease, upgradedRelease, err := manager.UpgradeRelease(ctx, release.ForceUpgrade(force))
upgradeOptions := []release.UpgradeOption{
release.ForceUpgrade(force),
}

// this function allows the user to set the value for the annotation,
// "helm.sdk.operatorframework.io/rollback-force" to false. However,
// it does keep the default set to true
forceRollback := func() bool {
val, ok := o.GetAnnotations()[helmRollbackForceAnnotation]
if !ok {
return true
}
r, err := strconv.ParseBool(val)
if err != nil {
return true
}

return r
}()
upgradeOptions = append(upgradeOptions, release.ForceRollback(forceRollback))

previousRelease, upgradedRelease, err := manager.UpgradeRelease(ctx, upgradeOptions...)
if err != nil {
log.Error(err, "Release failed")
status.SetCondition(types.HelmAppCondition{
Expand Down
32 changes: 24 additions & 8 deletions internal/helm/release/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ type manager struct {
}

type InstallOption func(*action.Install) error
type UpgradeOption func(*action.Upgrade) error
type UpgradeOption func(UpgradeConfig) error
type UninstallOption func(*action.Uninstall) error

// ReleaseName returns the name of the release.
Expand Down Expand Up @@ -196,28 +196,44 @@ func (m manager) InstallRelease(ctx context.Context, opts ...InstallOption) (*rp
}

func ForceUpgrade(force bool) UpgradeOption {
return func(u *action.Upgrade) error {
u.Force = force
return func(u UpgradeConfig) error {
u.upgrade.Force = force
return nil
}
}

func ForceRollback(rollback bool) UpgradeOption {
return func(u UpgradeConfig) error {
u.rollback = rollback
return nil
}
}

type UpgradeConfig struct {
upgrade *action.Upgrade
rollback bool
}

// UpgradeRelease performs a Helm release upgrade.
func (m manager) UpgradeRelease(ctx context.Context, opts ...UpgradeOption) (*rpb.Release, *rpb.Release, error) {
upgrade := action.NewUpgrade(m.actionConfig)
upgrade.Namespace = m.namespace
config := UpgradeConfig{
upgrade: action.NewUpgrade(m.actionConfig),
}
// upgrade := action.NewUpgrade(m.actionConfig)
config.upgrade.Namespace = m.namespace

for _, o := range opts {
if err := o(upgrade); err != nil {
if err := o(config); err != nil {
return nil, nil, fmt.Errorf("failed to apply upgrade option: %w", err)
}
}

upgradedRelease, err := upgrade.Run(m.releaseName, m.chart, m.values)
upgradedRelease, err := config.upgrade.Run(m.releaseName, m.chart, m.values)
if err != nil {
// Workaround for helm/helm#3338
if upgradedRelease != nil {
rollback := action.NewRollback(m.actionConfig)
rollback.Force = true
rollback.Force = config.rollback

// As of Helm 2.13, if UpgradeRelease returns a non-nil release, that
// means the release was also recorded in the release store.
Expand Down
2 changes: 1 addition & 1 deletion testdata/ansible/memcached-operator/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ endif

# Set the Operator SDK version to use. By default, what is installed on the system is used.
# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
OPERATOR_SDK_VERSION ?= v1.30.0
OPERATOR_SDK_VERSION ?= v1.31.0

# Image URL to use all building/pushing image targets
IMG ?= controller:latest
Expand Down
2 changes: 1 addition & 1 deletion testdata/go/v3/memcached-operator/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ endif

# Set the Operator SDK version to use. By default, what is installed on the system is used.
# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
OPERATOR_SDK_VERSION ?= v1.30.0
OPERATOR_SDK_VERSION ?= v1.31.0

# Image URL to use all building/pushing image targets
IMG ?= controller:latest
Expand Down
2 changes: 1 addition & 1 deletion testdata/go/v3/monitoring/memcached-operator/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ endif

# Set the Operator SDK version to use. By default, what is installed on the system is used.
# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
OPERATOR_SDK_VERSION ?= v1.30.0
OPERATOR_SDK_VERSION ?= v1.31.0

# Image URL to use all building/pushing image targets
IMG ?= controller:latest
Expand Down
2 changes: 1 addition & 1 deletion testdata/go/v4-alpha/memcached-operator/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ endif

# Set the Operator SDK version to use. By default, what is installed on the system is used.
# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
OPERATOR_SDK_VERSION ?= v1.30.0
OPERATOR_SDK_VERSION ?= v1.31.0

# Image URL to use all building/pushing image targets
IMG ?= controller:latest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ endif

# Set the Operator SDK version to use. By default, what is installed on the system is used.
# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
OPERATOR_SDK_VERSION ?= v1.30.0
OPERATOR_SDK_VERSION ?= v1.31.0

# Image URL to use all building/pushing image targets
IMG ?= controller:latest
Expand Down
2 changes: 1 addition & 1 deletion testdata/helm/memcached-operator/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ endif

# Set the Operator SDK version to use. By default, what is installed on the system is used.
# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
OPERATOR_SDK_VERSION ?= v1.30.0
OPERATOR_SDK_VERSION ?= v1.31.0

# Image URL to use all building/pushing image targets
IMG ?= controller:latest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,19 @@ metadata:
The value that is present under this key must be in the h/m/s format. For example, 1h2m4s, 3m0s, 4s are all valid values, but 1x3m9s is invalid.

**NOTE**: This is just one way of specifying the reconcile period for Helm-based operators. There are two other ways: using the `--reconcile-period` command-line flag and under the 'reconcilePeriod' key in the watches.yaml file. If these three methods are used simultaneously to specify reconcile period (which they should not be), the order of precedence is as follows:
Custom Resource Annotations > watches.yaml > command-line flag.
Custom Resource Annotations > watches.yaml > command-line flag.

## `helm.sdk.operatorframework.io/rollback-force`

Whenever a helm-based operator encounters an error during reconcilliation, by default, it would attempt to perform a rollback with the `--force` option. While this works as expected in most scenarios, there are a few edge cases where performing a rollback with `--force` could have undesired side effects.

```sh
...
metadata:
name: nginx-sample
annotations:
helm.sdk.operatorframework.io/rollback-force: false
...
```

Adding annotation to the custom resource, `helm.sdk.operatorframework.io/rollback-force: false` therefore allows a user, to change the default behavior of the helm-based operator whereby, rollbacks will be performed without the `--force` option whenever an error is encountered.