feat(lambda): configurable retries for log retention custom resource (#8258)

This prevents throttling issues on stacks with a lot of Lambdas.

fixes #8257

Implemented configurable `maxRetries` and `base` properties as part of the LogRetentionRetryOptions. The AWS SDK also supports specifying a customBackoff function. I skipped that as it's hard to implement in the current setup (impossible to provide a callback function in the event JSON).

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: jaapvanblaaderen

packages/@aws-cdk/aws-lambda/README.md

@@ -266,6 +266,9 @@ The `logRetention` property can be used to set a different expiration period.
 It is possible to obtain the function's log group as a `logs.ILogGroup` by calling the `logGroup` property of the
 `Function` construct.
 
+By default, CDK uses the AWS SDK retry options when creating a log group. The `logRetentionRetryOptions` property 
+allows you to customize the maximum number of retries and base backoff duration.
+
 *Note* that, if either `logRetention` is set or `logGroup` property is called, a [CloudFormation custom
 resource](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cfn-customresource.html) is added
 to the stack that pre-creates the log group as part of the stack deployment, if it already doesn't exist, and sets the

packages/@aws-cdk/aws-lambda/lib/function.ts

@@ -12,7 +12,7 @@ import { calculateFunctionHash, trimFromStart } from './function-hash';
 import { Version, VersionOptions } from './lambda-version';
 import { CfnFunction } from './lambda.generated';
 import { ILayerVersion } from './layers';
-import { LogRetention } from './log-retention';
+import { LogRetention, LogRetentionRetryOptions } from './log-retention';
 import { Runtime } from './runtime';
 
 /**
@@ -232,6 +232,14 @@ export interface FunctionOptions extends EventInvokeConfigOptions {
    */
   readonly logRetentionRole?: iam.IRole;
 
+  /**
+   * When log retention is specified, a custom resource attempts to create the CloudWatch log group.
+   * These options control the retry policy when interacting with CloudWatch APIs.
+   *
+   * @default - Default AWS SDK retry options.
+   */
+  readonly logRetentionRetryOptions?: LogRetentionRetryOptions;
+
   /**
    * Options for the `lambda.Version` resource automatically created by the
    * `fn.currentVersion` method.
@@ -544,6 +552,7 @@ export class Function extends FunctionBase {
         logGroupName: `/aws/lambda/${this.functionName}`,
         retention: props.logRetention,
         role: props.logRetentionRole,
+        logRetentionRetryOptions: props.logRetentionRetryOptions,
       });
       this._logGroup = logs.LogGroup.fromLogGroupArn(this, 'LogGroup', logretention.logGroupArn);
     }

packages/@aws-cdk/aws-lambda/lib/log-retention-provider/index.ts

@@ -2,15 +2,23 @@
 
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as AWS from 'aws-sdk';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { RetryDelayOptions } from 'aws-sdk/lib/config';
+
+interface SdkRetryOptions {
+  maxRetries?: number;
+  retryOptions?: RetryDelayOptions;
+}
 
 /**
  * Creates a log group and doesn't throw if it exists.
  *
- * @param logGroupName the name of the log group to create
+ * @param logGroupName the name of the log group to create.
+ * @param options CloudWatch API SDK options.
  */
-async function createLogGroupSafe(logGroupName: string) {
+async function createLogGroupSafe(logGroupName: string, options?: SdkRetryOptions) {
   try { // Try to create the log group
-    const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28' });
+    const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...options });
     await cloudwatchlogs.createLogGroup({ logGroupName }).promise();
   } catch (e) {
     if (e.code !== 'ResourceAlreadyExistsException') {
@@ -23,10 +31,11 @@ async function createLogGroupSafe(logGroupName: string) {
  * Puts or deletes a retention policy on a log group.
  *
  * @param logGroupName the name of the log group to create
+ * @param options CloudWatch API SDK options.
  * @param retentionInDays the number of days to retain the log events in the specified log group.
  */
-async function setRetentionPolicy(logGroupName: string, retentionInDays?: number) {
-  const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28' });
+async function setRetentionPolicy(logGroupName: string, options?: SdkRetryOptions, retentionInDays?: number) {
+  const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...options });
   if (!retentionInDays) {
     await cloudwatchlogs.deleteRetentionPolicy({ logGroupName }).promise();
   } else {
@@ -41,10 +50,13 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
     // The target log group
     const logGroupName = event.ResourceProperties.LogGroupName;
 
+    // Parse to AWS SDK retry options
+    const retryOptions = parseRetryOptions(event.ResourceProperties.SdkRetry);
+
     if (event.RequestType === 'Create' || event.RequestType === 'Update') {
       // Act on the target log group
-      await createLogGroupSafe(logGroupName);
-      await setRetentionPolicy(logGroupName, parseInt(event.ResourceProperties.RetentionInDays, 10));
+      await createLogGroupSafe(logGroupName, retryOptions);
+      await setRetentionPolicy(logGroupName, retryOptions, parseInt(event.ResourceProperties.RetentionInDays, 10));
 
       if (event.RequestType === 'Create') {
         // Set a retention policy of 1 day on the logs of this function. The log
@@ -56,8 +68,8 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
         // same time. This can sometime result in an OperationAbortedException. To
         // avoid this and because this operation is not critical we catch all errors.
         try {
-          await createLogGroupSafe(`/aws/lambda/${context.functionName}`);
-          await setRetentionPolicy(`/aws/lambda/${context.functionName}`, 1);
+          await createLogGroupSafe(`/aws/lambda/${context.functionName}`, retryOptions);
+          await setRetentionPolicy(`/aws/lambda/${context.functionName}`, retryOptions, 1);
         } catch (e) {
           console.log(e);
         }
@@ -108,4 +120,19 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
       }
     });
   }
+
+  function parseRetryOptions(rawOptions: any): SdkRetryOptions {
+    const retryOptions: SdkRetryOptions = {};
+    if (rawOptions) {
+      if (rawOptions.maxRetries) {
+        retryOptions.maxRetries = parseInt(rawOptions.maxRetries, 10);
+      }
+      if (rawOptions.base) {
+        retryOptions.retryOptions = {
+          base: parseInt(rawOptions.base, 10),
+        };
+      }
+    }
+    return retryOptions;
+  }
 }

packages/@aws-cdk/aws-lambda/lib/log-retention.ts

@@ -26,6 +26,31 @@ export interface LogRetentionProps {
    * @default - A new role is created
    */
   readonly role?: iam.IRole;
+
+  /**
+   * Retry options for all AWS API calls.
+   *
+   * @default - AWS SDK default retry options
+   */
+  readonly logRetentionRetryOptions?: LogRetentionRetryOptions;
+}
+
+/**
+ * Retry options for all AWS API calls.
+ */
+export interface LogRetentionRetryOptions {
+  /**
+   * The maximum amount of retries.
+   *
+   * @default 3 (AWS SDK default)
+   */
+  readonly maxRetries?: number;
+  /**
+   * The base duration to use in the exponential backoff for operation retries.
+   *
+   * @default Duration.millis(100) (AWS SDK default)
+   */
+  readonly base?: cdk.Duration;
 }
 
 /**
@@ -64,11 +89,16 @@ export class LogRetention extends cdk.Construct {
 
     // Need to use a CfnResource here to prevent lerna dependency cycles
     // @aws-cdk/aws-cloudformation -> @aws-cdk/aws-lambda -> @aws-cdk/aws-cloudformation
+    const retryOptions = props.logRetentionRetryOptions;
     const resource = new cdk.CfnResource(this, 'Resource', {
       type: 'Custom::LogRetention',
       properties: {
         ServiceToken: provider.functionArn,
         LogGroupName: props.logGroupName,
+        SdkRetry: retryOptions ? {
+          maxRetries: retryOptions.maxRetries,
+          base: retryOptions.base?.toMilliseconds(),
+        } : undefined,
         RetentionInDays: props.retention === logs.RetentionDays.INFINITE ? undefined : props.retention,
       },
     });

packages/@aws-cdk/aws-lambda/test/integ.log-retention.expected.json

@@ -133,7 +133,7 @@
       "Properties": {
         "Code": {
           "S3Bucket": {
-            "Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE"
+            "Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049"
           },
           "S3Key": {
             "Fn::Join": [
@@ -146,7 +146,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
+                          "Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
                         }
                       ]
                     }
@@ -159,7 +159,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
+                          "Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
                         }
                       ]
                     }
@@ -331,17 +331,17 @@
     }
   },
   "Parameters": {
-    "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE": {
+    "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
+      "Description": "S3 bucket for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
     },
-    "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583": {
+    "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1": {
       "Type": "String",
-      "Description": "S3 key for asset version \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
+      "Description": "S3 key for asset version \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
     },
-    "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eArtifactHashB967D42A": {
+    "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3ArtifactHash31AA1F7C": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
+      "Description": "Artifact hash for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
     }
   }
 }

packages/@aws-cdk/aws-lambda/test/test.log-retention-provider.ts

@@ -296,4 +296,41 @@ export = {
 
     test.done();
   },
+
+  async 'custom log retention retry options'(test: Test) {
+    AWS.mock('CloudWatchLogs', 'createLogGroup', sinon.fake.resolves({}));
+    AWS.mock('CloudWatchLogs', 'putRetentionPolicy', sinon.fake.resolves({}));
+    AWS.mock('CloudWatchLogs', 'deleteRetentionPolicy', sinon.fake.resolves({}));
+
+    const event = {
+      ...eventCommon,
+      RequestType: 'Create',
+      ResourceProperties: {
+        ServiceToken: 'token',
+        RetentionInDays: '30',
+        LogGroupName: 'group',
+        SdkRetry: {
+          maxRetries: '5',
+          base: '300',
+        },
+      },
+    };
+
+    const request = createRequest('SUCCESS');
+
+    await provider.handler(event as AWSLambda.CloudFormationCustomResourceCreateEvent, context);
+
+    sinon.assert.calledWith(AWSSDK.CloudWatchLogs as any, {
+      apiVersion: '2014-03-28',
+      maxRetries: 5,
+      retryOptions: {
+        base: 300,
+      },
+    });
+
+    test.equal(request.isDone(), true);
+
+    test.done();
+  },
+
 };

packages/@aws-cdk/aws-rds/test/integ.instance.lit.expected.json

@@ -967,7 +967,7 @@
       "Properties": {
         "Code": {
           "S3Bucket": {
-            "Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE"
+            "Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049"
           },
           "S3Key": {
             "Fn::Join": [
@@ -980,7 +980,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
+                          "Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
                         }
                       ]
                     }
@@ -993,7 +993,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
+                          "Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
                         }
                       ]
                     }
@@ -1108,17 +1108,17 @@
     }
   },
   "Parameters": {
-    "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE": {
+    "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
+      "Description": "S3 bucket for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
     },
-    "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583": {
+    "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1": {
       "Type": "String",
-      "Description": "S3 key for asset version \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
+      "Description": "S3 key for asset version \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
     },
-    "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eArtifactHashB967D42A": {
+    "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3ArtifactHash31AA1F7C": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
+      "Description": "Artifact hash for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
     }
   }
-}
+}