From 8401fb4f7f88e11d4de608207b6e59de973f069f Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Wed, 14 Jul 2021 10:04:26 -0700 Subject: [PATCH 01/52] feat(aws-kinesisfirehose): DeliveryStream API and basic S3 destination --- packages/@aws-cdk/aws-ec2/lib/connections.ts | 3 + .../.eslintrc.js | 3 + .../.gitignore | 19 + .../.npmignore | 28 ++ .../aws-kinesisfirehose-destinations/LICENSE | 201 +++++++++ .../aws-kinesisfirehose-destinations/NOTICE | 2 + .../README.md | 22 + .../jest.config.js | 2 + .../lib/index.ts | 1 + .../lib/s3.ts | 39 ++ .../package.json | 112 +++++ .../rosetta/default.ts-fixture | 11 + .../integ.s3-all-properties.expected.json | 392 ++++++++++++++++++ .../test/integ.s3-all-properties.ts | 28 ++ .../test/integ.s3-basic.expected.json | 392 ++++++++++++++++++ .../test/integ.s3-basic.ts | 20 + .../test/s3-destination.test.ts | 116 ++++++ .../@aws-cdk/aws-kinesisfirehose/README.md | 210 +++++++++- .../lib/delivery-stream.ts | 256 ++++++++++++ .../aws-kinesisfirehose/lib/destination.ts | 99 +++++ .../@aws-cdk/aws-kinesisfirehose/lib/index.ts | 3 + .../@aws-cdk/aws-kinesisfirehose/package.json | 30 +- .../rosetta/default.ts-fixture | 11 + .../test/delivery-stream.test.ts | 281 +++++++++++++ .../test/destination.test.ts | 158 +++++++ .../test/integ.delivery-stream.expected.json | 145 +++++++ .../test/integ.delivery-stream.ts | 37 ++ .../test/kinesisfirehose.test.ts | 6 - .../region-info/build-tools/fact-tables.ts | 29 ++ .../build-tools/generate-static-data.ts | 10 +- packages/@aws-cdk/region-info/lib/fact.ts | 5 + .../@aws-cdk/region-info/lib/region-info.ts | 7 + packages/aws-cdk-lib/package.json | 1 + packages/decdk/package.json | 1 + packages/monocdk/package.json | 1 + tools/pkglint/lib/rules.ts | 1 + 36 files changed, 2671 insertions(+), 11 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.expected.json create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-destination.test.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts delete mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts diff --git a/packages/@aws-cdk/aws-ec2/lib/connections.ts b/packages/@aws-cdk/aws-ec2/lib/connections.ts index 0ecccea97fdb2..b68c04299cdf3 100644 --- a/packages/@aws-cdk/aws-ec2/lib/connections.ts +++ b/packages/@aws-cdk/aws-ec2/lib/connections.ts @@ -20,6 +20,9 @@ import { ISecurityGroup } from './security-group'; * An object that has a Connections object */ export interface IConnectable { + /** + * The network connections associated with this resource. + */ readonly connections: Connections; } diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js new file mode 100644 index 0000000000000..61dd8dd001f63 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js @@ -0,0 +1,3 @@ +const baseConfig = require('cdk-build-tools/config/eslintrc'); +baseConfig.parserOptions.project = __dirname + '/tsconfig.json'; +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore new file mode 100644 index 0000000000000..147448f7df4fe --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore @@ -0,0 +1,19 @@ +*.js +tsconfig.json +*.js.map +*.d.ts +*.generated.ts +dist +lib/generated/resources.ts +.jsii + +.LAST_BUILD +.nyc_output +coverage +nyc.config.js +.LAST_PACKAGE +*.snk +!.eslintrc.js +!jest.config.js + +junit.xml \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore new file mode 100644 index 0000000000000..aaabf1df59065 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore @@ -0,0 +1,28 @@ +# Don't include original .ts files when doing `npm pack` +*.ts +!*.d.ts +coverage +.nyc_output +*.tgz + +dist +.LAST_PACKAGE +.LAST_BUILD +!*.js + +# Include .jsii +!.jsii + +*.snk + +*.tsbuildinfo + +tsconfig.json +.eslintrc.js +jest.config.js + +# exclude cdk artifacts +**/cdk.out +junit.xml +test/ +!*.lit.ts \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE b/packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE new file mode 100644 index 0000000000000..28e4bdcec77ec --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE b/packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE new file mode 100644 index 0000000000000..5fc3826926b5b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE @@ -0,0 +1,2 @@ +AWS Cloud Development Kit (AWS CDK) +Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md b/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md new file mode 100644 index 0000000000000..efe753ddbd6a6 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md @@ -0,0 +1,22 @@ +# Amazon Kinesis Data Firehose Destinations Library + + +--- + +![cdk-constructs: Experimental](https://img.shields.io/badge/cdk--constructs-experimental-important.svg?style=for-the-badge) + +> The APIs of higher level constructs in this module are experimental and under active development. +> They are subject to non-backward compatible changes or removal in any future version. These are +> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be +> announced in the release notes. This means that while you may use them, you may need to update +> your source code when upgrading to a newer version of this package. + +--- + + + +This library provides constructs for adding destinations to a Amazon Kinesis Data Firehose +delivery stream. Destinations can be added by specifying the `destination` prop when +defining a delivery stream. + +See [Amazon Kinesis Data Firehose module README](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisfirehose-readme.html) for usage examples. diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js b/packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js new file mode 100644 index 0000000000000..54e28beb9798b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js @@ -0,0 +1,2 @@ +const baseConfig = require('cdk-build-tools/config/jest.config'); +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts new file mode 100644 index 0000000000000..cb717f27167ea --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts @@ -0,0 +1 @@ +export * from './s3'; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3.ts new file mode 100644 index 0000000000000..42334c18dfea6 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3.ts @@ -0,0 +1,39 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import { CfnDeliveryStream } from '@aws-cdk/aws-kinesisfirehose'; +import * as s3 from '@aws-cdk/aws-s3'; +import { Construct } from 'constructs'; + +/** + * Props for defining an S3 destination of a Kinesis Data Firehose delivery stream. + */ +export interface S3Props extends firehose.DestinationProps { } + +/** + * An S3 bucket destination for data from a Kinesis Data Firehose delivery stream. + */ +export class S3 extends firehose.DestinationBase { + constructor(private readonly bucket: s3.IBucket, s3Props: S3Props = {}) { + super(s3Props); + } + + bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { + return { + properties: { + extendedS3DestinationConfiguration: this.createExtendedS3DestinationConfiguration(scope, options.deliveryStream), + }, + }; + } + + private createExtendedS3DestinationConfiguration( + scope: Construct, + deliveryStream: firehose.IDeliveryStream, + ): CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty { + this.bucket.grantReadWrite(deliveryStream); + return { + cloudWatchLoggingOptions: this.createLoggingOptions(scope, deliveryStream, 'S3Destination'), + roleArn: (deliveryStream.grantPrincipal as iam.IRole).roleArn, + bucketArn: this.bucket.bucketArn, + }; + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json new file mode 100644 index 0000000000000..0d5b22fb53352 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json @@ -0,0 +1,112 @@ +{ + "name": "@aws-cdk/aws-kinesisfirehose-destinations", + "version": "0.0.0", + "description": "CDK Destinations Constructs for AWS Kinesis Firehose", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "jsii": { + "outdir": "dist", + "targets": { + "java": { + "package": "software.amazon.awscdk.services.kinesisfirehose.destinations", + "maven": { + "groupId": "software.amazon.awscdk", + "artifactId": "kinesisfirehose-destinations" + } + }, + "dotnet": { + "namespace": "Amazon.CDK.AWS.KinesisFirehose.Destinations", + "packageId": "Amazon.CDK.AWS.KinesisFirehose.Destinations", + "iconUrl": "https://raw.githubusercontent.com/aws/aws-cdk/master/logo/default-256-dark.png" + }, + "python": { + "distName": "aws-cdk.aws-kinesisfirehose-destinations", + "module": "aws_cdk.aws_kinesisfirehose_destinations", + "classifiers": [ + "Framework :: AWS CDK", + "Framework :: AWS CDK :: 1" + ] + } + }, + "projectReferences": true + }, + "repository": { + "type": "git", + "url": "https://github.com/aws/aws-cdk.git", + "directory": "packages/@aws-cdk/aws-kinesisfirehose-destinations" + }, + "scripts": { + "build": "cdk-build", + "watch": "cdk-watch", + "lint": "cdk-lint", + "test": "cdk-test", + "integ": "cdk-integ", + "pkglint": "pkglint -f", + "package": "cdk-package", + "awslint": "cdk-awslint", + "build+test+package": "yarn build+test && yarn package", + "build+test": "yarn build && yarn test", + "compat": "cdk-compat", + "rosetta:extract": "yarn --silent jsii-rosetta extract", + "build+extract": "yarn build && yarn rosetta:extract", + "build+test+extract": "yarn build+test && yarn rosetta:extract" + }, + "keywords": [ + "aws", + "cdk", + "constructs", + "kinesisfirehose" + ], + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com", + "organization": true + }, + "license": "Apache-2.0", + "devDependencies": { + "@types/aws-lambda": "^8.10.77", + "@types/jest": "^26.0.23", + "cdk-build-tools": "0.0.0", + "cdk-integ-tools": "0.0.0", + "cfn2ts": "0.0.0", + "jest": "^26.6.3", + "pkglint": "0.0.0", + "@aws-cdk/assert-internal": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0" + }, + "dependencies": { + "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", + "@aws-cdk/core": "0.0.0", + "constructs": "^3.3.69" + }, + "homepage": "https://github.com/aws/aws-cdk", + "peerDependencies": { + "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", + "@aws-cdk/core": "0.0.0", + "constructs": "^3.3.69" + }, + "engines": { + "node": ">= 10.13.0 <13 || >=13.7.0" + }, + "stability": "experimental", + "maturity": "experimental", + "awslint": { + "exclude": [] + }, + "awscdkio": { + "announce": false + }, + "cdk-build": { + "jest": true, + "env": { + "AWSLINT_BASE_CONSTRUCT": true + } + }, + "publishConfig": { + "tag": "latest" + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture new file mode 100644 index 0000000000000..8c002eb1618f7 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture @@ -0,0 +1,11 @@ +// Fixture with packages imported, but nothing else +import { Construct } from '@aws-cdk/core'; +import { S3 } from '@aws-cdk/aws-kinesisfirehose-destinations'; + +class Fixture extends Construct { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.expected.json new file mode 100644 index 0000000000000..0d60c11048969 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.expected.json @@ -0,0 +1,392 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "BucketPolicyE9A3008A": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "Bucket83908E77" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "BucketAutoDeleteObjectsCustomResourceBAFD23C2": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "Bucket83908E77" + } + }, + "DependsOn": [ + "BucketPolicyE9A3008A" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" + } + ] + } + ] + } + ] + ] + } + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "__entrypoint__.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": "nodejs12.x", + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "Bucket83908E77" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + }, + "LogGroupF5B46931": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "LogGroupS3Destination70CE1003": { + "Type": "AWS::Logs::LogStream", + "Properties": { + "LogGroupName": { + "Ref": "LogGroupF5B46931" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "DeliveryStreamServiceRole964EEBCC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "LogGroupF5B46931", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", + "Roles": [ + { + "Ref": "DeliveryStreamServiceRole964EEBCC" + } + ] + } + }, + "DeliveryStreamF6D5572D": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamType": "DirectPut", + "ExtendedS3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "CloudWatchLoggingOptions": { + "Enabled": true, + "LogGroupName": { + "Ref": "LogGroupF5B46931" + }, + "LogStreamName": { + "Ref": "LogGroupS3Destination70CE1003" + } + }, + "RoleARN": { + "Fn::GetAtt": [ + "DeliveryStreamServiceRole964EEBCC", + "Arn" + ] + } + } + }, + "DependsOn": [ + "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", + "DeliveryStreamServiceRole964EEBCC" + ] + } + }, + "Parameters": { + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B": { + "Type": "String", + "Description": "S3 bucket for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + }, + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51": { + "Type": "String", + "Description": "S3 key for asset version \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + }, + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1ArtifactHash9ECACDFD": { + "Type": "String", + "Description": "Artifact hash for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + } + }, + "Mappings": { + "DeliveryStreamFirehoseCIDRMappingE9233479": { + "af-south-1": { + "FirehoseCidrBlock": "13.244.121.224/27" + }, + "ap-east-1": { + "FirehoseCidrBlock": "18.162.221.32/27" + }, + "ap-northeast-1": { + "FirehoseCidrBlock": "13.113.196.224/27" + }, + "ap-northeast-2": { + "FirehoseCidrBlock": "13.209.1.64/27" + }, + "ap-northeast-3": { + "FirehoseCidrBlock": "13.208.177.192/27" + }, + "ap-south-1": { + "FirehoseCidrBlock": "13.232.67.32/27" + }, + "ap-southeast-1": { + "FirehoseCidrBlock": "13.228.64.192/27" + }, + "ap-southeast-2": { + "FirehoseCidrBlock": "13.210.67.224/27" + }, + "ca-central-1": { + "FirehoseCidrBlock": "35.183.92.128/27" + }, + "cn-north-1": { + "FirehoseCidrBlock": "52.81.151.32/27" + }, + "cn-northwest-1": { + "FirehoseCidrBlock": "161.189.23.64/27" + }, + "eu-central-1": { + "FirehoseCidrBlock": "35.158.127.160/27" + }, + "eu-north-1": { + "FirehoseCidrBlock": "13.53.63.224/27" + }, + "eu-south-1": { + "FirehoseCidrBlock": "15.161.135.128/27" + }, + "eu-west-1": { + "FirehoseCidrBlock": "52.19.239.192/27" + }, + "eu-west-2": { + "FirehoseCidrBlock": "18.130.1.96/27" + }, + "eu-west-3": { + "FirehoseCidrBlock": "35.180.1.96/27" + }, + "me-south-1": { + "FirehoseCidrBlock": "15.185.91.0/27" + }, + "sa-east-1": { + "FirehoseCidrBlock": "18.228.1.128/27" + }, + "us-east-1": { + "FirehoseCidrBlock": "52.70.63.192/27" + }, + "us-east-2": { + "FirehoseCidrBlock": "13.58.135.96/27" + }, + "us-gov-east-1": { + "FirehoseCidrBlock": "18.253.138.96/27" + }, + "us-gov-west-1": { + "FirehoseCidrBlock": "52.61.204.160/27" + }, + "us-west-1": { + "FirehoseCidrBlock": "13.57.135.192/27" + }, + "us-west-2": { + "FirehoseCidrBlock": "52.89.255.224/27" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts new file mode 100644 index 0000000000000..a0a9569ca91a8 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts @@ -0,0 +1,28 @@ +#!/usr/bin/env node +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as logs from '@aws-cdk/aws-logs'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as destinations from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-firehose-delivery-stream-s3-all-properties'); + +const bucket = new s3.Bucket(stack, 'Bucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, +}); + +const logGroup = new logs.LogGroup(stack, 'LogGroup', { + removalPolicy: cdk.RemovalPolicy.DESTROY, +}); + +new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [new destinations.S3(bucket, { + logging: true, + logGroup: logGroup, + })], +}); + +app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json new file mode 100644 index 0000000000000..49f6c015db745 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json @@ -0,0 +1,392 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "BucketPolicyE9A3008A": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "Bucket83908E77" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "BucketAutoDeleteObjectsCustomResourceBAFD23C2": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "Bucket83908E77" + } + }, + "DependsOn": [ + "BucketPolicyE9A3008A" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" + } + ] + } + ] + } + ] + ] + } + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "__entrypoint__.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": "nodejs12.x", + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "Bucket83908E77" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + }, + "DeliveryStreamServiceRole964EEBCC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "DeliveryStreamLogGroup9D8FA3BB", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", + "Roles": [ + { + "Ref": "DeliveryStreamServiceRole964EEBCC" + } + ] + } + }, + "DeliveryStreamLogGroup9D8FA3BB": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "DeliveryStreamLogGroupS3DestinationE25573DB": { + "Type": "AWS::Logs::LogStream", + "Properties": { + "LogGroupName": { + "Ref": "DeliveryStreamLogGroup9D8FA3BB" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "DeliveryStreamF6D5572D": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamType": "DirectPut", + "ExtendedS3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "CloudWatchLoggingOptions": { + "Enabled": true, + "LogGroupName": { + "Ref": "DeliveryStreamLogGroup9D8FA3BB" + }, + "LogStreamName": { + "Ref": "DeliveryStreamLogGroupS3DestinationE25573DB" + } + }, + "RoleARN": { + "Fn::GetAtt": [ + "DeliveryStreamServiceRole964EEBCC", + "Arn" + ] + } + } + }, + "DependsOn": [ + "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", + "DeliveryStreamServiceRole964EEBCC" + ] + } + }, + "Parameters": { + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B": { + "Type": "String", + "Description": "S3 bucket for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + }, + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51": { + "Type": "String", + "Description": "S3 key for asset version \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + }, + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1ArtifactHash9ECACDFD": { + "Type": "String", + "Description": "Artifact hash for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + } + }, + "Mappings": { + "DeliveryStreamFirehoseCIDRMappingE9233479": { + "af-south-1": { + "FirehoseCidrBlock": "13.244.121.224/27" + }, + "ap-east-1": { + "FirehoseCidrBlock": "18.162.221.32/27" + }, + "ap-northeast-1": { + "FirehoseCidrBlock": "13.113.196.224/27" + }, + "ap-northeast-2": { + "FirehoseCidrBlock": "13.209.1.64/27" + }, + "ap-northeast-3": { + "FirehoseCidrBlock": "13.208.177.192/27" + }, + "ap-south-1": { + "FirehoseCidrBlock": "13.232.67.32/27" + }, + "ap-southeast-1": { + "FirehoseCidrBlock": "13.228.64.192/27" + }, + "ap-southeast-2": { + "FirehoseCidrBlock": "13.210.67.224/27" + }, + "ca-central-1": { + "FirehoseCidrBlock": "35.183.92.128/27" + }, + "cn-north-1": { + "FirehoseCidrBlock": "52.81.151.32/27" + }, + "cn-northwest-1": { + "FirehoseCidrBlock": "161.189.23.64/27" + }, + "eu-central-1": { + "FirehoseCidrBlock": "35.158.127.160/27" + }, + "eu-north-1": { + "FirehoseCidrBlock": "13.53.63.224/27" + }, + "eu-south-1": { + "FirehoseCidrBlock": "15.161.135.128/27" + }, + "eu-west-1": { + "FirehoseCidrBlock": "52.19.239.192/27" + }, + "eu-west-2": { + "FirehoseCidrBlock": "18.130.1.96/27" + }, + "eu-west-3": { + "FirehoseCidrBlock": "35.180.1.96/27" + }, + "me-south-1": { + "FirehoseCidrBlock": "15.185.91.0/27" + }, + "sa-east-1": { + "FirehoseCidrBlock": "18.228.1.128/27" + }, + "us-east-1": { + "FirehoseCidrBlock": "52.70.63.192/27" + }, + "us-east-2": { + "FirehoseCidrBlock": "13.58.135.96/27" + }, + "us-gov-east-1": { + "FirehoseCidrBlock": "18.253.138.96/27" + }, + "us-gov-west-1": { + "FirehoseCidrBlock": "52.61.204.160/27" + }, + "us-west-1": { + "FirehoseCidrBlock": "13.57.135.192/27" + }, + "us-west-2": { + "FirehoseCidrBlock": "52.89.255.224/27" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts new file mode 100644 index 0000000000000..3bc9a03b254ac --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts @@ -0,0 +1,20 @@ +#!/usr/bin/env node +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as destinations from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-firehose-delivery-stream-s3-basic'); + +const bucket = new s3.Bucket(stack, 'Bucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, +}); + +new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [new destinations.S3(bucket)], +}); + +app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-destination.test.ts new file mode 100644 index 0000000000000..75863078110e7 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-destination.test.ts @@ -0,0 +1,116 @@ +import '@aws-cdk/assert-internal/jest'; +import * as iam from '@aws-cdk/aws-iam'; +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as firehosedestinations from '../lib'; + +describe('S3 destination', () => { + let stack: cdk.Stack; + let bucket: s3.IBucket; + let deliveryStreamRole: iam.IRole; + let deliveryStream: firehose.IDeliveryStream; + + beforeEach(() => { + stack = new cdk.Stack(); + bucket = new s3.Bucket(stack, 'destination'); + deliveryStreamRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DeliveryStreamRole'); + deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'Delivery Stream', { + deliveryStreamName: 'mydeliverystream', + role: deliveryStreamRole, + }); + }); + + it('provides defaults when no configuration is provided', () => { + const destination = new firehosedestinations.S3(bucket); + + const destinationProperties = destination.bind(stack, { deliveryStream }).properties; + + expect(stack.resolve(destinationProperties)).toStrictEqual({ + extendedS3DestinationConfiguration: { + bucketArn: stack.resolve(bucket.bucketArn), + cloudWatchLoggingOptions: { + enabled: true, + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupS3Destination70CE1003', + }, + }, + roleArn: stack.resolve(deliveryStreamRole.roleArn), + }, + }); + }); + + it('allows full configuration', () => { + const destination = new firehosedestinations.S3(bucket, { + logging: true, + }); + + const destinationProperties = destination.bind(stack, { deliveryStream }).properties; + + expect(stack.resolve(destinationProperties)).toStrictEqual({ + extendedS3DestinationConfiguration: { + bucketArn: stack.resolve(bucket.bucketArn), + cloudWatchLoggingOptions: { + enabled: true, + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupS3Destination70CE1003', + }, + }, + roleArn: deliveryStreamRole.roleArn, + }, + }); + }); + + it('grants read/write access to the bucket', () => { + const destination = new firehosedestinations.S3(bucket); + + destination.bind(stack, { deliveryStream }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + Roles: ['DeliveryStreamRole'], + PolicyDocument: { + Statement: [ + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'destinationDB878FB5', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'destinationDB878FB5', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + }, + }); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 9c4d9f96c6f36..f04fe56e8f3da 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -9,8 +9,216 @@ > > [CFN Resources]: https://docs.aws.amazon.com/cdk/latest/guide/constructs.html#constructs_lib +![cdk-constructs: Experimental](https://img.shields.io/badge/cdk--constructs-experimental-important.svg?style=for-the-badge) + +> The APIs of higher level constructs in this module are experimental and under active development. +> They are subject to non-backward compatible changes or removal in any future version. These are +> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be +> announced in the release notes. This means that while you may use them, you may need to update +> your source code when upgrading to a newer version of this package. + --- -This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project. +[Amazon Kinesis Data Firehose](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html) +is a service for fully-managed delivery of real-time streaming data to storage services +such as Amazon S3, Amazon Redshift, Amazon Elasticsearch, Splunk, or any custom HTTP +endpoint or third-party services such as Datadog, Dynatrace, LogicMonitor, MongoDB, New +Relic, and Sumo Logic. + +Kinesis Data Firehose delivery streams are distinguished from Kinesis data streams in +their models of consumtpion. Whereas consumers read from a data stream by actively pulling +data from the stream, a delivery stream pushes data to its destination on a regular +cadence. This means that data streams are intended to have consumers that do on-demand +processing, like AWS Lambda or Amazon EC2. On the other hand, delivery streams are +intended to have destinations that are sources for offline processing and analytics, such +as Amazon S3 and Amazon Redshift. + +This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) +project. It allows you to define Kinesis Data Firehose delivery streams. + +## Defining a Delivery Stream + +In order to define a Delivery Stream, you must specify a destination. An S3 bucket can be +used as a destination. More supported destinations are covered [below](#destinations). + +```ts +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; +import * as s3 from '@aws-cdk/aws-s3'; + +const bucket = new s3.Bucket(this, 'Bucket'); +new DeliveryStream(this, 'Delivery Stream', { + destinations: [new destinations.S3(bucket)], +}); +``` + +The above example defines the following resources: + +- An S3 bucket +- A Kinesis Data Firehose delivery stream with Direct PUT as the source and CloudWatch + error logging turned on. +- An IAM role which gives the delivery stream permission to write to the S3 bucket. + +## Sources + +There are two main methods of sourcing input data: Kinesis Data Streams and via a "direct +put". This construct library currently only supports "direct put". See [#15500](https://github.com/aws/aws-cdk/issues/15500) to track the status of adding support for Kinesis Data Streams. + +See: [Sending Data to a Delivery Stream](https://docs.aws.amazon.com/firehose/latest/dev/basic-write.html) +in the *Kinesis Data Firehose Developer Guide*. + +### Direct Put + +Data must be provided via "direct put", ie., by using a `PutRecord` or `PutRecordBatch` API call. There are a number of ways of doing +so, such as: + +- Kinesis Agent: a standalone Java application that monitors and delivers files while + handling file rotation, checkpointing, and retries. See: [Writing to Kinesis Data Firehose Using Kinesis Agent](https://docs.aws.amazon.com/firehose/latest/dev/writing-with-agents.html) + in the *Kinesis Data Firehose Developer Guide*. +- AWS SDK: a general purpose solution that allows you to deliver data to a delivery stream + from anywhere using Java, .NET, Node.js, Python, or Ruby. See: [Writing to Kinesis Data Firehose Using the AWS SDK](https://docs.aws.amazon.com/firehose/latest/dev/writing-with-sdk.html) + in the *Kinesis Data Firehose Developer Guide*. +- CloudWatch Logs: subscribe to a log group and receive filtered log events directly into + a delivery stream. See: [logs-destinations](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-logs-destinations-readme.html). +- Eventbridge: add an event rule target to send events to a delivery stream based on the + rule filtering. See: [events-targets](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-events-targets-readme.html). +- SNS: add a subscription to send all notifications from the topic to a delivery + stream. See: [sns-subscriptions](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-sns-subscriptions-readme.html). +- IoT: add an action to an IoT rule to send various IoT information to a delivery stream + +## Destinations + +The following destinations are supported. See [kinesisfirehose-destinations](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisfirehose-destinations-readme.html) +for the implementations of these destinations. + +### S3 + +Defining a delivery stream with an S3 bucket destination: + +```ts +import * as s3 from '@aws-cdk/aws-s3'; +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; + +const bucket = new s3.Bucket(this, 'Bucket'); + +const s3Destination = new destinations.S3(bucket); + +new DeliveryStream(this, 'Delivery Stream', { + destinations: [s3Destination], +}); +``` + +## Monitoring + +Kinesis Data Firehose is integrated with CloudWatch, so you can monitor the performance of +your delivery streams via logs and metrics. + +### Logs + +Kinesis Data Firehose will send logs to CloudWatch when data transformation or data +delivery fails. The CDK will enable logging by default and create a CloudWatch LogGroup +and LogStream for your Delivery Stream. + +You can provide a specific log group to specify where the CDK will create the log streams +where log events will be sent: + +```ts fixture=with-destination +import * as logs from '@aws-cdk/aws-logs'; + +const logGroup = new logs.LogGroup(this, 'Log Group'); +new DeliveryStream(this, 'Delivery Stream', { + logGroup: logGroup, + destinations: [destination], +}); +``` + +Logging can also be disabled: + +```ts fixture=with-destination +new DeliveryStream(this, 'Delivery Stream', { + loggingEnabled: false, + destinations: [destination], +}); +``` + +See: [Monitoring using CloudWatch Logs](https://docs.aws.amazon.com/firehose/latest/dev/monitoring-with-cloudwatch-logs.html) +in the *Kinesis Data Firehose Developer Guide*. + +## Specifying an IAM role + +The DeliveryStream class automatically creates an IAM role with all the minimum necessary +permissions for Kinesis Data Firehose to access the resources referenced by your delivery +stream. For example: an Elasticsearch domain, a Redshift cluster, a backup or destination +S3 bucket, a Lambda data transformer, an AWS Glue table schema, etc. If you wish, you may +specify your own IAM role. It must have the correct permissions, or delivery stream +creation or data delivery may fail. + +```ts fixture=with-bucket +import * as iam from '@aws-cdk/aws-iam'; + +const role = new iam.Role(this, 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), +} +bucket.grantWrite(role); +new DeliveryStream(stack, 'Delivery Stream', { + destinations: [new destinations.S3(bucket)], + role: role, +}); +``` + +See [Controlling Access](https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html) +in the *Kinesis Data Firehose Developer Guide*. + +## Granting application access to a delivery stream + +IAM roles, users or groups which need to be able to work with delivery streams should be +granted IAM permissions. + +Any object that implements the `IGrantable` interface (ie., has an associated principal) +can be granted permissions to a delivery stream by calling: + +- `grantPutRecords(principal)` - grants the principal the ability to put records onto the + delivery stream +- `grant(principal, ...actions)` - grants the principal permission to a custom set of + actions + +```ts fixture=with-delivery-stream +import * as iam from '@aws-cdk/aws-iam'; +const lambdaRole = new iam.Role(this, 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), +} + +// Give the role permissions to write data to the delivery stream +deliveryStream.grantPutRecords(lambdaRole); +``` + +The following write permissions are provided to a service principal by the `grantPutRecords()` method: + +- `firehose:PutRecord` +- `firehose:PutRecordBatch` + +## Granting a delivery stream access to a resource + +Conversely to the above, Kinesis Data Firehose requires permissions in order for delivery +streams to interact with resources that you own. For example, if an S3 bucket is specified +as a destination of a delivery stream, the delivery stream must be granted permissions to +put and get objects from the bucket. When using the built-in AWS service destinations +found in the `@aws-cdk/aws-kinesisfirehose-destinations` module, the CDK grants the +permissions automatically. However, custom or third-party destinations may require custom +permissions. In this case, use the delivery stream as an `IGrantable`, as follows: + +```ts fixture=with-delivery-stream +/// !hide +const myDestinationResource = { + grantWrite(grantee: IGrantable) {} +} +/// !show +myDestinationResource.grantWrite(deliveryStream); +``` + +## Multiple destinations + +Though the delivery stream allows specifying an array of destinations, only one +destination per delivery stream is currently allowed. This limitation is enforced at +compile time and will throw an error. diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts new file mode 100644 index 0000000000000..6a7d40e434507 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -0,0 +1,256 @@ +import * as cloudwatch from '@aws-cdk/aws-cloudwatch'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as iam from '@aws-cdk/aws-iam'; +import * as cdk from '@aws-cdk/core'; +import { RegionInfo } from '@aws-cdk/region-info'; +import { Construct } from 'constructs'; +import { IDestination } from './destination'; +import { CfnDeliveryStream } from './kinesisfirehose.generated'; + +const PUT_RECORD_ACTIONS = [ + 'firehose:PutRecord', + 'firehose:PutRecordBatch', +]; + +/** + * Represents a Kinesis Data Firehose delivery stream. + */ +export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.IConnectable, cdk.ITaggable { + /** + * The ARN of the delivery stream. + * + * @attribute + */ + readonly deliveryStreamArn: string; + + /** + * The name of the delivery stream. + * + * @attribute + */ + readonly deliveryStreamName: string; + + /** + * Grant the `grantee` identity permissions to perform `actions`. + */ + grant(grantee: iam.IGrantable, ...actions: string[]): iam.Grant; + + /** + * Grant the `grantee` identity permissions to perform `firehose:PutRecord` and `firehose:PutRecordBatch` actions on this delivery stream. + */ + grantPutRecords(grantee: iam.IGrantable): iam.Grant; + + /** + * Return the given named metric for this delivery stream. + */ + metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric; +} + +/** + * Base class for new and imported Kinesis Data Firehose delivery streams. + */ +export abstract class DeliveryStreamBase extends cdk.Resource implements IDeliveryStream { + + public abstract readonly deliveryStreamName: string; + + public abstract readonly deliveryStreamArn: string; + + public abstract readonly grantPrincipal: iam.IPrincipal; + + /** + * Network connections between Kinesis Data Firehose and other resources, i.e. Redshift cluster. + */ + public readonly connections: ec2.Connections; + + public readonly tags = new cdk.TagManager(cdk.TagType.STANDARD, 'AWS::KinesisFirehose::DeliveryStream'); + + constructor(scope: Construct, id: string) { + super(scope, id); + + this.connections = setConnections(this); + } + + public grant(grantee: iam.IGrantable, ...actions: string[]): iam.Grant { + return iam.Grant.addToPrincipal({ + resourceArns: [this.deliveryStreamArn], + grantee: grantee, + actions: actions, + }); + } + + public grantPutRecords(grantee: iam.IGrantable): iam.Grant { + return this.grant(grantee, ...PUT_RECORD_ACTIONS); + } + + public metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return new cloudwatch.Metric({ + namespace: 'AWS/Firehose', + metricName: metricName, + dimensions: { + DeliveryStreamName: this.deliveryStreamName, + }, + ...props, + }).attachTo(this); + } +} + +/** + * Properties for a new delivery stream. + */ +export interface DeliveryStreamProps { + /** + * The destinations that this delivery stream will deliver data to. + * + * Only a singleton array is supported at this time. + */ + readonly destinations: IDestination[]; + + /** + * A name for the delivery stream. + * + * @default - a name is generated by CloudFormation. + */ + readonly deliveryStreamName?: string; + + /** + * The IAM role assumed by Kinesis Data Firehose to read from sources, invoke processors, and write to destinations. + * + * @default - a role will be created with default permissions. + */ + readonly role?: iam.IRole; +} + +/** + * A full specification of a delivery stream that can be used to import it fluently into the CDK application. + */ +export interface DeliveryStreamAttributes { + /** + * The ARN of the delivery stream. + * + * At least one of deliveryStreamArn and deliveryStreamName must be provided. + * + * @default - derived from `deliveryStreamName`. + */ + readonly deliveryStreamArn?: string; + + /** + * The name of the delivery stream + * + * At least one of deliveryStreamName and deliveryStreamArn must be provided. + * + * @default - derived from `deliveryStreamArn`. + */ + readonly deliveryStreamName?: string; + + + /** + * The IAM role associated with this delivery stream. + * + * Assumed by Kinesis Data Firehose to read from sources, invoke processors, and write to destinations. + * + * @default - the imported stream cannot be granted access to other resources as an `iam.IGrantable`. + */ + readonly role?: iam.IRole; +} + +/** + * Create a Kinesis Data Firehose delivery stream + * + * @resource AWS::KinesisFirehose::DeliveryStream + */ +export class DeliveryStream extends DeliveryStreamBase { + /** + * Import an existing delivery stream from its name. + */ + static fromDeliveryStreamName(scope: Construct, id: string, deliveryStreamName: string): IDeliveryStream { + return this.fromDeliveryStreamAttributes(scope, id, { deliveryStreamName }); + } + + /** + * Import an existing delivery stream from its ARN. + */ + static fromDeliveryStreamArn(scope: Construct, id: string, deliveryStreamArn: string): IDeliveryStream { + return this.fromDeliveryStreamAttributes(scope, id, { deliveryStreamArn }); + } + + /** + * Import an existing delivery stream from its attributes. + */ + static fromDeliveryStreamAttributes(scope: Construct, id: string, attrs: DeliveryStreamAttributes): IDeliveryStream { + if (!attrs.deliveryStreamName && !attrs.deliveryStreamArn) { + throw new Error('Either deliveryStreamName or deliveryStreamArn must be provided in DeliveryStreamAttributes'); + } + const deliveryStreamName = attrs.deliveryStreamName ?? cdk.Stack.of(scope).parseArn(attrs.deliveryStreamArn!).resourceName; + if (!deliveryStreamName) { + throw new Error(`Could not import delivery stream from malformatted ARN ${attrs.deliveryStreamArn}: could not determine resource name`); + } + const deliveryStreamArn = attrs.deliveryStreamArn ?? cdk.Stack.of(scope).formatArn({ + service: 'firehose', + resource: 'deliverystream', + resourceName: attrs.deliveryStreamName, + }); + class Import extends DeliveryStreamBase { + public readonly deliveryStreamName = deliveryStreamName!; + public readonly deliveryStreamArn = deliveryStreamArn; + public readonly grantPrincipal = attrs.role ?? new iam.UnknownPrincipal({ resource: this }); + } + return new Import(scope, id); + } + + readonly deliveryStreamName: string; + + readonly deliveryStreamArn: string; + + readonly grantPrincipal: iam.IPrincipal; + + constructor(scope: Construct, id: string, props: DeliveryStreamProps) { + super(scope, id); + + const role = props.role ?? new iam.Role(this, 'Service Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); + this.grantPrincipal = role; + + if (props.destinations.length !== 1) { + throw new Error(`Only one destination is allowed per delivery stream, given ${props.destinations.length}`); + } + const destinationConfig = props.destinations[0].bind(this, { deliveryStream: this }); + + const resource = new CfnDeliveryStream(this, 'Resource', { + deliveryStreamName: props.deliveryStreamName, + deliveryStreamType: 'DirectPut', + ...destinationConfig.properties, + }); + resource.node.addDependency(this.grantPrincipal); + + this.deliveryStreamArn = this.getResourceArnAttribute(resource.attrArn, { + service: 'kinesis', + resource: 'deliverystream', + resourceName: this.physicalName, + }); + this.deliveryStreamName = this.getResourceNameAttribute(resource.ref); + } +} + +function setConnections(scope: Construct) { + const region = cdk.Stack.of(scope).region; + let cidrBlock = RegionInfo.get(region).firehoseCidrBlock; + if (!cidrBlock) { + const mapping: {[region: string]: { FirehoseCidrBlock: string }} = {}; + RegionInfo.regions.forEach((regionInfo) => { + if (regionInfo.firehoseCidrBlock) { + mapping[regionInfo.name] = { + FirehoseCidrBlock: regionInfo.firehoseCidrBlock, + }; + } + }); + const cfnMapping = new cdk.CfnMapping(scope, 'Firehose CIDR Mapping', { + mapping, + }); + cidrBlock = cdk.Fn.findInMap(cfnMapping.logicalId, region, 'FirehoseCidrBlock'); + } + + return new ec2.Connections({ + peer: ec2.Peer.ipv4(cidrBlock), + }); +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts new file mode 100644 index 0000000000000..fecd1094b884b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -0,0 +1,99 @@ +import * as logs from '@aws-cdk/aws-logs'; +import { Construct } from 'constructs'; +import { IDeliveryStream } from './delivery-stream'; +import { CfnDeliveryStream } from './kinesisfirehose.generated'; + +/** + * A Kinesis Data Firehose delivery stream destination configuration. + */ +export interface DestinationConfig { + /** + * Schema-less properties that will be injected directly into `CfnDeliveryStream`. + */ + readonly properties: object; +} + +/** + * Options when binding a destination to a delivery stream. + */ +export interface DestinationBindOptions { + /** + * The delivery stream. + */ + readonly deliveryStream: IDeliveryStream; +} + +/** + * A Kinesis Data Firehose delivery stream destination. + */ +export interface IDestination { + /** + * Binds this destination to the Kinesis Data Firehose delivery stream. + * + * Implementers should use this method to bind resources to the stack and initialize values using the provided stream. + */ + bind(scope: Construct, options: DestinationBindOptions): DestinationConfig; +} + +/** + * Generic properties for defining a delivery stream destination. + */ +export interface DestinationProps { + /** + * If true, log errors when data transformation or data delivery fails. + * + * If `logGroup` is provided, this will be implicitly set to `true`. + * + * @default true - errors are logged. + */ + readonly logging?: boolean; + + /** + * The CloudWatch log group where log streams will be created to hold error logs. + * + * @default - if `logging` is set to `true`, a log group will be created for you. + */ + readonly logGroup?: logs.ILogGroup; +} + +/** + * Abstract base class that destination types can extend to benefit from methods that create generic configuration. + */ +export abstract class DestinationBase implements IDestination { + private logGroups: { [logGroupId: string]: logs.ILogGroup } = {}; + + constructor(protected readonly props: DestinationProps = {}) { } + + abstract bind(scope: Construct, options: DestinationBindOptions): DestinationConfig; + + protected createLoggingOptions( + scope: Construct, + deliveryStream: IDeliveryStream, + streamId: string, + ): CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { + return this._createLoggingOptions(scope, deliveryStream, streamId, 'LogGroup', this.props.logging, this.props.logGroup); + } + + private _createLoggingOptions( + scope: Construct, + deliveryStream: IDeliveryStream, + streamId: string, + logGroupId: string, + logging?: boolean, + propsLogGroup?: logs.ILogGroup, + ): CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { + if (logging === false && propsLogGroup) { + throw new Error('logging cannot be set to false when logGroup is provided'); + } + if (logging !== false || propsLogGroup) { + this.logGroups[logGroupId] = this.logGroups[logGroupId] ?? propsLogGroup ?? new logs.LogGroup(scope, logGroupId); + this.logGroups[logGroupId].grantWrite(deliveryStream); + return { + enabled: true, + logGroupName: this.logGroups[logGroupId].logGroupName, + logStreamName: this.logGroups[logGroupId].addStream(streamId).logStreamName, + }; + } + return undefined; + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts index dd7beef14d159..3eddb6dec468e 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts @@ -1,2 +1,5 @@ +export * from './delivery-stream'; +export * from './destination'; + // AWS::KinesisFirehose CloudFormation Resources: export * from './kinesisfirehose.generated'; diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index e3aaff7954822..02d64257f825b 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -77,26 +77,52 @@ "cdk-build-tools": "0.0.0", "cfn2ts": "0.0.0", "pkglint": "0.0.0", - "@aws-cdk/assert-internal": "0.0.0" + "@aws-cdk/assert-internal": "0.0.0", + "cdk-integ-tools": "0.0.0" }, "dependencies": { + "@aws-cdk/aws-cloudwatch": "0.0.0", + "@aws-cdk/aws-ec2": "0.0.0", + "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesis": "0.0.0", + "@aws-cdk/aws-kms": "0.0.0", + "@aws-cdk/aws-lambda": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/custom-resources": "0.0.0", + "@aws-cdk/region-info": "0.0.0", "constructs": "^3.3.69" }, "homepage": "https://github.com/aws/aws-cdk", "peerDependencies": { + "@aws-cdk/aws-cloudwatch": "0.0.0", + "@aws-cdk/aws-ec2": "0.0.0", + "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesis": "0.0.0", + "@aws-cdk/aws-kms": "0.0.0", + "@aws-cdk/aws-lambda": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/custom-resources": "0.0.0", + "@aws-cdk/region-info": "0.0.0", "constructs": "^3.3.69" }, "engines": { "node": ">= 10.13.0 <13 || >=13.7.0" }, "stability": "experimental", - "maturity": "cfn-only", + "maturity": "experimental", "awscdkio": { "announce": false }, "publishConfig": { "tag": "latest" + }, + "awslint": { + "exclude": [ + "no-unused-type:@aws-cdk/aws-kinesisfirehose.Compression" + ] } } diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture new file mode 100644 index 0000000000000..8a68efc25aa8e --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture @@ -0,0 +1,11 @@ +// Fixture with packages imported, but nothing else +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts new file mode 100644 index 0000000000000..ae76cc9af7e4c --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -0,0 +1,281 @@ +import '@aws-cdk/assert-internal/jest'; +import { ABSENT } from '@aws-cdk/assert-internal'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as iam from '@aws-cdk/aws-iam'; +import * as cdk from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import * as firehose from '../lib'; + +describe('delivery stream', () => { + let stack: cdk.Stack; + + const bucketArn = 'arn:aws:s3:::my-bucket'; + const roleArn = 'arn:aws:iam::111122223333:role/my-role'; + const mockS3Destination: firehose.IDestination = { + bind(_scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + return { + properties: { + s3DestinationConfiguration: { + bucketArn: bucketArn, + roleArn: roleArn, + }, + }, + }; + }, + }; + + beforeEach(() => { + stack = new cdk.Stack(); + }); + + test('creates stream with default values', () => { + new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + DeliveryStreamEncryptionConfigurationInput: ABSENT, + DeliveryStreamName: ABSENT, + DeliveryStreamType: 'DirectPut', + KinesisStreamSourceConfiguration: ABSENT, + S3DestinationConfiguration: { + BucketARN: bucketArn, + RoleARN: roleArn, + }, + }); + }); + + test('provided role is set as grant principal', () => { + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); + + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + role: role, + }); + + expect(deliveryStream.grantPrincipal).toBe(role); + }); + + test('not providing role creates one', () => { + new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + expect(stack).toHaveResourceLike('AWS::IAM::Role', { + AssumeRolePolicyDocument: { + Statement: [ + { + Principal: { + Service: 'firehose.amazonaws.com', + }, + }, + ], + }, + }); + }); + + test('grant provides access to stream', () => { + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + deliveryStream.grant(role, 'firehose:PutRecord'); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 'firehose:PutRecord', + Resource: stack.resolve(deliveryStream.deliveryStreamArn), + }, + ], + }, + Roles: [stack.resolve(role.roleName)], + }); + }); + + test('grantPutRecords provides PutRecord* access to stream', () => { + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + deliveryStream.grantPutRecords(role); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'firehose:PutRecord', + 'firehose:PutRecordBatch', + ], + Resource: stack.resolve(deliveryStream.deliveryStreamArn), + }, + ], + }, + Roles: [stack.resolve(role.roleName)], + }); + }); + + test('supplying 0 or multiple destinations throws', () => { + expect(() => new firehose.DeliveryStream(stack, 'No Destinations', { + destinations: [], + })).toThrowError(/Only one destination is allowed per delivery stream/); + expect(() => new firehose.DeliveryStream(stack, 'Too Many Destinations', { + destinations: [mockS3Destination, mockS3Destination], + })).toThrowError(/Only one destination is allowed per delivery stream/); + }); + + describe('metric methods provide a Metric with configured and attached properties', () => { + beforeEach(() => { + stack = new cdk.Stack(undefined, undefined, { env: { account: '000000000000', region: 'us-west-1' } }); + }); + + test('metric', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + const metric = deliveryStream.metric('IncomingRecords'); + + expect(metric).toMatchObject({ + account: stack.account, + region: stack.region, + namespace: 'AWS/Firehose', + metricName: 'IncomingRecords', + dimensions: { + DeliveryStreamName: deliveryStream.deliveryStreamName, + }, + }); + }); + }); + + test('allows connections for Firehose IP addresses using map when region not specified', () => { + const vpc = new ec2.Vpc(stack, 'VPC'); + const securityGroup = new ec2.SecurityGroup(stack, 'Security Group', { vpc }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + securityGroup.connections.allowFrom(deliveryStream, ec2.Port.allTcp()); + + expect(stack).toHaveResourceLike('AWS::EC2::SecurityGroup', { + SecurityGroupIngress: [ + { + CidrIp: { + 'Fn::FindInMap': [ + 'DeliveryStreamFirehoseCIDRMappingE9233479', + { + Ref: 'AWS::Region', + }, + 'FirehoseCidrBlock', + ], + }, + }, + ], + }); + }); + + test('allows connections for Firehose IP addresses using literal when region specified', () => { + stack = new cdk.Stack(undefined, undefined, { env: { region: 'us-west-1' } }); + const vpc = new ec2.Vpc(stack, 'VPC'); + const securityGroup = new ec2.SecurityGroup(stack, 'Security Group', { vpc }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + securityGroup.connections.allowFrom(deliveryStream, ec2.Port.allTcp()); + + expect(stack).toHaveResourceLike('AWS::EC2::SecurityGroup', { + SecurityGroupIngress: [ + { + CidrIp: '13.57.135.192/27', + }, + ], + }); + }); + + test('can add tags', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + cdk.Tags.of(deliveryStream).add('tagKey', 'tagValue'); + + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + Tags: [ + { + Key: 'tagKey', + Value: 'tagValue', + }, + ], + }); + }); + + describe('importing', () => { + test('from name', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamName(stack, 'DeliveryStream', 'mydeliverystream'); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(stack.resolve(deliveryStream.deliveryStreamArn)).toStrictEqual({ + 'Fn::Join': ['', ['arn:', stack.resolve(stack.partition), ':firehose:', stack.resolve(stack.region), ':', stack.resolve(stack.account), ':deliverystream/mydeliverystream']], + }); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from ARN', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamArn(stack, 'DeliveryStream', 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream'); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(deliveryStream.deliveryStreamArn).toBe('arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream'); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from attributes (just name)', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamName: 'mydeliverystream' }); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(stack.resolve(deliveryStream.deliveryStreamArn)).toStrictEqual({ + 'Fn::Join': ['', ['arn:', stack.resolve(stack.partition), ':firehose:', stack.resolve(stack.region), ':', stack.resolve(stack.account), ':deliverystream/mydeliverystream']], + }); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from attributes (just ARN)', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamArn: 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream' }); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(deliveryStream.deliveryStreamArn).toBe('arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream'); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from attributes (with role)', () => { + const role = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DeliveryStreamRole'); + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamName: 'mydeliverystream', role }); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(stack.resolve(deliveryStream.deliveryStreamArn)).toStrictEqual({ + 'Fn::Join': ['', ['arn:', stack.resolve(stack.partition), ':firehose:', stack.resolve(stack.region), ':', stack.resolve(stack.account), ':deliverystream/mydeliverystream']], + }); + expect(deliveryStream.grantPrincipal).toBe(role); + }); + + test('throws when malformatted ARN', () => { + expect(() => firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamArn: 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/' })) + .toThrowError(/Could not import delivery stream from malformatted ARN/); + }); + + test('throws when without name or ARN', () => { + expect(() => firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', {})) + .toThrowError('Either deliveryStreamName or deliveryStreamArn must be provided in DeliveryStreamAttributes'); + }); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts new file mode 100644 index 0000000000000..a5be8f409025d --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts @@ -0,0 +1,158 @@ +import '@aws-cdk/assert-internal/jest'; +import * as iam from '@aws-cdk/aws-iam'; +import * as logs from '@aws-cdk/aws-logs'; +import * as cdk from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import * as firehose from '../lib'; + +describe('destination', () => { + let stack: cdk.Stack; + let deliveryStreamRole: iam.IRole; + let deliveryStream: firehose.IDeliveryStream; + + const deliveryStreamRoleArn = 'arn:aws:iam::111122223333:role/DeliveryStreamRole'; + + beforeEach(() => { + stack = new cdk.Stack(); + deliveryStreamRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', deliveryStreamRoleArn); + deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'Delivery Stream', { + deliveryStreamName: 'mydeliverystream', + role: deliveryStreamRole, + }); + }); + + describe('createLoggingOptions', () => { + class LoggingDestination extends firehose.DestinationBase { + public bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { + return { + properties: { + testDestinationConfig: { + loggingConfig: this.createLoggingOptions(scope, options.deliveryStream, 'streamId'), + }, + }, + }; + } + } + + test('creates resources and configuration by default', () => { + const testDestination = new LoggingDestination(); + + const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); + expect(stack.resolve(testDestinationConfig)).toStrictEqual({ + properties: { + testDestinationConfig: { + loggingConfig: { + enabled: true, + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupstreamId3B940622', + }, + }, + }, + }, + }); + }); + test('does not create resources or configuration if disabled', () => { + const testDestination = new LoggingDestination({ logging: false }); + + const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + + expect(stack.resolve(testDestinationConfig)).toStrictEqual({ + properties: { + testDestinationConfig: {}, + }, + }); + }); + + test('creates configuration if log group provided', () => { + const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); + + const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + + expect(stack.resolve(testDestinationConfig)).toMatchObject({ + properties: { + testDestinationConfig: { + loggingConfig: { + enabled: true, + }, + }, + }, + }); + }); + + test('throws error if logging disabled but log group provided', () => { + const testDestination = new LoggingDestination({ logging: false, logGroup: new logs.LogGroup(stack, 'Log Group') }); + + expect(() => testDestination.bind(stack, { deliveryStream })).toThrowError('logging cannot be set to false when logGroup is provided'); + }); + + test('uses provided log group', () => { + const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); + + const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + + expect(stack).toCountResources('AWS::Logs::LogGroup', 1); + expect(stack.resolve(testDestinationConfig)).toMatchObject({ + properties: { + testDestinationConfig: { + loggingConfig: { + enabled: true, + logGroupName: { + Ref: 'LogGroupD9735569', + }, + logStreamName: { + Ref: 'LogGroupstreamIdA1293DC2', + }, + }, + }, + }, + }); + }); + + test('re-uses log group if called multiple times', () => { + const testDestination = new class extends firehose.DestinationBase { + public bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { + return { + properties: { + testDestinationConfig: { + loggingConfig: this.createLoggingOptions(scope, options.deliveryStream, 'streamId'), + anotherLoggingConfig: this.createLoggingOptions(scope, options.deliveryStream, 'anotherStreamId'), + }, + }, + }; + } + }(); + + const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + + expect(stack).toCountResources('AWS::Logs::LogGroup', 1); + expect(stack.resolve(testDestinationConfig)).toMatchObject({ + properties: { + testDestinationConfig: { + loggingConfig: { + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupstreamId3B940622', + }, + }, + anotherLoggingConfig: { + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupanotherStreamIdF2754481', + }, + }, + }, + }, + }); + }); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json new file mode 100644 index 0000000000000..2d783a749781a --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json @@ -0,0 +1,145 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "Role1ABCC5F0": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "DeliveryStreamServiceRole964EEBCC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "DeliveryStreamF6D5572D": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamType": "DirectPut", + "S3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "RoleARN": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + } + } + }, + "DependsOn": [ + "DeliveryStreamServiceRole964EEBCC" + ] + } + }, + "Mappings": { + "DeliveryStreamFirehoseCIDRMappingE9233479": { + "af-south-1": { + "FirehoseCidrBlock": "13.244.121.224/27" + }, + "ap-east-1": { + "FirehoseCidrBlock": "18.162.221.32/27" + }, + "ap-northeast-1": { + "FirehoseCidrBlock": "13.113.196.224/27" + }, + "ap-northeast-2": { + "FirehoseCidrBlock": "13.209.1.64/27" + }, + "ap-northeast-3": { + "FirehoseCidrBlock": "13.208.177.192/27" + }, + "ap-south-1": { + "FirehoseCidrBlock": "13.232.67.32/27" + }, + "ap-southeast-1": { + "FirehoseCidrBlock": "13.228.64.192/27" + }, + "ap-southeast-2": { + "FirehoseCidrBlock": "13.210.67.224/27" + }, + "ca-central-1": { + "FirehoseCidrBlock": "35.183.92.128/27" + }, + "cn-north-1": { + "FirehoseCidrBlock": "52.81.151.32/27" + }, + "cn-northwest-1": { + "FirehoseCidrBlock": "161.189.23.64/27" + }, + "eu-central-1": { + "FirehoseCidrBlock": "35.158.127.160/27" + }, + "eu-north-1": { + "FirehoseCidrBlock": "13.53.63.224/27" + }, + "eu-south-1": { + "FirehoseCidrBlock": "15.161.135.128/27" + }, + "eu-west-1": { + "FirehoseCidrBlock": "52.19.239.192/27" + }, + "eu-west-2": { + "FirehoseCidrBlock": "18.130.1.96/27" + }, + "eu-west-3": { + "FirehoseCidrBlock": "35.180.1.96/27" + }, + "me-south-1": { + "FirehoseCidrBlock": "15.185.91.0/27" + }, + "sa-east-1": { + "FirehoseCidrBlock": "18.228.1.128/27" + }, + "us-east-1": { + "FirehoseCidrBlock": "52.70.63.192/27" + }, + "us-east-2": { + "FirehoseCidrBlock": "13.58.135.96/27" + }, + "us-gov-east-1": { + "FirehoseCidrBlock": "18.253.138.96/27" + }, + "us-gov-west-1": { + "FirehoseCidrBlock": "52.61.204.160/27" + }, + "us-west-1": { + "FirehoseCidrBlock": "13.57.135.192/27" + }, + "us-west-2": { + "FirehoseCidrBlock": "52.89.255.224/27" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts new file mode 100644 index 0000000000000..7bcbb788399c8 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts @@ -0,0 +1,37 @@ +#!/usr/bin/env node +import * as iam from '@aws-cdk/aws-iam'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as constructs from 'constructs'; +import * as firehose from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-firehose-delivery-stream'); + +const bucket = new s3.Bucket(stack, 'Bucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, +}); + +const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), +}); + +const mockS3Destination: firehose.IDestination = { + bind(_scope: constructs.Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + return { + properties: { + s3DestinationConfiguration: { + bucketArn: bucket.bucketArn, + roleArn: role.roleArn, + }, + }, + }; + }, +}; + +new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], +}); + +app.synth(); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts deleted file mode 100644 index c4505ad966984..0000000000000 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import '@aws-cdk/assert-internal/jest'; -import {} from '../lib'; - -test('No tests are specified for this package', () => { - expect(true).toBe(true); -}); diff --git a/packages/@aws-cdk/region-info/build-tools/fact-tables.ts b/packages/@aws-cdk/region-info/build-tools/fact-tables.ts index c2ce689f3aaf3..28ec007b00c84 100644 --- a/packages/@aws-cdk/region-info/build-tools/fact-tables.ts +++ b/packages/@aws-cdk/region-info/build-tools/fact-tables.ts @@ -158,3 +158,32 @@ export const APPMESH_ECR_ACCOUNTS: { [region: string]: string } = { 'us-west-1': '840364872350', 'us-west-2': '840364872350', }; + +// https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-rs-vpc +export const FIREHOSE_CIDR_BLOCKS: { [region: string]: string } = { + 'af-south-1': '13.244.121.224', + 'ap-east-1': '18.162.221.32', + 'ap-northeast-1': '13.113.196.224', + 'ap-northeast-2': '13.209.1.64', + 'ap-northeast-3': '13.208.177.192', + 'ap-south-1': '13.232.67.32', + 'ap-southeast-1': '13.228.64.192', + 'ap-southeast-2': '13.210.67.224', + 'ca-central-1': '35.183.92.128', + 'cn-north-1': '52.81.151.32', + 'cn-northwest-1': '161.189.23.64', + 'eu-central-1': '35.158.127.160', + 'eu-north-1': '13.53.63.224', + 'eu-south-1': '15.161.135.128', + 'eu-west-1': '52.19.239.192', + 'eu-west-2': '18.130.1.96', + 'eu-west-3': '35.180.1.96', + 'me-south-1': '15.185.91.0', + 'sa-east-1': '18.228.1.128', + 'us-east-1': '52.70.63.192', + 'us-east-2': '13.58.135.96', + 'us-gov-east-1': '18.253.138.96', + 'us-gov-west-1': '52.61.204.160', + 'us-west-1': '13.57.135.192', + 'us-west-2': '52.89.255.224', +}; diff --git a/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts b/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts index d23704b6d0062..63455b72ef665 100644 --- a/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts +++ b/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts @@ -3,14 +3,15 @@ import * as fs from 'fs-extra'; import { Default } from '../lib/default'; import { AWS_REGIONS, AWS_SERVICES } from './aws-entities'; import { - APPMESH_ECR_ACCOUNTS, AWS_CDK_METADATA, AWS_OLDER_REGIONS, DLC_REPOSITORY_ACCOUNTS, ELBV2_ACCOUNTS, PARTITION_MAP, - ROUTE_53_BUCKET_WEBSITE_ZONE_IDS, + APPMESH_ECR_ACCOUNTS, AWS_CDK_METADATA, AWS_OLDER_REGIONS, DLC_REPOSITORY_ACCOUNTS, ELBV2_ACCOUNTS, FIREHOSE_CIDR_BLOCKS, + PARTITION_MAP, ROUTE_53_BUCKET_WEBSITE_ZONE_IDS, } from './fact-tables'; async function main(): Promise { checkRegions(APPMESH_ECR_ACCOUNTS); checkRegions(DLC_REPOSITORY_ACCOUNTS); checkRegions(ELBV2_ACCOUNTS); + checkRegions(FIREHOSE_CIDR_BLOCKS); checkRegions(ROUTE_53_BUCKET_WEBSITE_ZONE_IDS); const lines = [ @@ -61,6 +62,11 @@ async function main(): Promise { registerFact(region, 'APPMESH_ECR_ACCOUNT', APPMESH_ECR_ACCOUNTS[region]); + const firehoseCidrBlock = FIREHOSE_CIDR_BLOCKS[region]; + if (firehoseCidrBlock) { + registerFact(region, 'FIREHOSE_CIDR_BLOCK', `${FIREHOSE_CIDR_BLOCKS[region]}/27`); + } + const vpcEndpointServiceNamePrefix = `${domainSuffix.split('.').reverse().join('.')}.vpce`; registerFact(region, 'VPC_ENDPOINT_SERVICE_NAME_PREFIX', vpcEndpointServiceNamePrefix); diff --git a/packages/@aws-cdk/region-info/lib/fact.ts b/packages/@aws-cdk/region-info/lib/fact.ts index 3b5e57835cc7e..6ccef0e8b794f 100644 --- a/packages/@aws-cdk/region-info/lib/fact.ts +++ b/packages/@aws-cdk/region-info/lib/fact.ts @@ -152,6 +152,11 @@ export class FactName { */ public static readonly APPMESH_ECR_ACCOUNT = 'appMeshRepositoryAccount'; + /** + * The CIDR block used by Kinesis Data Firehose servers. + */ + public static readonly FIREHOSE_CIDR_BLOCK = 'firehoseCidrBlock'; + /** * The name of the regional service principal for a given service. * diff --git a/packages/@aws-cdk/region-info/lib/region-info.ts b/packages/@aws-cdk/region-info/lib/region-info.ts index 042b3cec9c177..9e28120a8da62 100644 --- a/packages/@aws-cdk/region-info/lib/region-info.ts +++ b/packages/@aws-cdk/region-info/lib/region-info.ts @@ -117,4 +117,11 @@ export class RegionInfo { public get appMeshRepositoryAccount(): string | undefined { return Fact.find(this.name, FactName.APPMESH_ECR_ACCOUNT); } + + /** + * The CIDR block used by Kinesis Data Firehose servers. + */ + public get firehoseCidrBlock(): string | undefined { + return Fact.find(this.name, FactName.FIREHOSE_CIDR_BLOCK); + } } diff --git a/packages/aws-cdk-lib/package.json b/packages/aws-cdk-lib/package.json index 8761fbbf8e44e..8451b4e93d3dc 100644 --- a/packages/aws-cdk-lib/package.json +++ b/packages/aws-cdk-lib/package.json @@ -239,6 +239,7 @@ "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-kinesisfirehose-destinations": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", diff --git a/packages/decdk/package.json b/packages/decdk/package.json index d3db39216e69b..3f80405b71cba 100644 --- a/packages/decdk/package.json +++ b/packages/decdk/package.json @@ -146,6 +146,7 @@ "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-kinesisfirehose-destinations": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", diff --git a/packages/monocdk/package.json b/packages/monocdk/package.json index a075a584e92bd..159d3d3e44d47 100644 --- a/packages/monocdk/package.json +++ b/packages/monocdk/package.json @@ -240,6 +240,7 @@ "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-kinesisfirehose-destinations": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", diff --git a/tools/pkglint/lib/rules.ts b/tools/pkglint/lib/rules.ts index a696b43bceabe..add3c58fee4b2 100644 --- a/tools/pkglint/lib/rules.ts +++ b/tools/pkglint/lib/rules.ts @@ -1634,6 +1634,7 @@ export class NoExperimentalDependents extends ValidationRule { ['@aws-cdk/aws-apigatewayv2-integrations', ['@aws-cdk/aws-apigatewayv2']], ['@aws-cdk/aws-apigatewayv2-authorizers', ['@aws-cdk/aws-apigatewayv2']], ['@aws-cdk/aws-events-targets', ['@aws-cdk/aws-kinesisfirehose']], + ['@aws-cdk/aws-kinesisfirehose-destinations', ['@aws-cdk/aws-kinesisfirehose']], ]); private readonly excludedModules = ['@aws-cdk/cloudformation-include']; From dca12f230139d71a599aa4b6b4a9e8f9479e22b8 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Wed, 14 Jul 2021 10:19:59 -0700 Subject: [PATCH 02/52] fix typo in aws-kinesisfirehose-destinations README --- packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md b/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md index efe753ddbd6a6..03ef4657b3f78 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md @@ -16,7 +16,7 @@ This library provides constructs for adding destinations to a Amazon Kinesis Data Firehose -delivery stream. Destinations can be added by specifying the `destination` prop when +delivery stream. Destinations can be added by specifying the `destinations` prop when defining a delivery stream. See [Amazon Kinesis Data Firehose module README](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisfirehose-readme.html) for usage examples. From 6fceebfabad9eb3481aec49bd177f5fa3f8f0733 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Wed, 14 Jul 2021 10:57:24 -0700 Subject: [PATCH 03/52] remove unnecessary deps --- packages/@aws-cdk/aws-kinesisfirehose/package.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index 02d64257f825b..4315a1a46da25 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -84,9 +84,6 @@ "@aws-cdk/aws-cloudwatch": "0.0.0", "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", - "@aws-cdk/aws-kinesis": "0.0.0", - "@aws-cdk/aws-kms": "0.0.0", - "@aws-cdk/aws-lambda": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", @@ -99,9 +96,6 @@ "@aws-cdk/aws-cloudwatch": "0.0.0", "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", - "@aws-cdk/aws-kinesis": "0.0.0", - "@aws-cdk/aws-kms": "0.0.0", - "@aws-cdk/aws-lambda": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", From 78e6a9cebd68d4c799f7ea11d4f2ccfa07a2f784 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 09:51:21 -0700 Subject: [PATCH 04/52] remove awslint exception for IConnectable.connections --- packages/@aws-cdk/aws-ec2/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/aws-ec2/package.json b/packages/@aws-cdk/aws-ec2/package.json index 52dede95bb318..3e03e4b60fd6d 100644 --- a/packages/@aws-cdk/aws-ec2/package.json +++ b/packages/@aws-cdk/aws-ec2/package.json @@ -288,7 +288,6 @@ "props-default-doc:@aws-cdk/aws-ec2.AclPortRange.from", "props-default-doc:@aws-cdk/aws-ec2.AclPortRange.to", "docs-public-apis:@aws-cdk/aws-ec2.ConnectionRule", - "docs-public-apis:@aws-cdk/aws-ec2.IConnectable.connections", "docs-public-apis:@aws-cdk/aws-ec2.IInstance", "docs-public-apis:@aws-cdk/aws-ec2.IPrivateSubnet", "docs-public-apis:@aws-cdk/aws-ec2.IPublicSubnet", From 5f7a15c2c121190d07fee66f1f605e316065a84d Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 09:53:21 -0700 Subject: [PATCH 05/52] Change name of S3 destination to S3Bucket --- .../lib/{s3.ts => s3-bucket.ts} | 2 +- .../test/{s3-destination.test.ts => s3-bucket.test.ts} | 0 packages/@aws-cdk/aws-kinesisfirehose/README.md | 6 +++--- 3 files changed, 4 insertions(+), 4 deletions(-) rename packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/{s3.ts => s3-bucket.ts} (95%) rename packages/@aws-cdk/aws-kinesisfirehose-destinations/test/{s3-destination.test.ts => s3-bucket.test.ts} (100%) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts similarity index 95% rename from packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3.ts rename to packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 42334c18dfea6..baffbd22f74ca 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -12,7 +12,7 @@ export interface S3Props extends firehose.DestinationProps { } /** * An S3 bucket destination for data from a Kinesis Data Firehose delivery stream. */ -export class S3 extends firehose.DestinationBase { +export class S3Bucket extends firehose.DestinationBase { constructor(private readonly bucket: s3.IBucket, s3Props: S3Props = {}) { super(s3Props); } diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-destination.test.ts rename to packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index f04fe56e8f3da..160c13cca7d3f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -49,7 +49,7 @@ import * as s3 from '@aws-cdk/aws-s3'; const bucket = new s3.Bucket(this, 'Bucket'); new DeliveryStream(this, 'Delivery Stream', { - destinations: [new destinations.S3(bucket)], + destinations: [new destinations.S3Bucket(bucket)], }); ``` @@ -102,7 +102,7 @@ import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; const bucket = new s3.Bucket(this, 'Bucket'); -const s3Destination = new destinations.S3(bucket); +const s3Destination = new destinations.S3Bucket(bucket); new DeliveryStream(this, 'Delivery Stream', { destinations: [s3Destination], @@ -162,7 +162,7 @@ const role = new iam.Role(this, 'Role', { } bucket.grantWrite(role); new DeliveryStream(stack, 'Delivery Stream', { - destinations: [new destinations.S3(bucket)], + destinations: [new destinations.S3Bucket(bucket)], role: role, }); ``` From efe2fe0c6851427810e6a3ed4b4d1c711517bdd2 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 10:59:50 -0700 Subject: [PATCH 06/52] add Role to DestinationBindOptions, update S3 to S3Bucket in tests --- .../aws-kinesisfirehose-destinations/lib/index.ts | 2 +- .../lib/s3-bucket.ts | 5 +++-- .../test/integ.s3-all-properties.ts | 2 +- .../test/integ.s3-basic.ts | 2 +- .../test/s3-bucket.test.ts | 12 ++++++------ .../aws-kinesisfirehose/lib/delivery-stream.ts | 2 +- .../@aws-cdk/aws-kinesisfirehose/lib/destination.ts | 6 ++++++ .../aws-kinesisfirehose/test/destination.test.ts | 12 ++++++------ 8 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts index cb717f27167ea..03d672ad93c9f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts @@ -1 +1 @@ -export * from './s3'; +export * from './s3-bucket'; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index baffbd22f74ca..8860bda8806e3 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -20,7 +20,7 @@ export class S3Bucket extends firehose.DestinationBase { bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { return { properties: { - extendedS3DestinationConfiguration: this.createExtendedS3DestinationConfiguration(scope, options.deliveryStream), + extendedS3DestinationConfiguration: this.createExtendedS3DestinationConfiguration(scope, options.deliveryStream, options.role), }, }; } @@ -28,11 +28,12 @@ export class S3Bucket extends firehose.DestinationBase { private createExtendedS3DestinationConfiguration( scope: Construct, deliveryStream: firehose.IDeliveryStream, + role: iam.IRole, ): CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty { this.bucket.grantReadWrite(deliveryStream); return { cloudWatchLoggingOptions: this.createLoggingOptions(scope, deliveryStream, 'S3Destination'), - roleArn: (deliveryStream.grantPrincipal as iam.IRole).roleArn, + roleArn: role.roleArn, bucketArn: this.bucket.bucketArn, }; } diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts index a0a9569ca91a8..222eaa6c0fb84 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts @@ -19,7 +19,7 @@ const logGroup = new logs.LogGroup(stack, 'LogGroup', { }); new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [new destinations.S3(bucket, { + destinations: [new destinations.S3Bucket(bucket, { logging: true, logGroup: logGroup, })], diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts index 3bc9a03b254ac..74485a6942510 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts @@ -14,7 +14,7 @@ const bucket = new s3.Bucket(stack, 'Bucket', { }); new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [new destinations.S3(bucket)], + destinations: [new destinations.S3Bucket(bucket)], }); app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts index 75863078110e7..718b2ade3cb0f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -22,9 +22,9 @@ describe('S3 destination', () => { }); it('provides defaults when no configuration is provided', () => { - const destination = new firehosedestinations.S3(bucket); + const destination = new firehosedestinations.S3Bucket(bucket); - const destinationProperties = destination.bind(stack, { deliveryStream }).properties; + const destinationProperties = destination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }).properties; expect(stack.resolve(destinationProperties)).toStrictEqual({ extendedS3DestinationConfiguration: { @@ -44,11 +44,11 @@ describe('S3 destination', () => { }); it('allows full configuration', () => { - const destination = new firehosedestinations.S3(bucket, { + const destination = new firehosedestinations.S3Bucket(bucket, { logging: true, }); - const destinationProperties = destination.bind(stack, { deliveryStream }).properties; + const destinationProperties = destination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }).properties; expect(stack.resolve(destinationProperties)).toStrictEqual({ extendedS3DestinationConfiguration: { @@ -68,9 +68,9 @@ describe('S3 destination', () => { }); it('grants read/write access to the bucket', () => { - const destination = new firehosedestinations.S3(bucket); + const destination = new firehosedestinations.S3Bucket(bucket); - destination.bind(stack, { deliveryStream }); + destination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }).properties; expect(stack).toHaveResourceLike('AWS::IAM::Policy', { Roles: ['DeliveryStreamRole'], diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 6a7d40e434507..d4259a0adbe43 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -214,7 +214,7 @@ export class DeliveryStream extends DeliveryStreamBase { if (props.destinations.length !== 1) { throw new Error(`Only one destination is allowed per delivery stream, given ${props.destinations.length}`); } - const destinationConfig = props.destinations[0].bind(this, { deliveryStream: this }); + const destinationConfig = props.destinations[0].bind(this, { deliveryStream: this, role: role }); const resource = new CfnDeliveryStream(this, 'Resource', { deliveryStreamName: props.deliveryStreamName, diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index fecd1094b884b..e7f38475bd0fc 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -1,3 +1,4 @@ +import * as iam from '@aws-cdk/aws-iam'; import * as logs from '@aws-cdk/aws-logs'; import { Construct } from 'constructs'; import { IDeliveryStream } from './delivery-stream'; @@ -21,6 +22,11 @@ export interface DestinationBindOptions { * The delivery stream. */ readonly deliveryStream: IDeliveryStream; + + /** + * The IAM service Role of the delivery stream. + */ + readonly role: iam.IRole; } /** diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts index a5be8f409025d..043be44a0605b 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts @@ -37,7 +37,7 @@ describe('destination', () => { test('creates resources and configuration by default', () => { const testDestination = new LoggingDestination(); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); expect(stack).toHaveResource('AWS::Logs::LogGroup'); expect(stack).toHaveResource('AWS::Logs::LogStream'); @@ -60,7 +60,7 @@ describe('destination', () => { test('does not create resources or configuration if disabled', () => { const testDestination = new LoggingDestination({ logging: false }); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); expect(stack.resolve(testDestinationConfig)).toStrictEqual({ properties: { @@ -72,7 +72,7 @@ describe('destination', () => { test('creates configuration if log group provided', () => { const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); expect(stack.resolve(testDestinationConfig)).toMatchObject({ properties: { @@ -88,13 +88,13 @@ describe('destination', () => { test('throws error if logging disabled but log group provided', () => { const testDestination = new LoggingDestination({ logging: false, logGroup: new logs.LogGroup(stack, 'Log Group') }); - expect(() => testDestination.bind(stack, { deliveryStream })).toThrowError('logging cannot be set to false when logGroup is provided'); + expect(() => testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole })).toThrowError('logging cannot be set to false when logGroup is provided'); }); test('uses provided log group', () => { const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); expect(stack.resolve(testDestinationConfig)).toMatchObject({ @@ -128,7 +128,7 @@ describe('destination', () => { } }(); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream }); + const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); expect(stack.resolve(testDestinationConfig)).toMatchObject({ From 4308f4426acb15f6e71d7647d27e1f50e15b4473 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:03:25 -0700 Subject: [PATCH 07/52] remove helper function to enforce ExtendedS3DestinationConfigurationProperty type and use local var instead --- .../lib/s3-bucket.ts | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 8860bda8806e3..7ee3c9217c076 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -1,4 +1,3 @@ -import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import { CfnDeliveryStream } from '@aws-cdk/aws-kinesisfirehose'; import * as s3 from '@aws-cdk/aws-s3'; @@ -18,23 +17,18 @@ export class S3Bucket extends firehose.DestinationBase { } bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { + + this.bucket.grantReadWrite(options.deliveryStream); + + const s3ExtendedConfig: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty = { + cloudWatchLoggingOptions: this.createLoggingOptions(scope, options.deliveryStream, 'S3Destination'), + roleArn: options.role.roleArn, + bucketArn: this.bucket.bucketArn, + }; return { properties: { - extendedS3DestinationConfiguration: this.createExtendedS3DestinationConfiguration(scope, options.deliveryStream, options.role), + extendedS3DestinationConfiguration: s3ExtendedConfig, }, }; } - - private createExtendedS3DestinationConfiguration( - scope: Construct, - deliveryStream: firehose.IDeliveryStream, - role: iam.IRole, - ): CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty { - this.bucket.grantReadWrite(deliveryStream); - return { - cloudWatchLoggingOptions: this.createLoggingOptions(scope, deliveryStream, 'S3Destination'), - roleArn: role.roleArn, - bucketArn: this.bucket.bucketArn, - }; - } } From 172ab435d79b61cd30bb4065ed9019a2c66fe879 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:04:26 -0700 Subject: [PATCH 08/52] remove lambda dependency --- packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json index 0d5b22fb53352..4106934c60303 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json @@ -64,7 +64,6 @@ }, "license": "Apache-2.0", "devDependencies": { - "@types/aws-lambda": "^8.10.77", "@types/jest": "^26.0.23", "cdk-build-tools": "0.0.0", "cdk-integ-tools": "0.0.0", From c0d85f56ee31596ddfb65f482d0c6ebb06d682f3 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:08:26 -0700 Subject: [PATCH 09/52] update README --- packages/@aws-cdk/aws-kinesisfirehose/README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 160c13cca7d3f..42962640dcf49 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -209,16 +209,13 @@ permissions automatically. However, custom or third-party destinations may requi permissions. In this case, use the delivery stream as an `IGrantable`, as follows: ```ts fixture=with-delivery-stream -/// !hide -const myDestinationResource = { - grantWrite(grantee: IGrantable) {} -} -/// !show -myDestinationResource.grantWrite(deliveryStream); +import * as lambda from '@aws-cdk/aws-lambda'; + +const function = new lambda.Function(...); +function.grantInvoke(deliveryStream); ``` ## Multiple destinations Though the delivery stream allows specifying an array of destinations, only one -destination per delivery stream is currently allowed. This limitation is enforced at -compile time and will throw an error. +destination per delivery stream is currently allowed. \ No newline at end of file From 3d34c4691c3755769433c3d144d2c803d95101e1 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:22:04 -0700 Subject: [PATCH 10/52] remove tagsand ITaggable interface --- packages/@aws-cdk/aws-kinesisfirehose/README.md | 2 +- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 42962640dcf49..a13cedd604688 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -218,4 +218,4 @@ function.grantInvoke(deliveryStream); ## Multiple destinations Though the delivery stream allows specifying an array of destinations, only one -destination per delivery stream is currently allowed. \ No newline at end of file +destination per delivery stream is currently allowed. diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index d4259a0adbe43..25fbeb9f498a6 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -15,7 +15,7 @@ const PUT_RECORD_ACTIONS = [ /** * Represents a Kinesis Data Firehose delivery stream. */ -export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.IConnectable, cdk.ITaggable { +export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.IConnectable { /** * The ARN of the delivery stream. * @@ -62,8 +62,6 @@ export abstract class DeliveryStreamBase extends cdk.Resource implements IDelive */ public readonly connections: ec2.Connections; - public readonly tags = new cdk.TagManager(cdk.TagType.STANDARD, 'AWS::KinesisFirehose::DeliveryStream'); - constructor(scope: Construct, id: string) { super(scope, id); From 5ceff5cadf618da51aa8ac71ad0765a80306d425 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:26:39 -0700 Subject: [PATCH 11/52] use splitArn instead of parseArn --- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 25fbeb9f498a6..e7ef97e89af04 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -178,7 +178,9 @@ export class DeliveryStream extends DeliveryStreamBase { if (!attrs.deliveryStreamName && !attrs.deliveryStreamArn) { throw new Error('Either deliveryStreamName or deliveryStreamArn must be provided in DeliveryStreamAttributes'); } - const deliveryStreamName = attrs.deliveryStreamName ?? cdk.Stack.of(scope).parseArn(attrs.deliveryStreamArn!).resourceName; + const deliveryStreamName = attrs.deliveryStreamName ?? + cdk.Stack.of(scope).splitArn(attrs.deliveryStreamArn!, cdk.ArnFormat.SLASH_RESOURCE_NAME).resourceName; + if (!deliveryStreamName) { throw new Error(`Could not import delivery stream from malformatted ARN ${attrs.deliveryStreamArn}: could not determine resource name`); } From ce4dde14327eaffdc501afaf4bae3f9232077684 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:28:13 -0700 Subject: [PATCH 12/52] update error message for bad input ARN --- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index e7ef97e89af04..85c6506825cb0 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -182,7 +182,7 @@ export class DeliveryStream extends DeliveryStreamBase { cdk.Stack.of(scope).splitArn(attrs.deliveryStreamArn!, cdk.ArnFormat.SLASH_RESOURCE_NAME).resourceName; if (!deliveryStreamName) { - throw new Error(`Could not import delivery stream from malformatted ARN ${attrs.deliveryStreamArn}: could not determine resource name`); + throw new Error(`No delivery stream name found in ARN: '${attrs.deliveryStreamArn}'`); } const deliveryStreamArn = attrs.deliveryStreamArn ?? cdk.Stack.of(scope).formatArn({ service: 'firehose', From 0ed658ff66ea13e16f2494fe4890d195583e16fc Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:29:21 -0700 Subject: [PATCH 13/52] add ArnFormat to formatArn() call --- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 85c6506825cb0..0a618fa4d4291 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -188,6 +188,7 @@ export class DeliveryStream extends DeliveryStreamBase { service: 'firehose', resource: 'deliverystream', resourceName: attrs.deliveryStreamName, + arnFormat: cdk.ArnFormat.SLASH_RESOURCE_NAME, }); class Import extends DeliveryStreamBase { public readonly deliveryStreamName = deliveryStreamName!; From 19255600c91bc7698b8bda74b285e28bd09b84b4 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:37:55 -0700 Subject: [PATCH 14/52] update error message in tests for bad input ARN --- .../@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts index ae76cc9af7e4c..874b80b7862d1 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -270,7 +270,7 @@ describe('delivery stream', () => { test('throws when malformatted ARN', () => { expect(() => firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamArn: 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/' })) - .toThrowError(/Could not import delivery stream from malformatted ARN/); + .toThrowError("No delivery stream name found in ARN: 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/'"); }); test('throws when without name or ARN', () => { From e46c595b14d3aeb21247493798376573efb149a2 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:38:31 -0700 Subject: [PATCH 15/52] pass deliveryStreamName as physicalName through to super constructor --- .../@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 0a618fa4d4291..7fcc2878df3ea 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -62,8 +62,8 @@ export abstract class DeliveryStreamBase extends cdk.Resource implements IDelive */ public readonly connections: ec2.Connections; - constructor(scope: Construct, id: string) { - super(scope, id); + constructor(scope: Construct, id: string, props: cdk.ResourceProps = {}) { + super(scope, id, props); this.connections = setConnections(this); } @@ -205,7 +205,9 @@ export class DeliveryStream extends DeliveryStreamBase { readonly grantPrincipal: iam.IPrincipal; constructor(scope: Construct, id: string, props: DeliveryStreamProps) { - super(scope, id); + super(scope, id, { + physicalName: props.deliveryStreamName, + }); const role = props.role ?? new iam.Role(this, 'Service Role', { assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), From d1463a2ef3555d4d5763642d86b67c554595ddab Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:39:25 -0700 Subject: [PATCH 16/52] move validation to beginning of constructor --- .../@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 7fcc2878df3ea..49ed8b9e8cb23 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -209,14 +209,15 @@ export class DeliveryStream extends DeliveryStreamBase { physicalName: props.deliveryStreamName, }); + if (props.destinations.length !== 1) { + throw new Error(`Only one destination is allowed per delivery stream, given ${props.destinations.length}`); + } + const role = props.role ?? new iam.Role(this, 'Service Role', { assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), }); this.grantPrincipal = role; - if (props.destinations.length !== 1) { - throw new Error(`Only one destination is allowed per delivery stream, given ${props.destinations.length}`); - } const destinationConfig = props.destinations[0].bind(this, { deliveryStream: this, role: role }); const resource = new CfnDeliveryStream(this, 'Resource', { From 1623199cd00f18078bad26b227db3180df422572 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:46:43 -0700 Subject: [PATCH 17/52] simplify createLoggingOptions --- .../aws-kinesisfirehose/lib/destination.ts | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index e7f38475bd0fc..0fd54e88ebe61 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -66,7 +66,7 @@ export interface DestinationProps { * Abstract base class that destination types can extend to benefit from methods that create generic configuration. */ export abstract class DestinationBase implements IDestination { - private logGroups: { [logGroupId: string]: logs.ILogGroup } = {}; + private logGroup?: logs.ILogGroup; constructor(protected readonly props: DestinationProps = {}) { } @@ -77,27 +77,16 @@ export abstract class DestinationBase implements IDestination { deliveryStream: IDeliveryStream, streamId: string, ): CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { - return this._createLoggingOptions(scope, deliveryStream, streamId, 'LogGroup', this.props.logging, this.props.logGroup); - } - - private _createLoggingOptions( - scope: Construct, - deliveryStream: IDeliveryStream, - streamId: string, - logGroupId: string, - logging?: boolean, - propsLogGroup?: logs.ILogGroup, - ): CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { - if (logging === false && propsLogGroup) { + if (this.props.logging === false && this.props.logGroup) { throw new Error('logging cannot be set to false when logGroup is provided'); } - if (logging !== false || propsLogGroup) { - this.logGroups[logGroupId] = this.logGroups[logGroupId] ?? propsLogGroup ?? new logs.LogGroup(scope, logGroupId); - this.logGroups[logGroupId].grantWrite(deliveryStream); + if (this.props.logging !== false || this.props.logGroup) { + this.logGroup = this.logGroup ?? this.props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); + this.logGroup.grantWrite(deliveryStream); return { enabled: true, - logGroupName: this.logGroups[logGroupId].logGroupName, - logStreamName: this.logGroups[logGroupId].addStream(streamId).logStreamName, + logGroupName: this.logGroup.logGroupName, + logStreamName: this.logGroup.addStream(streamId).logStreamName, }; } return undefined; From 16a54e52e4fff53a3304173e710f8a55863288a3 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 15 Jul 2021 11:49:05 -0700 Subject: [PATCH 18/52] remove awslint exclude --- packages/@aws-cdk/aws-kinesisfirehose/package.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index e0f1436fd6655..01afe8f53e2e6 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -113,10 +113,5 @@ }, "publishConfig": { "tag": "latest" - }, - "awslint": { - "exclude": [ - "no-unused-type:@aws-cdk/aws-kinesisfirehose.Compression" - ] } } From b0113924bca921db958a56e0faa04fca00635743 Mon Sep 17 00:00:00 2001 From: Madeline Kusters <80541297+madeline-k@users.noreply.github.com> Date: Fri, 16 Jul 2021 05:43:47 -0700 Subject: [PATCH 19/52] feat(aws-kinesisfirehose): specific metrics functions for DeliveryStream (#15545) Closes #15543 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../@aws-cdk/aws-kinesisfirehose/README.md | 37 ++++++++ .../lib/delivery-stream.ts | 65 +++++++++++++ .../test/delivery-stream.test.ts | 95 +++++++++++++++++++ 3 files changed, 197 insertions(+) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index a13cedd604688..6ceeccc26c44c 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -145,6 +145,43 @@ new DeliveryStream(this, 'Delivery Stream', { See: [Monitoring using CloudWatch Logs](https://docs.aws.amazon.com/firehose/latest/dev/monitoring-with-cloudwatch-logs.html) in the *Kinesis Data Firehose Developer Guide*. +### Metrics + +Kinesis Data Firehose sends metrics to CloudWatch so that you can collect and analyze the +performance of the delivery stream, including data delivery, data ingestion, data +transformation, format conversion, API usage, encryption, and resource usage. You can then +use CloudWatch alarms to alert you, for example, when data freshness (the age of the +oldest record in the delivery stream) exceeds the buffering limit (indicating that data is +not being delivered to your destination), or when the rate of incoming records exceeds the +limit of records per second (indicating data is flowing into your delivery stream faster +than it is configured to process). + +CDK provides methods for accessing delivery stream metrics with default configuration, +such as `metricIncomingBytes`, and `metricIncomingRecords` (see [`IDeliveryStream`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesisfirehose.IDeliveryStream.html) +for a full list). CDK also provides a generic `metric` method that can be used to produce +metric configurations for any metric provided by Kinesis Data Firehose; the configurations +are pre-populated with the correct dimensions for the delivery stream. + +```ts fixture=with-delivery-stream +import * as cloudwatch from '@aws-cdk/aws-cloudwatch'; +// Alarm that triggers when the per-second average of incoming bytes exceeds 90% of the current service limit +const incomingBytesPercentOfLimit = new cloudwatch.MathExpression({ + expression: 'incomingBytes / 300 / bytePerSecLimit', + usingMetrics: { + incomingBytes: deliveryStream.metricIncomingBytes({ statistic: cloudwatch.Statistic.SUM }), + bytePerSecLimit: deliveryStream.metric('BytesPerSecondLimit'), + }, +}); +new Alarm(this, 'Alarm', { + metric: incomingBytesPercentOfLimit, + threshold: 0.9, + evaluationPeriods: 3, +}); +``` + +See: [Monitoring Using CloudWatch Metrics](https://docs.aws.amazon.com/firehose/latest/dev/monitoring-with-cloudwatch-metrics.html) +in the *Kinesis Data Firehose Developer Guide*. + ## Specifying an IAM role The DeliveryStream class automatically creates an IAM role with all the minimum necessary diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 49ed8b9e8cb23..9a2aa81c6f573 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -5,6 +5,7 @@ import * as cdk from '@aws-cdk/core'; import { RegionInfo } from '@aws-cdk/region-info'; import { Construct } from 'constructs'; import { IDestination } from './destination'; +import { FirehoseMetrics } from './kinesisfirehose-canned-metrics.generated'; import { CfnDeliveryStream } from './kinesisfirehose.generated'; const PUT_RECORD_ACTIONS = [ @@ -44,6 +45,43 @@ export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.ICon * Return the given named metric for this delivery stream. */ metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric; + + /** + * Metric for the number of bytes ingested successfully into the delivery stream over the specified time period after throttling. + * + * By default, this metric will be calculated as an average over a period of 5 minutes. + */ + metricIncomingBytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric; + + /** + * Metric for the number of records ingested successfully into the delivery stream over the specified time period after throttling. + * + * By default, this metric will be calculated as an average over a period of 5 minutes. + */ + metricIncomingRecords(props?: cloudwatch.MetricOptions): cloudwatch.Metric; + + /** + * Metric for the number of bytes delivered to Amazon S3 for backup over the specified time period. + * + * By default, this metric will be calculated as an average over a period of 5 minutes. + */ + metricBackupToS3Bytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric; + + /** + * Metric for the age (from getting into Kinesis Data Firehose to now) of the oldest record in Kinesis Data Firehose. + * + * Any record older than this age has been delivered to the Amazon S3 bucket for backup. + * + * By default, this metric will be calculated as an average over a period of 5 minutes. + */ + metricBackupToS3DataFreshness(props?: cloudwatch.MetricOptions): cloudwatch.Metric; + + /** + * Metric for the number of records delivered to Amazon S3 for backup over the specified time period. + * + * By default, this metric will be calculated as an average over a period of 5 minutes. + */ + metricBackupToS3Records(props?: cloudwatch.MetricOptions): cloudwatch.Metric; } /** @@ -90,6 +128,33 @@ export abstract class DeliveryStreamBase extends cdk.Resource implements IDelive ...props, }).attachTo(this); } + + public metricIncomingBytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return this.cannedMetric(FirehoseMetrics.incomingBytesAverage, props); + } + + public metricIncomingRecords(props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return this.cannedMetric(FirehoseMetrics.incomingRecordsAverage, props); + } + + public metricBackupToS3Bytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return this.cannedMetric(FirehoseMetrics.backupToS3BytesAverage, props); + } + + public metricBackupToS3DataFreshness(props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return this.cannedMetric(FirehoseMetrics.backupToS3DataFreshnessAverage, props); + } + + public metricBackupToS3Records(props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return this.cannedMetric(FirehoseMetrics.backupToS3RecordsAverage, props); + } + + private cannedMetric(fn: (dims: { DeliveryStreamName: string }) => cloudwatch.MetricProps, props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return new cloudwatch.Metric({ + ...fn({ DeliveryStreamName: this.deliveryStreamName }), + ...props, + }).attachTo(this); + } } /** diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts index 874b80b7862d1..1dd87902bff62 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -1,5 +1,6 @@ import '@aws-cdk/assert-internal/jest'; import { ABSENT } from '@aws-cdk/assert-internal'; +import * as cloudwatch from '@aws-cdk/aws-cloudwatch'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as cdk from '@aws-cdk/core'; @@ -156,6 +157,100 @@ describe('delivery stream', () => { }, }); }); + test('metricIncomingBytes', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + const metric = deliveryStream.metricIncomingBytes(); + + expect(metric).toMatchObject({ + account: stack.account, + region: stack.region, + namespace: 'AWS/Firehose', + metricName: 'IncomingBytes', + statistic: cloudwatch.Statistic.AVERAGE, + dimensions: { + DeliveryStreamName: deliveryStream.deliveryStreamName, + }, + }); + }); + + test('metricIncomingRecords', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + const metric = deliveryStream.metricIncomingRecords(); + + expect(metric).toMatchObject({ + account: stack.account, + region: stack.region, + namespace: 'AWS/Firehose', + metricName: 'IncomingRecords', + statistic: cloudwatch.Statistic.AVERAGE, + dimensions: { + DeliveryStreamName: deliveryStream.deliveryStreamName, + }, + }); + }); + + test('metricBackupToS3Bytes', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + const metric = deliveryStream.metricBackupToS3Bytes(); + + expect(metric).toMatchObject({ + account: stack.account, + region: stack.region, + namespace: 'AWS/Firehose', + metricName: 'BackupToS3.Bytes', + statistic: cloudwatch.Statistic.AVERAGE, + dimensions: { + DeliveryStreamName: deliveryStream.deliveryStreamName, + }, + }); + }); + + test('metricBackupToS3DataFreshness', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + const metric = deliveryStream.metricBackupToS3DataFreshness(); + + expect(metric).toMatchObject({ + account: stack.account, + region: stack.region, + namespace: 'AWS/Firehose', + metricName: 'BackupToS3.DataFreshness', + statistic: cloudwatch.Statistic.AVERAGE, + dimensions: { + DeliveryStreamName: deliveryStream.deliveryStreamName, + }, + }); + }); + + test('metricBackupToS3Records', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + const metric = deliveryStream.metricBackupToS3Records(); + + expect(metric).toMatchObject({ + account: stack.account, + region: stack.region, + namespace: 'AWS/Firehose', + metricName: 'BackupToS3.Records', + statistic: cloudwatch.Statistic.AVERAGE, + dimensions: { + DeliveryStreamName: deliveryStream.deliveryStreamName, + }, + }); + }); }); test('allows connections for Firehose IP addresses using map when region not specified', () => { From 56aa16a6594c5b615fbf7bc1ba93c783779993ff Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 17:14:30 -0700 Subject: [PATCH 20/52] change name of s3 destination props to S3BucketProps --- .../aws-kinesisfirehose-destinations/lib/s3-bucket.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 7ee3c9217c076..4a326146f68a3 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -6,13 +6,13 @@ import { Construct } from 'constructs'; /** * Props for defining an S3 destination of a Kinesis Data Firehose delivery stream. */ -export interface S3Props extends firehose.DestinationProps { } +export interface S3BucketProps extends firehose.DestinationProps { } /** * An S3 bucket destination for data from a Kinesis Data Firehose delivery stream. */ export class S3Bucket extends firehose.DestinationBase { - constructor(private readonly bucket: s3.IBucket, s3Props: S3Props = {}) { + constructor(private readonly bucket: s3.IBucket, s3Props: S3BucketProps = {}) { super(s3Props); } From f70fc31744dcbfdb332d920fd02c650a4b5b7987 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 17:14:46 -0700 Subject: [PATCH 21/52] remove basic s3 integration test --- .../test/integ.s3-basic.expected.json | 392 ------------------ .../test/integ.s3-basic.ts | 20 - 2 files changed, 412 deletions(-) delete mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json delete mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json deleted file mode 100644 index 49f6c015db745..0000000000000 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.expected.json +++ /dev/null @@ -1,392 +0,0 @@ -{ - "Resources": { - "Bucket83908E77": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" - }, - "BucketPolicyE9A3008A": { - "Type": "AWS::S3::BucketPolicy", - "Properties": { - "Bucket": { - "Ref": "Bucket83908E77" - }, - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*" - ], - "Effect": "Allow", - "Principal": { - "AWS": { - "Fn::GetAtt": [ - "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", - "Arn" - ] - } - }, - "Resource": [ - { - "Fn::GetAtt": [ - "Bucket83908E77", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "Bucket83908E77", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "BucketAutoDeleteObjectsCustomResourceBAFD23C2": { - "Type": "Custom::S3AutoDeleteObjects", - "Properties": { - "ServiceToken": { - "Fn::GetAtt": [ - "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", - "Arn" - ] - }, - "BucketName": { - "Ref": "Bucket83908E77" - } - }, - "DependsOn": [ - "BucketPolicyE9A3008A" - ], - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" - }, - "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ] - }, - "ManagedPolicyArns": [ - { - "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - } - ] - } - }, - "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": { - "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B" - }, - "S3Key": { - "Fn::Join": [ - "", - [ - { - "Fn::Select": [ - 0, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" - } - ] - } - ] - }, - { - "Fn::Select": [ - 1, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" - } - ] - } - ] - } - ] - ] - } - }, - "Timeout": 900, - "MemorySize": 128, - "Handler": "__entrypoint__.handler", - "Role": { - "Fn::GetAtt": [ - "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", - "Arn" - ] - }, - "Runtime": "nodejs12.x", - "Description": { - "Fn::Join": [ - "", - [ - "Lambda function for auto-deleting objects in ", - { - "Ref": "Bucket83908E77" - }, - " S3 bucket." - ] - ] - } - }, - "DependsOn": [ - "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" - ] - }, - "DeliveryStreamServiceRole964EEBCC": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "firehose.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - } - } - }, - "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:Abort*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "Bucket83908E77", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "Bucket83908E77", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "DeliveryStreamLogGroup9D8FA3BB", - "Arn" - ] - } - } - ], - "Version": "2012-10-17" - }, - "PolicyName": "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", - "Roles": [ - { - "Ref": "DeliveryStreamServiceRole964EEBCC" - } - ] - } - }, - "DeliveryStreamLogGroup9D8FA3BB": { - "Type": "AWS::Logs::LogGroup", - "Properties": { - "RetentionInDays": 731 - }, - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" - }, - "DeliveryStreamLogGroupS3DestinationE25573DB": { - "Type": "AWS::Logs::LogStream", - "Properties": { - "LogGroupName": { - "Ref": "DeliveryStreamLogGroup9D8FA3BB" - } - }, - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" - }, - "DeliveryStreamF6D5572D": { - "Type": "AWS::KinesisFirehose::DeliveryStream", - "Properties": { - "DeliveryStreamType": "DirectPut", - "ExtendedS3DestinationConfiguration": { - "BucketARN": { - "Fn::GetAtt": [ - "Bucket83908E77", - "Arn" - ] - }, - "CloudWatchLoggingOptions": { - "Enabled": true, - "LogGroupName": { - "Ref": "DeliveryStreamLogGroup9D8FA3BB" - }, - "LogStreamName": { - "Ref": "DeliveryStreamLogGroupS3DestinationE25573DB" - } - }, - "RoleARN": { - "Fn::GetAtt": [ - "DeliveryStreamServiceRole964EEBCC", - "Arn" - ] - } - } - }, - "DependsOn": [ - "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", - "DeliveryStreamServiceRole964EEBCC" - ] - } - }, - "Parameters": { - "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B": { - "Type": "String", - "Description": "S3 bucket for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" - }, - "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51": { - "Type": "String", - "Description": "S3 key for asset version \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" - }, - "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1ArtifactHash9ECACDFD": { - "Type": "String", - "Description": "Artifact hash for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" - } - }, - "Mappings": { - "DeliveryStreamFirehoseCIDRMappingE9233479": { - "af-south-1": { - "FirehoseCidrBlock": "13.244.121.224/27" - }, - "ap-east-1": { - "FirehoseCidrBlock": "18.162.221.32/27" - }, - "ap-northeast-1": { - "FirehoseCidrBlock": "13.113.196.224/27" - }, - "ap-northeast-2": { - "FirehoseCidrBlock": "13.209.1.64/27" - }, - "ap-northeast-3": { - "FirehoseCidrBlock": "13.208.177.192/27" - }, - "ap-south-1": { - "FirehoseCidrBlock": "13.232.67.32/27" - }, - "ap-southeast-1": { - "FirehoseCidrBlock": "13.228.64.192/27" - }, - "ap-southeast-2": { - "FirehoseCidrBlock": "13.210.67.224/27" - }, - "ca-central-1": { - "FirehoseCidrBlock": "35.183.92.128/27" - }, - "cn-north-1": { - "FirehoseCidrBlock": "52.81.151.32/27" - }, - "cn-northwest-1": { - "FirehoseCidrBlock": "161.189.23.64/27" - }, - "eu-central-1": { - "FirehoseCidrBlock": "35.158.127.160/27" - }, - "eu-north-1": { - "FirehoseCidrBlock": "13.53.63.224/27" - }, - "eu-south-1": { - "FirehoseCidrBlock": "15.161.135.128/27" - }, - "eu-west-1": { - "FirehoseCidrBlock": "52.19.239.192/27" - }, - "eu-west-2": { - "FirehoseCidrBlock": "18.130.1.96/27" - }, - "eu-west-3": { - "FirehoseCidrBlock": "35.180.1.96/27" - }, - "me-south-1": { - "FirehoseCidrBlock": "15.185.91.0/27" - }, - "sa-east-1": { - "FirehoseCidrBlock": "18.228.1.128/27" - }, - "us-east-1": { - "FirehoseCidrBlock": "52.70.63.192/27" - }, - "us-east-2": { - "FirehoseCidrBlock": "13.58.135.96/27" - }, - "us-gov-east-1": { - "FirehoseCidrBlock": "18.253.138.96/27" - }, - "us-gov-west-1": { - "FirehoseCidrBlock": "52.61.204.160/27" - }, - "us-west-1": { - "FirehoseCidrBlock": "13.57.135.192/27" - }, - "us-west-2": { - "FirehoseCidrBlock": "52.89.255.224/27" - } - } - } -} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts deleted file mode 100644 index 74485a6942510..0000000000000 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-basic.ts +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env node -import * as firehose from '@aws-cdk/aws-kinesisfirehose'; -import * as s3 from '@aws-cdk/aws-s3'; -import * as cdk from '@aws-cdk/core'; -import * as destinations from '../lib'; - -const app = new cdk.App(); - -const stack = new cdk.Stack(app, 'aws-cdk-firehose-delivery-stream-s3-basic'); - -const bucket = new s3.Bucket(stack, 'Bucket', { - removalPolicy: cdk.RemovalPolicy.DESTROY, - autoDeleteObjects: true, -}); - -new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [new destinations.S3Bucket(bucket)], -}); - -app.synth(); \ No newline at end of file From 73f1777a3cb121d2b225b6521e99130e230431a9 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 17:17:04 -0700 Subject: [PATCH 22/52] rename integration tests --- ...all-properties.expected.json => integ.s3-bucket.expected.json} | 0 .../test/{integ.s3-all-properties.ts => integ.s3-bucket.ts} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/@aws-cdk/aws-kinesisfirehose-destinations/test/{integ.s3-all-properties.expected.json => integ.s3-bucket.expected.json} (100%) rename packages/@aws-cdk/aws-kinesisfirehose-destinations/test/{integ.s3-all-properties.ts => integ.s3-bucket.ts} (100%) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json similarity index 100% rename from packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.expected.json rename to packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.ts similarity index 100% rename from packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-all-properties.ts rename to packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.ts From 314c4ca2d96620c0ddabbaeccead0210a61bc125 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 17:19:15 -0700 Subject: [PATCH 23/52] de-exports DeliveryStreamBase --- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 9a2aa81c6f573..c33289da817ab 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -87,7 +87,7 @@ export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.ICon /** * Base class for new and imported Kinesis Data Firehose delivery streams. */ -export abstract class DeliveryStreamBase extends cdk.Resource implements IDeliveryStream { +abstract class DeliveryStreamBase extends cdk.Resource implements IDeliveryStream { public abstract readonly deliveryStreamName: string; From 64131d1310eea02b6ab1f585349742c266c1242d Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Fri, 16 Jul 2021 17:13:41 -0700 Subject: [PATCH 24/52] leverage construct tree to store LogGroup and prevent creating it multiple times --- .../@aws-cdk/aws-kinesisfirehose/lib/destination.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index 0fd54e88ebe61..47f87c1d3a0c2 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -1,6 +1,6 @@ import * as iam from '@aws-cdk/aws-iam'; import * as logs from '@aws-cdk/aws-logs'; -import { Construct } from 'constructs'; +import { Construct, Node } from 'constructs'; import { IDeliveryStream } from './delivery-stream'; import { CfnDeliveryStream } from './kinesisfirehose.generated'; @@ -66,7 +66,6 @@ export interface DestinationProps { * Abstract base class that destination types can extend to benefit from methods that create generic configuration. */ export abstract class DestinationBase implements IDestination { - private logGroup?: logs.ILogGroup; constructor(protected readonly props: DestinationProps = {}) { } @@ -81,12 +80,12 @@ export abstract class DestinationBase implements IDestination { throw new Error('logging cannot be set to false when logGroup is provided'); } if (this.props.logging !== false || this.props.logGroup) { - this.logGroup = this.logGroup ?? this.props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); - this.logGroup.grantWrite(deliveryStream); + const logGroup = Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? this.props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); + logGroup.grantWrite(deliveryStream); return { enabled: true, - logGroupName: this.logGroup.logGroupName, - logStreamName: this.logGroup.addStream(streamId).logStreamName, + logGroupName: logGroup.logGroupName, + logStreamName: logGroup.addStream(streamId).logStreamName, }; } return undefined; From 6c78eeba352b767f3b9aa58be20c744cb7d65aef Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Fri, 16 Jul 2021 17:47:54 -0700 Subject: [PATCH 25/52] Revert "feat(aws-kinesisfirehose): specific metrics functions for DeliveryStream (#15545)" This reverts commit b0113924bca921db958a56e0faa04fca00635743. --- .../@aws-cdk/aws-kinesisfirehose/README.md | 37 -------- .../lib/delivery-stream.ts | 65 ------------- .../test/delivery-stream.test.ts | 95 ------------------- 3 files changed, 197 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 6ceeccc26c44c..a13cedd604688 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -145,43 +145,6 @@ new DeliveryStream(this, 'Delivery Stream', { See: [Monitoring using CloudWatch Logs](https://docs.aws.amazon.com/firehose/latest/dev/monitoring-with-cloudwatch-logs.html) in the *Kinesis Data Firehose Developer Guide*. -### Metrics - -Kinesis Data Firehose sends metrics to CloudWatch so that you can collect and analyze the -performance of the delivery stream, including data delivery, data ingestion, data -transformation, format conversion, API usage, encryption, and resource usage. You can then -use CloudWatch alarms to alert you, for example, when data freshness (the age of the -oldest record in the delivery stream) exceeds the buffering limit (indicating that data is -not being delivered to your destination), or when the rate of incoming records exceeds the -limit of records per second (indicating data is flowing into your delivery stream faster -than it is configured to process). - -CDK provides methods for accessing delivery stream metrics with default configuration, -such as `metricIncomingBytes`, and `metricIncomingRecords` (see [`IDeliveryStream`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesisfirehose.IDeliveryStream.html) -for a full list). CDK also provides a generic `metric` method that can be used to produce -metric configurations for any metric provided by Kinesis Data Firehose; the configurations -are pre-populated with the correct dimensions for the delivery stream. - -```ts fixture=with-delivery-stream -import * as cloudwatch from '@aws-cdk/aws-cloudwatch'; -// Alarm that triggers when the per-second average of incoming bytes exceeds 90% of the current service limit -const incomingBytesPercentOfLimit = new cloudwatch.MathExpression({ - expression: 'incomingBytes / 300 / bytePerSecLimit', - usingMetrics: { - incomingBytes: deliveryStream.metricIncomingBytes({ statistic: cloudwatch.Statistic.SUM }), - bytePerSecLimit: deliveryStream.metric('BytesPerSecondLimit'), - }, -}); -new Alarm(this, 'Alarm', { - metric: incomingBytesPercentOfLimit, - threshold: 0.9, - evaluationPeriods: 3, -}); -``` - -See: [Monitoring Using CloudWatch Metrics](https://docs.aws.amazon.com/firehose/latest/dev/monitoring-with-cloudwatch-metrics.html) -in the *Kinesis Data Firehose Developer Guide*. - ## Specifying an IAM role The DeliveryStream class automatically creates an IAM role with all the minimum necessary diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index c33289da817ab..339181c7d7c8d 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -5,7 +5,6 @@ import * as cdk from '@aws-cdk/core'; import { RegionInfo } from '@aws-cdk/region-info'; import { Construct } from 'constructs'; import { IDestination } from './destination'; -import { FirehoseMetrics } from './kinesisfirehose-canned-metrics.generated'; import { CfnDeliveryStream } from './kinesisfirehose.generated'; const PUT_RECORD_ACTIONS = [ @@ -45,43 +44,6 @@ export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.ICon * Return the given named metric for this delivery stream. */ metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric; - - /** - * Metric for the number of bytes ingested successfully into the delivery stream over the specified time period after throttling. - * - * By default, this metric will be calculated as an average over a period of 5 minutes. - */ - metricIncomingBytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric; - - /** - * Metric for the number of records ingested successfully into the delivery stream over the specified time period after throttling. - * - * By default, this metric will be calculated as an average over a period of 5 minutes. - */ - metricIncomingRecords(props?: cloudwatch.MetricOptions): cloudwatch.Metric; - - /** - * Metric for the number of bytes delivered to Amazon S3 for backup over the specified time period. - * - * By default, this metric will be calculated as an average over a period of 5 minutes. - */ - metricBackupToS3Bytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric; - - /** - * Metric for the age (from getting into Kinesis Data Firehose to now) of the oldest record in Kinesis Data Firehose. - * - * Any record older than this age has been delivered to the Amazon S3 bucket for backup. - * - * By default, this metric will be calculated as an average over a period of 5 minutes. - */ - metricBackupToS3DataFreshness(props?: cloudwatch.MetricOptions): cloudwatch.Metric; - - /** - * Metric for the number of records delivered to Amazon S3 for backup over the specified time period. - * - * By default, this metric will be calculated as an average over a period of 5 minutes. - */ - metricBackupToS3Records(props?: cloudwatch.MetricOptions): cloudwatch.Metric; } /** @@ -128,33 +90,6 @@ abstract class DeliveryStreamBase extends cdk.Resource implements IDeliveryStrea ...props, }).attachTo(this); } - - public metricIncomingBytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric { - return this.cannedMetric(FirehoseMetrics.incomingBytesAverage, props); - } - - public metricIncomingRecords(props?: cloudwatch.MetricOptions): cloudwatch.Metric { - return this.cannedMetric(FirehoseMetrics.incomingRecordsAverage, props); - } - - public metricBackupToS3Bytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric { - return this.cannedMetric(FirehoseMetrics.backupToS3BytesAverage, props); - } - - public metricBackupToS3DataFreshness(props?: cloudwatch.MetricOptions): cloudwatch.Metric { - return this.cannedMetric(FirehoseMetrics.backupToS3DataFreshnessAverage, props); - } - - public metricBackupToS3Records(props?: cloudwatch.MetricOptions): cloudwatch.Metric { - return this.cannedMetric(FirehoseMetrics.backupToS3RecordsAverage, props); - } - - private cannedMetric(fn: (dims: { DeliveryStreamName: string }) => cloudwatch.MetricProps, props?: cloudwatch.MetricOptions): cloudwatch.Metric { - return new cloudwatch.Metric({ - ...fn({ DeliveryStreamName: this.deliveryStreamName }), - ...props, - }).attachTo(this); - } } /** diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts index 1dd87902bff62..874b80b7862d1 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -1,6 +1,5 @@ import '@aws-cdk/assert-internal/jest'; import { ABSENT } from '@aws-cdk/assert-internal'; -import * as cloudwatch from '@aws-cdk/aws-cloudwatch'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as cdk from '@aws-cdk/core'; @@ -157,100 +156,6 @@ describe('delivery stream', () => { }, }); }); - test('metricIncomingBytes', () => { - const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [mockS3Destination], - }); - - const metric = deliveryStream.metricIncomingBytes(); - - expect(metric).toMatchObject({ - account: stack.account, - region: stack.region, - namespace: 'AWS/Firehose', - metricName: 'IncomingBytes', - statistic: cloudwatch.Statistic.AVERAGE, - dimensions: { - DeliveryStreamName: deliveryStream.deliveryStreamName, - }, - }); - }); - - test('metricIncomingRecords', () => { - const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [mockS3Destination], - }); - - const metric = deliveryStream.metricIncomingRecords(); - - expect(metric).toMatchObject({ - account: stack.account, - region: stack.region, - namespace: 'AWS/Firehose', - metricName: 'IncomingRecords', - statistic: cloudwatch.Statistic.AVERAGE, - dimensions: { - DeliveryStreamName: deliveryStream.deliveryStreamName, - }, - }); - }); - - test('metricBackupToS3Bytes', () => { - const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [mockS3Destination], - }); - - const metric = deliveryStream.metricBackupToS3Bytes(); - - expect(metric).toMatchObject({ - account: stack.account, - region: stack.region, - namespace: 'AWS/Firehose', - metricName: 'BackupToS3.Bytes', - statistic: cloudwatch.Statistic.AVERAGE, - dimensions: { - DeliveryStreamName: deliveryStream.deliveryStreamName, - }, - }); - }); - - test('metricBackupToS3DataFreshness', () => { - const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [mockS3Destination], - }); - - const metric = deliveryStream.metricBackupToS3DataFreshness(); - - expect(metric).toMatchObject({ - account: stack.account, - region: stack.region, - namespace: 'AWS/Firehose', - metricName: 'BackupToS3.DataFreshness', - statistic: cloudwatch.Statistic.AVERAGE, - dimensions: { - DeliveryStreamName: deliveryStream.deliveryStreamName, - }, - }); - }); - - test('metricBackupToS3Records', () => { - const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { - destinations: [mockS3Destination], - }); - - const metric = deliveryStream.metricBackupToS3Records(); - - expect(metric).toMatchObject({ - account: stack.account, - region: stack.region, - namespace: 'AWS/Firehose', - metricName: 'BackupToS3.Records', - statistic: cloudwatch.Statistic.AVERAGE, - dimensions: { - DeliveryStreamName: deliveryStream.deliveryStreamName, - }, - }); - }); }); test('allows connections for Firehose IP addresses using map when region not specified', () => { From ccd93332c435e26ee4c4ef1593877f4d4aaa7832 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 17:53:30 -0700 Subject: [PATCH 26/52] cleanup s3 destination unit tests to do less hardcoding --- .../test/s3-bucket.test.ts | 103 +++++++++--------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts index 718b2ade3cb0f..b37fb34e1ffda 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -1,6 +1,8 @@ import '@aws-cdk/assert-internal/jest'; +import { ABSENT, anything } from '@aws-cdk/assert-internal'; import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as logs from '@aws-cdk/aws-logs'; import * as s3 from '@aws-cdk/aws-s3'; import * as cdk from '@aws-cdk/core'; import * as firehosedestinations from '../lib'; @@ -9,60 +11,66 @@ describe('S3 destination', () => { let stack: cdk.Stack; let bucket: s3.IBucket; let deliveryStreamRole: iam.IRole; - let deliveryStream: firehose.IDeliveryStream; beforeEach(() => { stack = new cdk.Stack(); - bucket = new s3.Bucket(stack, 'destination'); - deliveryStreamRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DeliveryStreamRole'); - deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'Delivery Stream', { - deliveryStreamName: 'mydeliverystream', - role: deliveryStreamRole, + bucket = new s3.Bucket(stack, 'Bucket'); + deliveryStreamRole = new iam.Role(stack, 'Delivery Stream Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), }); }); it('provides defaults when no configuration is provided', () => { - const destination = new firehosedestinations.S3Bucket(bucket); - - const destinationProperties = destination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }).properties; + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket)], + role: deliveryStreamRole, + }); - expect(stack.resolve(destinationProperties)).toStrictEqual({ - extendedS3DestinationConfiguration: { - bucketArn: stack.resolve(bucket.bucketArn), - cloudWatchLoggingOptions: { - enabled: true, - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupS3Destination70CE1003', - }, + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + BucketARN: stack.resolve(bucket.bucketArn), + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: anything(), + LogStreamName: anything(), }, - roleArn: stack.resolve(deliveryStreamRole.roleArn), + RoleARN: stack.resolve(deliveryStreamRole.roleArn), }, }); + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); }); - it('allows full configuration', () => { - const destination = new firehosedestinations.S3Bucket(bucket, { - logging: true, + it('allows disabling logging', () => { + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { + logging: false, + })], + role: deliveryStreamRole, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: ABSENT, + }, }); + }); - const destinationProperties = destination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }).properties; + it('allows providing a log group', () => { + const logGroup = logs.LogGroup.fromLogGroupName(stack, 'Log Group', 'evergreen'); - expect(stack.resolve(destinationProperties)).toStrictEqual({ - extendedS3DestinationConfiguration: { - bucketArn: stack.resolve(bucket.bucketArn), - cloudWatchLoggingOptions: { - enabled: true, - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupS3Destination70CE1003', - }, + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { + logGroup, + })], + role: deliveryStreamRole, + }); + + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: { + LogGroupName: 'evergreen', }, - roleArn: deliveryStreamRole.roleArn, }, }); }); @@ -70,10 +78,13 @@ describe('S3 destination', () => { it('grants read/write access to the bucket', () => { const destination = new firehosedestinations.S3Bucket(bucket); - destination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }).properties; + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [destination], + role: deliveryStreamRole, + }); expect(stack).toHaveResourceLike('AWS::IAM::Policy', { - Roles: ['DeliveryStreamRole'], + Roles: [stack.resolve(deliveryStreamRole.roleName)], PolicyDocument: { Statement: [ { @@ -87,22 +98,12 @@ describe('S3 destination', () => { ], Effect: 'Allow', Resource: [ - { - 'Fn::GetAtt': [ - 'destinationDB878FB5', - 'Arn', - ], - }, + stack.resolve(bucket.bucketArn), { 'Fn::Join': [ '', [ - { - 'Fn::GetAtt': [ - 'destinationDB878FB5', - 'Arn', - ], - }, + stack.resolve(bucket.bucketArn), '/*', ], ], From e37d8eacef4875833f4a0adcdb79aeb28adecc54 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 18:24:49 -0700 Subject: [PATCH 27/52] make snippets compile and enforce strict compilation --- .../package.json | 9 +++++- .../@aws-cdk/aws-kinesisfirehose/README.md | 31 +++++++++++++------ .../@aws-cdk/aws-kinesisfirehose/package.json | 9 +++++- .../rosetta/with-bucket.ts-fixture | 13 ++++++++ .../rosetta/with-delivery-stream.ts-fixture | 12 +++++++ .../rosetta/with-destination.ts-fixture | 12 +++++++ 6 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json index 4106934c60303..7411521bb2f93 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json @@ -28,7 +28,14 @@ ] } }, - "projectReferences": true + "projectReferences": true, + "metadata": { + "jsii": { + "rosetta": { + "strict": true + } + } + } }, "repository": { "type": "git", diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index a13cedd604688..66dc007849eb4 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -123,21 +123,28 @@ and LogStream for your Delivery Stream. You can provide a specific log group to specify where the CDK will create the log streams where log events will be sent: -```ts fixture=with-destination +```ts fixture=with-bucket +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; import * as logs from '@aws-cdk/aws-logs'; const logGroup = new logs.LogGroup(this, 'Log Group'); -new DeliveryStream(this, 'Delivery Stream', { +const destination = new destinations.S3Bucket(bucket, { logGroup: logGroup, +}); +new DeliveryStream(this, 'Delivery Stream', { destinations: [destination], }); ``` Logging can also be disabled: -```ts fixture=with-destination +```ts fixture=with-bucket +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; + +const destination = new destinations.S3Bucket(bucket, { + logging: false, +}); new DeliveryStream(this, 'Delivery Stream', { - loggingEnabled: false, destinations: [destination], }); ``` @@ -155,13 +162,14 @@ specify your own IAM role. It must have the correct permissions, or delivery str creation or data delivery may fail. ```ts fixture=with-bucket +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; import * as iam from '@aws-cdk/aws-iam'; const role = new iam.Role(this, 'Role', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), -} +}); bucket.grantWrite(role); -new DeliveryStream(stack, 'Delivery Stream', { +new DeliveryStream(this, 'Delivery Stream', { destinations: [new destinations.S3Bucket(bucket)], role: role, }); @@ -187,7 +195,7 @@ can be granted permissions to a delivery stream by calling: import * as iam from '@aws-cdk/aws-iam'; const lambdaRole = new iam.Role(this, 'Role', { assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), -} +}); // Give the role permissions to write data to the delivery stream deliveryStream.grantPutRecords(lambdaRole); @@ -211,8 +219,13 @@ permissions. In this case, use the delivery stream as an `IGrantable`, as follow ```ts fixture=with-delivery-stream import * as lambda from '@aws-cdk/aws-lambda'; -const function = new lambda.Function(...); -function.grantInvoke(deliveryStream); +const fn = new lambda.Function(this, 'Function', { + code: lambda.Code.fromInline('exports.handler = (event) => {}'), + runtime: lambda.Runtime.NODEJS_14_X, + handler: 'index.handler', +}); + +fn.grantInvoke(deliveryStream); ``` ## Multiple destinations diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index 01afe8f53e2e6..9a74a96b1a340 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -28,7 +28,14 @@ ] } }, - "projectReferences": true + "projectReferences": true, + "metadata": { + "jsii": { + "rosetta": { + "strict": true + } + } + } }, "repository": { "type": "git", diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture new file mode 100644 index 0000000000000..d0851cff49639 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture @@ -0,0 +1,13 @@ +// Fixture with a bucket already created +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; +import * as s3 from '@aws-cdk/aws-s3'; +declare const bucket: s3.Bucket; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture new file mode 100644 index 0000000000000..c7b75b20d2c1b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture @@ -0,0 +1,12 @@ +// Fixture with a delivery stream already created +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; +declare const deliveryStream: DeliveryStream; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture new file mode 100644 index 0000000000000..37d78bf7a43d3 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture @@ -0,0 +1,12 @@ +// Fixture with a destination already created +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; +declare const destination: IDestination; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} From cd1618b0aa0cec7347f143aa1df7fa1c3512b4ca Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 18:38:34 -0700 Subject: [PATCH 28/52] remove IDeliveryStream from DestinationBindOptions, only service role needed --- .../lib/s3-bucket.ts | 4 ++-- .../lib/delivery-stream.ts | 2 +- .../aws-kinesisfirehose/lib/destination.ts | 9 ++------ .../test/destination.test.ts | 23 ++++++++----------- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 4a326146f68a3..303f48db31460 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -18,10 +18,10 @@ export class S3Bucket extends firehose.DestinationBase { bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { - this.bucket.grantReadWrite(options.deliveryStream); + this.bucket.grantReadWrite(options.role); const s3ExtendedConfig: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty = { - cloudWatchLoggingOptions: this.createLoggingOptions(scope, options.deliveryStream, 'S3Destination'), + cloudWatchLoggingOptions: this.createLoggingOptions(scope, options.role, 'S3Destination'), roleArn: options.role.roleArn, bucketArn: this.bucket.bucketArn, }; diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 339181c7d7c8d..25627dfcfc43f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -218,7 +218,7 @@ export class DeliveryStream extends DeliveryStreamBase { }); this.grantPrincipal = role; - const destinationConfig = props.destinations[0].bind(this, { deliveryStream: this, role: role }); + const destinationConfig = props.destinations[0].bind(this, { role: role }); const resource = new CfnDeliveryStream(this, 'Resource', { deliveryStreamName: props.deliveryStreamName, diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index 47f87c1d3a0c2..04615195c9669 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -1,7 +1,6 @@ import * as iam from '@aws-cdk/aws-iam'; import * as logs from '@aws-cdk/aws-logs'; import { Construct, Node } from 'constructs'; -import { IDeliveryStream } from './delivery-stream'; import { CfnDeliveryStream } from './kinesisfirehose.generated'; /** @@ -18,10 +17,6 @@ export interface DestinationConfig { * Options when binding a destination to a delivery stream. */ export interface DestinationBindOptions { - /** - * The delivery stream. - */ - readonly deliveryStream: IDeliveryStream; /** * The IAM service Role of the delivery stream. @@ -73,7 +68,7 @@ export abstract class DestinationBase implements IDestination { protected createLoggingOptions( scope: Construct, - deliveryStream: IDeliveryStream, + serviceRole: iam.IRole, streamId: string, ): CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { if (this.props.logging === false && this.props.logGroup) { @@ -81,7 +76,7 @@ export abstract class DestinationBase implements IDestination { } if (this.props.logging !== false || this.props.logGroup) { const logGroup = Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? this.props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); - logGroup.grantWrite(deliveryStream); + logGroup.grantWrite(serviceRole); return { enabled: true, logGroupName: logGroup.logGroupName, diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts index 043be44a0605b..1170030c9b401 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts @@ -8,17 +8,12 @@ import * as firehose from '../lib'; describe('destination', () => { let stack: cdk.Stack; let deliveryStreamRole: iam.IRole; - let deliveryStream: firehose.IDeliveryStream; const deliveryStreamRoleArn = 'arn:aws:iam::111122223333:role/DeliveryStreamRole'; beforeEach(() => { stack = new cdk.Stack(); deliveryStreamRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', deliveryStreamRoleArn); - deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'Delivery Stream', { - deliveryStreamName: 'mydeliverystream', - role: deliveryStreamRole, - }); }); describe('createLoggingOptions', () => { @@ -27,7 +22,7 @@ describe('destination', () => { return { properties: { testDestinationConfig: { - loggingConfig: this.createLoggingOptions(scope, options.deliveryStream, 'streamId'), + loggingConfig: this.createLoggingOptions(scope, options.role, 'streamId'), }, }, }; @@ -37,7 +32,7 @@ describe('destination', () => { test('creates resources and configuration by default', () => { const testDestination = new LoggingDestination(); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); expect(stack).toHaveResource('AWS::Logs::LogGroup'); expect(stack).toHaveResource('AWS::Logs::LogStream'); @@ -60,7 +55,7 @@ describe('destination', () => { test('does not create resources or configuration if disabled', () => { const testDestination = new LoggingDestination({ logging: false }); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); expect(stack.resolve(testDestinationConfig)).toStrictEqual({ properties: { @@ -72,7 +67,7 @@ describe('destination', () => { test('creates configuration if log group provided', () => { const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); expect(stack.resolve(testDestinationConfig)).toMatchObject({ properties: { @@ -88,13 +83,13 @@ describe('destination', () => { test('throws error if logging disabled but log group provided', () => { const testDestination = new LoggingDestination({ logging: false, logGroup: new logs.LogGroup(stack, 'Log Group') }); - expect(() => testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole })).toThrowError('logging cannot be set to false when logGroup is provided'); + expect(() => testDestination.bind(stack, { role: deliveryStreamRole })).toThrowError('logging cannot be set to false when logGroup is provided'); }); test('uses provided log group', () => { const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); expect(stack.resolve(testDestinationConfig)).toMatchObject({ @@ -120,15 +115,15 @@ describe('destination', () => { return { properties: { testDestinationConfig: { - loggingConfig: this.createLoggingOptions(scope, options.deliveryStream, 'streamId'), - anotherLoggingConfig: this.createLoggingOptions(scope, options.deliveryStream, 'anotherStreamId'), + loggingConfig: this.createLoggingOptions(scope, options.role, 'streamId'), + anotherLoggingConfig: this.createLoggingOptions(scope, options.role, 'anotherStreamId'), }, }, }; } }(); - const testDestinationConfig = testDestination.bind(stack, { deliveryStream: deliveryStream, role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); expect(stack.resolve(testDestinationConfig)).toMatchObject({ From bfe6dd4e05afe91fb5f3fbd8f15c656d8c114df9 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 18:43:35 -0700 Subject: [PATCH 29/52] remove explicit dependency delivery stream -> service role unneeded for this PR and also should set dependencies on individual grants --- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 25627dfcfc43f..0048456cdcd52 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -225,7 +225,6 @@ export class DeliveryStream extends DeliveryStreamBase { deliveryStreamType: 'DirectPut', ...destinationConfig.properties, }); - resource.node.addDependency(this.grantPrincipal); this.deliveryStreamArn = this.getResourceArnAttribute(resource.attrArn, { service: 'kinesis', From dcd088d9f6ac944cf1fa53abdf506abe2e188b50 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 19:01:09 -0700 Subject: [PATCH 30/52] Revert "remove explicit dependency delivery stream -> service role" This reverts commit bfe6dd4e05afe91fb5f3fbd8f15c656d8c114df9. --- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 0048456cdcd52..25627dfcfc43f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -225,6 +225,7 @@ export class DeliveryStream extends DeliveryStreamBase { deliveryStreamType: 'DirectPut', ...destinationConfig.properties, }); + resource.node.addDependency(this.grantPrincipal); this.deliveryStreamArn = this.getResourceArnAttribute(resource.attrArn, { service: 'kinesis', From 0a7e010be0da72884a958b2f3a26647e368382a5 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Fri, 16 Jul 2021 19:52:44 -0700 Subject: [PATCH 31/52] destination has separate role --- .../lib/s3-bucket.ts | 18 ++++---- .../test/integ.s3-bucket.expected.json | 36 ++++++++++++---- .../test/s3-bucket.test.ts | 42 ++++++++++++++----- .../lib/delivery-stream.ts | 8 ++-- .../aws-kinesisfirehose/lib/destination.ts | 19 +++++---- .../test/destination.test.ts | 28 ++++++------- .../test/integ.delivery-stream.expected.json | 5 +-- 7 files changed, 101 insertions(+), 55 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 303f48db31460..552fe018f78e8 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -1,7 +1,7 @@ +import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; -import { CfnDeliveryStream } from '@aws-cdk/aws-kinesisfirehose'; import * as s3 from '@aws-cdk/aws-s3'; -import { Construct } from 'constructs'; +import { Construct, Node } from 'constructs'; /** * Props for defining an S3 destination of a Kinesis Data Firehose delivery stream. @@ -16,13 +16,17 @@ export class S3Bucket extends firehose.DestinationBase { super(s3Props); } - bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { + bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + const role = this.props.role ?? new iam.Role(scope, 'S3 Destination Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); - this.bucket.grantReadWrite(options.role); + const bucketGrant = this.bucket.grantReadWrite(role); + Node.of(scope).addDependency(bucketGrant); - const s3ExtendedConfig: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty = { - cloudWatchLoggingOptions: this.createLoggingOptions(scope, options.role, 'S3Destination'), - roleArn: options.role.roleArn, + const s3ExtendedConfig: firehose.CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty = { + cloudWatchLoggingOptions: this.createLoggingOptions(scope, role, 'S3Destination'), + roleArn: role.roleArn, bucketArn: this.bucket.bucketArn, }; return { diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json index 0d60c11048969..c3e33cb73f21b 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json @@ -198,9 +198,32 @@ ], "Version": "2012-10-17" } - } + }, + "DependsOn": [ + "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7" + ] + }, + "DeliveryStreamS3DestinationRole500FC089": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "DependsOn": [ + "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7" + ] }, - "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF": { + "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { @@ -254,10 +277,10 @@ ], "Version": "2012-10-17" }, - "PolicyName": "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", + "PolicyName": "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7", "Roles": [ { - "Ref": "DeliveryStreamServiceRole964EEBCC" + "Ref": "DeliveryStreamS3DestinationRole500FC089" } ] } @@ -284,15 +307,14 @@ }, "RoleARN": { "Fn::GetAtt": [ - "DeliveryStreamServiceRole964EEBCC", + "DeliveryStreamS3DestinationRole500FC089", "Arn" ] } } }, "DependsOn": [ - "DeliveryStreamServiceRoleDefaultPolicyB87D9ACF", - "DeliveryStreamServiceRole964EEBCC" + "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7" ] } }, diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts index b37fb34e1ffda..748b55bc8fb24 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -1,5 +1,5 @@ import '@aws-cdk/assert-internal/jest'; -import { ABSENT, anything } from '@aws-cdk/assert-internal'; +import { ABSENT, Capture, anything, MatchStyle } from '@aws-cdk/assert-internal'; import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as logs from '@aws-cdk/aws-logs'; @@ -10,20 +10,19 @@ import * as firehosedestinations from '../lib'; describe('S3 destination', () => { let stack: cdk.Stack; let bucket: s3.IBucket; - let deliveryStreamRole: iam.IRole; + let destinationRole: iam.IRole; beforeEach(() => { stack = new cdk.Stack(); bucket = new s3.Bucket(stack, 'Bucket'); - deliveryStreamRole = new iam.Role(stack, 'Delivery Stream Role', { + destinationRole = new iam.Role(stack, 'Destination Role', { assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), }); }); it('provides defaults when no configuration is provided', () => { new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new firehosedestinations.S3Bucket(bucket)], - role: deliveryStreamRole, + destinations: [new firehosedestinations.S3Bucket(bucket, { role: destinationRole })], }); expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { @@ -34,7 +33,7 @@ describe('S3 destination', () => { LogGroupName: anything(), LogStreamName: anything(), }, - RoleARN: stack.resolve(deliveryStreamRole.roleArn), + RoleARN: stack.resolve(destinationRole.roleArn), }, }); expect(stack).toHaveResource('AWS::Logs::LogGroup'); @@ -46,7 +45,6 @@ describe('S3 destination', () => { destinations: [new firehosedestinations.S3Bucket(bucket, { logging: false, })], - role: deliveryStreamRole, }); expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { @@ -63,7 +61,6 @@ describe('S3 destination', () => { destinations: [new firehosedestinations.S3Bucket(bucket, { logGroup, })], - role: deliveryStreamRole, }); expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { @@ -75,16 +72,39 @@ describe('S3 destination', () => { }); }); + it('creates a role when none is provided', () => { + const capturedRoleArn = Capture.aString(); + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket)], + }); + + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + RoleARN: { + 'Fn::GetAtt': [ + capturedRoleArn.capture(), + 'Arn', + ], + }, + }, + }); + expect(stack).toMatchTemplate({ + [capturedRoleArn.capturedValue]: { + Type: 'AWS::IAM::Role', + }, + }, MatchStyle.SUPERSET); + }); + it('grants read/write access to the bucket', () => { - const destination = new firehosedestinations.S3Bucket(bucket); + const destination = new firehosedestinations.S3Bucket(bucket, { role: destinationRole }); new firehose.DeliveryStream(stack, 'DeliveryStream', { destinations: [destination], - role: deliveryStreamRole, }); expect(stack).toHaveResourceLike('AWS::IAM::Policy', { - Roles: [stack.resolve(deliveryStreamRole.roleName)], + Roles: [stack.resolve(destinationRole.roleName)], PolicyDocument: { Statement: [ { diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 25627dfcfc43f..09d9a74c78100 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -111,7 +111,9 @@ export interface DeliveryStreamProps { readonly deliveryStreamName?: string; /** - * The IAM role assumed by Kinesis Data Firehose to read from sources, invoke processors, and write to destinations. + * The IAM role associated with this delivery stream. + * + * Assumed by Kinesis Data Firehose to read from sources and encrypt data server-side. * * @default - a role will be created with default permissions. */ @@ -140,11 +142,10 @@ export interface DeliveryStreamAttributes { */ readonly deliveryStreamName?: string; - /** * The IAM role associated with this delivery stream. * - * Assumed by Kinesis Data Firehose to read from sources, invoke processors, and write to destinations. + * Assumed by Kinesis Data Firehose to read from sources and encrypt data server-side. * * @default - the imported stream cannot be granted access to other resources as an `iam.IGrantable`. */ @@ -225,7 +226,6 @@ export class DeliveryStream extends DeliveryStreamBase { deliveryStreamType: 'DirectPut', ...destinationConfig.properties, }); - resource.node.addDependency(this.grantPrincipal); this.deliveryStreamArn = this.getResourceArnAttribute(resource.attrArn, { service: 'kinesis', diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index 04615195c9669..de63566a68b14 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -17,11 +17,6 @@ export interface DestinationConfig { * Options when binding a destination to a delivery stream. */ export interface DestinationBindOptions { - - /** - * The IAM service Role of the delivery stream. - */ - readonly role: iam.IRole; } /** @@ -55,6 +50,15 @@ export interface DestinationProps { * @default - if `logging` is set to `true`, a log group will be created for you. */ readonly logGroup?: logs.ILogGroup; + + /** + * The IAM role associated with this destination. + * + * Assumed by Kinesis Data Firehose to invoke processors and write to destinations + * + * @default - a role will be created with default permissions. + */ + readonly role?: iam.IRole; } /** @@ -68,7 +72,7 @@ export abstract class DestinationBase implements IDestination { protected createLoggingOptions( scope: Construct, - serviceRole: iam.IRole, + role: iam.IRole, streamId: string, ): CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { if (this.props.logging === false && this.props.logGroup) { @@ -76,7 +80,8 @@ export abstract class DestinationBase implements IDestination { } if (this.props.logging !== false || this.props.logGroup) { const logGroup = Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? this.props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); - logGroup.grantWrite(serviceRole); + const logGroupGrant = logGroup.grantWrite(role); + Node.of(scope).addDependency(logGroupGrant); return { enabled: true, logGroupName: logGroup.logGroupName, diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts index 1170030c9b401..3332b82c523cd 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts @@ -7,22 +7,20 @@ import * as firehose from '../lib'; describe('destination', () => { let stack: cdk.Stack; - let deliveryStreamRole: iam.IRole; - - const deliveryStreamRoleArn = 'arn:aws:iam::111122223333:role/DeliveryStreamRole'; + let destinationRole: iam.IRole; beforeEach(() => { stack = new cdk.Stack(); - deliveryStreamRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', deliveryStreamRoleArn); + destinationRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DestinationRole'); }); describe('createLoggingOptions', () => { class LoggingDestination extends firehose.DestinationBase { - public bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { + public bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { return { properties: { testDestinationConfig: { - loggingConfig: this.createLoggingOptions(scope, options.role, 'streamId'), + loggingConfig: this.createLoggingOptions(scope, destinationRole, 'streamId'), }, }, }; @@ -32,7 +30,7 @@ describe('destination', () => { test('creates resources and configuration by default', () => { const testDestination = new LoggingDestination(); - const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); expect(stack).toHaveResource('AWS::Logs::LogGroup'); expect(stack).toHaveResource('AWS::Logs::LogStream'); @@ -55,7 +53,7 @@ describe('destination', () => { test('does not create resources or configuration if disabled', () => { const testDestination = new LoggingDestination({ logging: false }); - const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); expect(stack.resolve(testDestinationConfig)).toStrictEqual({ properties: { @@ -67,7 +65,7 @@ describe('destination', () => { test('creates configuration if log group provided', () => { const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); - const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); expect(stack.resolve(testDestinationConfig)).toMatchObject({ properties: { @@ -83,13 +81,13 @@ describe('destination', () => { test('throws error if logging disabled but log group provided', () => { const testDestination = new LoggingDestination({ logging: false, logGroup: new logs.LogGroup(stack, 'Log Group') }); - expect(() => testDestination.bind(stack, { role: deliveryStreamRole })).toThrowError('logging cannot be set to false when logGroup is provided'); + expect(() => testDestination.bind(stack, { role: destinationRole })).toThrowError('logging cannot be set to false when logGroup is provided'); }); test('uses provided log group', () => { const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); - const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); expect(stack.resolve(testDestinationConfig)).toMatchObject({ @@ -111,19 +109,19 @@ describe('destination', () => { test('re-uses log group if called multiple times', () => { const testDestination = new class extends firehose.DestinationBase { - public bind(scope: Construct, options: firehose.DestinationBindOptions): firehose.DestinationConfig { + public bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { return { properties: { testDestinationConfig: { - loggingConfig: this.createLoggingOptions(scope, options.role, 'streamId'), - anotherLoggingConfig: this.createLoggingOptions(scope, options.role, 'anotherStreamId'), + loggingConfig: this.createLoggingOptions(scope, destinationRole, 'streamId'), + anotherLoggingConfig: this.createLoggingOptions(scope, destinationRole, 'anotherStreamId'), }, }, }; } }(); - const testDestinationConfig = testDestination.bind(stack, { role: deliveryStreamRole }); + const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); expect(stack.resolve(testDestinationConfig)).toMatchObject({ diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json index 2d783a749781a..ee2fbab429f40 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json @@ -57,10 +57,7 @@ ] } } - }, - "DependsOn": [ - "DeliveryStreamServiceRole964EEBCC" - ] + } } }, "Mappings": { From c99c2425e6930153da55a46bc4eaca2d59e27ec9 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Mon, 19 Jul 2021 10:07:50 -0700 Subject: [PATCH 32/52] update @types/jest dependency to match all other modules --- packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json index 7411521bb2f93..24da18f9c3947 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json @@ -71,7 +71,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@types/jest": "^26.0.23", + "@types/jest": "^26.0.24", "cdk-build-tools": "0.0.0", "cdk-integ-tools": "0.0.0", "cfn2ts": "0.0.0", From 157d3ca22db617596583938543d3da7fca16cdc2 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Mon, 19 Jul 2021 12:10:31 -0700 Subject: [PATCH 33/52] loose snippet compilation to avoid cyclic dependency --- packages/@aws-cdk/aws-kinesisfirehose/package.json | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index 9a74a96b1a340..01afe8f53e2e6 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -28,14 +28,7 @@ ] } }, - "projectReferences": true, - "metadata": { - "jsii": { - "rosetta": { - "strict": true - } - } - } + "projectReferences": true }, "repository": { "type": "git", From db9ea210cca71e675ba0def4b7e1c528feffd09e Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Mon, 19 Jul 2021 12:20:07 -0700 Subject: [PATCH 34/52] update README --- packages/@aws-cdk/aws-kinesisfirehose/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 66dc007849eb4..0e68ac9a586ce 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -231,4 +231,5 @@ fn.grantInvoke(deliveryStream); ## Multiple destinations Though the delivery stream allows specifying an array of destinations, only one -destination per delivery stream is currently allowed. +destination per delivery stream is currently allowed. This limitation is enforced at CDK +synthesis time and will throw an error. From 49a839eda9387a96c7ec46a77ca2d2a8dc132b57 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Mon, 19 Jul 2021 13:51:43 -0700 Subject: [PATCH 35/52] Use L1 types in DestinationConfig returned from bind() --- .../lib/s3-bucket.ts | 11 +- .../lib/delivery-stream.ts | 2 +- .../aws-kinesisfirehose/lib/destination.ts | 6 +- .../test/delivery-stream.test.ts | 10 +- .../test/destination.test.ts | 154 ++++++++---------- .../test/integ.delivery-stream.expected.json | 2 +- .../test/integ.delivery-stream.ts | 8 +- 7 files changed, 89 insertions(+), 104 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 552fe018f78e8..8cfe9b840bd95 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -24,14 +24,11 @@ export class S3Bucket extends firehose.DestinationBase { const bucketGrant = this.bucket.grantReadWrite(role); Node.of(scope).addDependency(bucketGrant); - const s3ExtendedConfig: firehose.CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty = { - cloudWatchLoggingOptions: this.createLoggingOptions(scope, role, 'S3Destination'), - roleArn: role.roleArn, - bucketArn: this.bucket.bucketArn, - }; return { - properties: { - extendedS3DestinationConfiguration: s3ExtendedConfig, + extendedS3DestinationConfiguration: { + cloudWatchLoggingOptions: this.createLoggingOptions(scope, role, 'S3Destination'), + roleArn: role.roleArn, + bucketArn: this.bucket.bucketArn, }, }; } diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 09d9a74c78100..e1ad4b1ec1ff3 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -224,7 +224,7 @@ export class DeliveryStream extends DeliveryStreamBase { const resource = new CfnDeliveryStream(this, 'Resource', { deliveryStreamName: props.deliveryStreamName, deliveryStreamType: 'DirectPut', - ...destinationConfig.properties, + ...destinationConfig, }); this.deliveryStreamArn = this.getResourceArnAttribute(resource.attrArn, { diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index de63566a68b14..c3e70b88c9a95 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -8,9 +8,11 @@ import { CfnDeliveryStream } from './kinesisfirehose.generated'; */ export interface DestinationConfig { /** - * Schema-less properties that will be injected directly into `CfnDeliveryStream`. + * S3 destination configuration properties. + * + * @default - S3 destination is not used. */ - readonly properties: object; + readonly extendedS3DestinationConfiguration?: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty; } /** diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts index 874b80b7862d1..564ffc69e4877 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -14,11 +14,9 @@ describe('delivery stream', () => { const mockS3Destination: firehose.IDestination = { bind(_scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { return { - properties: { - s3DestinationConfiguration: { - bucketArn: bucketArn, - roleArn: roleArn, - }, + extendedS3DestinationConfiguration: { + bucketArn: bucketArn, + roleArn: roleArn, }, }; }, @@ -38,7 +36,7 @@ describe('delivery stream', () => { DeliveryStreamName: ABSENT, DeliveryStreamType: 'DirectPut', KinesisStreamSourceConfiguration: ABSENT, - S3DestinationConfiguration: { + ExtendedS3DestinationConfiguration: { BucketARN: bucketArn, RoleARN: roleArn, }, diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts index 3332b82c523cd..c684dcb57e073 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts @@ -1,3 +1,4 @@ +import { anything } from '@aws-cdk/assert-internal'; import '@aws-cdk/assert-internal/jest'; import * as iam from '@aws-cdk/aws-iam'; import * as logs from '@aws-cdk/aws-logs'; @@ -11,69 +12,77 @@ describe('destination', () => { beforeEach(() => { stack = new cdk.Stack(); - destinationRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DestinationRole'); + destinationRole = new iam.Role(stack, 'DeliveryStreamRole', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); }); describe('createLoggingOptions', () => { class LoggingDestination extends firehose.DestinationBase { public bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { return { - properties: { - testDestinationConfig: { - loggingConfig: this.createLoggingOptions(scope, destinationRole, 'streamId'), - }, + extendedS3DestinationConfiguration: { + bucketArn: 'arn:aws:s3:::destination-bucket', + roleArn: destinationRole.roleArn, + cloudWatchLoggingOptions: this.createLoggingOptions(scope, destinationRole, 'streamId'), }, + }; } } test('creates resources and configuration by default', () => { - const testDestination = new LoggingDestination(); - - const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new LoggingDestination()], + }); expect(stack).toHaveResource('AWS::Logs::LogGroup'); expect(stack).toHaveResource('AWS::Logs::LogStream'); - expect(stack.resolve(testDestinationConfig)).toStrictEqual({ - properties: { - testDestinationConfig: { - loggingConfig: { - enabled: true, - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupstreamId3B940622', - }, - }, + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + BucketARN: 'arn:aws:s3:::destination-bucket', + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: anything(), + LogStreamName: anything(), }, + RoleARN: stack.resolve(destinationRole.roleArn), }, }); }); - test('does not create resources or configuration if disabled', () => { - const testDestination = new LoggingDestination({ logging: false }); - const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); - - expect(stack.resolve(testDestinationConfig)).toStrictEqual({ - properties: { - testDestinationConfig: {}, + test('does not create resources or configuration if disabled', () => { + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new LoggingDestination({ + logging: false, + })], + }); + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + BucketARN: 'arn:aws:s3:::destination-bucket', + RoleARN: stack.resolve(destinationRole.roleArn), }, }); }); test('creates configuration if log group provided', () => { - const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); - - const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new LoggingDestination({ + logGroup: new logs.LogGroup(stack, 'LogGroup'), + })], + }); - expect(stack.resolve(testDestinationConfig)).toMatchObject({ - properties: { - testDestinationConfig: { - loggingConfig: { - enabled: true, - }, + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + BucketARN: 'arn:aws:s3:::destination-bucket', + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: anything(), + LogStreamName: anything(), }, + RoleARN: stack.resolve(destinationRole.roleArn), }, }); }); @@ -85,67 +94,48 @@ describe('destination', () => { }); test('uses provided log group', () => { - const testDestination = new LoggingDestination({ logGroup: new logs.LogGroup(stack, 'Log Group') }); + const providedLogGroup = new logs.LogGroup(stack, 'LogGroup'); - const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new LoggingDestination({ + logGroup: providedLogGroup, + })], + }); - expect(stack).toCountResources('AWS::Logs::LogGroup', 1); - expect(stack.resolve(testDestinationConfig)).toMatchObject({ - properties: { - testDestinationConfig: { - loggingConfig: { - enabled: true, - logGroupName: { - Ref: 'LogGroupD9735569', - }, - logStreamName: { - Ref: 'LogGroupstreamIdA1293DC2', - }, - }, + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + BucketARN: 'arn:aws:s3:::destination-bucket', + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: stack.resolve(providedLogGroup.logGroupName), + LogStreamName: anything(), }, + RoleARN: stack.resolve(destinationRole.roleArn), }, }); }); test('re-uses log group if called multiple times', () => { - const testDestination = new class extends firehose.DestinationBase { + class TestDestination extends firehose.DestinationBase { public bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + let loggingOptions = this.createLoggingOptions(scope, destinationRole, 'streamId'); + loggingOptions = this.createLoggingOptions(scope, destinationRole, 'anotherStreamId'); return { - properties: { - testDestinationConfig: { - loggingConfig: this.createLoggingOptions(scope, destinationRole, 'streamId'), - anotherLoggingConfig: this.createLoggingOptions(scope, destinationRole, 'anotherStreamId'), - }, + extendedS3DestinationConfiguration: { + bucketArn: 'arn:aws:s3:::destination-bucket', + roleArn: destinationRole.roleArn, + cloudWatchLoggingOptions: loggingOptions, }, }; } - }(); - - const testDestinationConfig = testDestination.bind(stack, { role: destinationRole }); + }; - expect(stack).toCountResources('AWS::Logs::LogGroup', 1); - expect(stack.resolve(testDestinationConfig)).toMatchObject({ - properties: { - testDestinationConfig: { - loggingConfig: { - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupstreamId3B940622', - }, - }, - anotherLoggingConfig: { - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupanotherStreamIdF2754481', - }, - }, - }, - }, + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new TestDestination()], }); + expect(stack).toCountResources('AWS::Logs::LogGroup', 1); }); }); }); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json index ee2fbab429f40..27e11bba8edb4 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json @@ -43,7 +43,7 @@ "Type": "AWS::KinesisFirehose::DeliveryStream", "Properties": { "DeliveryStreamType": "DirectPut", - "S3DestinationConfiguration": { + "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "Bucket83908E77", diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts index 7bcbb788399c8..b11b75f52fc42 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts @@ -20,11 +20,9 @@ const role = new iam.Role(stack, 'Role', { const mockS3Destination: firehose.IDestination = { bind(_scope: constructs.Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { return { - properties: { - s3DestinationConfiguration: { - bucketArn: bucket.bucketArn, - roleArn: role.roleArn, - }, + extendedS3DestinationConfiguration: { + bucketArn: bucket.bucketArn, + roleArn: role.roleArn, }, }; }, From a2c85bd07d008be819c053a8c32cf67bab047cec Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Mon, 19 Jul 2021 14:42:00 -0700 Subject: [PATCH 36/52] remove DestinationBase, move createLoggingOptions and DestinationProps to dests module --- .../lib/common.ts | 32 ++++ .../lib/index.ts | 1 + .../lib/private/helpers.ts | 54 +++++++ .../lib/private/index.ts | 1 + .../lib/s3-bucket.ts | 12 +- .../package.json | 5 +- .../test/helpers.test.ts | 86 +++++++++++ .../aws-kinesisfirehose/lib/destination.ts | 65 +------- .../@aws-cdk/aws-kinesisfirehose/package.json | 11 +- .../test/destination.test.ts | 141 ------------------ 10 files changed, 188 insertions(+), 220 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts delete mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts new file mode 100644 index 0000000000000..4f4dddcdffbb4 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts @@ -0,0 +1,32 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as logs from '@aws-cdk/aws-logs'; + +/** + * Generic properties for defining a delivery stream destination. + */ +export interface DestinationProps { + /** + * If true, log errors when data transformation or data delivery fails. + * + * If `logGroup` is provided, this will be implicitly set to `true`. + * + * @default true - errors are logged. + */ + readonly logging?: boolean; + + /** + * The CloudWatch log group where log streams will be created to hold error logs. + * + * @default - if `logging` is set to `true`, a log group will be created for you. + */ + readonly logGroup?: logs.ILogGroup; + + /** + * The IAM role associated with this destination. + * + * Assumed by Kinesis Data Firehose to invoke processors and write to destinations + * + * @default - a role will be created with default permissions. + */ + readonly role?: iam.IRole; +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts index 03d672ad93c9f..7297f91a768c8 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts @@ -1 +1,2 @@ +export * from './common'; export * from './s3-bucket'; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts new file mode 100644 index 0000000000000..93c0d937ec203 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts @@ -0,0 +1,54 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as logs from '@aws-cdk/aws-logs'; +import { Construct, Node } from 'constructs'; + +export interface DestinationLoggingProps { + /** + * If true, log errors when data transformation or data delivery fails. + * + * If `logGroup` is provided, this will be implicitly set to `true`. + * + * @default true - errors are logged. + */ + readonly logging?: boolean; + + /** + * The CloudWatch log group where log streams will be created to hold error logs. + * + * @default - if `logging` is set to `true`, a log group will be created for you. + */ + readonly logGroup?: logs.ILogGroup; + + /** + * The IAM role associated with this destination. + */ + readonly role: iam.IRole; + + /** + * The ID of the stream that is created in the log group where logs will be placed. + * + * Must be unique within the log group, so should be different every time this function is called. + */ + readonly streamId: string; +} + +export function createLoggingOptions( + scope: Construct, + props: DestinationLoggingProps, +): firehose.CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { + if (props.logging === false && props.logGroup) { + throw new Error('logging cannot be set to false when logGroup is provided'); + } + if (props.logging !== false || props.logGroup) { + const logGroup = Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); + const logGroupGrant = logGroup.grantWrite(props.role); + Node.of(scope).addDependency(logGroupGrant); + return { + enabled: true, + logGroupName: logGroup.logGroupName, + logStreamName: logGroup.addStream(props.streamId).logStreamName, + }; + } + return undefined; +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts new file mode 100644 index 0000000000000..c5f595cf9d428 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts @@ -0,0 +1 @@ +export * from './helpers'; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 8cfe9b840bd95..257bf651fc02d 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -2,19 +2,19 @@ import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as s3 from '@aws-cdk/aws-s3'; import { Construct, Node } from 'constructs'; +import { DestinationProps } from './common'; +import { createLoggingOptions } from './private/helpers'; /** * Props for defining an S3 destination of a Kinesis Data Firehose delivery stream. */ -export interface S3BucketProps extends firehose.DestinationProps { } +export interface S3BucketProps extends DestinationProps { } /** * An S3 bucket destination for data from a Kinesis Data Firehose delivery stream. */ -export class S3Bucket extends firehose.DestinationBase { - constructor(private readonly bucket: s3.IBucket, s3Props: S3BucketProps = {}) { - super(s3Props); - } +export class S3Bucket implements firehose.IDestination { + constructor(private readonly bucket: s3.IBucket, private props: S3BucketProps = {}) { } bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { const role = this.props.role ?? new iam.Role(scope, 'S3 Destination Role', { @@ -26,7 +26,7 @@ export class S3Bucket extends firehose.DestinationBase { return { extendedS3DestinationConfiguration: { - cloudWatchLoggingOptions: this.createLoggingOptions(scope, role, 'S3Destination'), + cloudWatchLoggingOptions: createLoggingOptions(scope, { logging: this.props.logging, logGroup: this.props.logGroup, role, streamId: 'S3Destination' }), roleArn: role.roleArn, bucketArn: this.bucket.bucketArn, }, diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json index 24da18f9c3947..92cb7c8aa4332 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json @@ -77,12 +77,12 @@ "cfn2ts": "0.0.0", "jest": "^26.6.3", "pkglint": "0.0.0", - "@aws-cdk/assert-internal": "0.0.0", - "@aws-cdk/aws-logs": "0.0.0" + "@aws-cdk/assert-internal": "0.0.0" }, "dependencies": { "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", "constructs": "^3.3.69" @@ -91,6 +91,7 @@ "peerDependencies": { "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", "constructs": "^3.3.69" diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts new file mode 100644 index 0000000000000..9d54c428d56d0 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts @@ -0,0 +1,86 @@ +import '@aws-cdk/assert-internal/jest'; +import * as iam from '@aws-cdk/aws-iam'; +import * as logs from '@aws-cdk/aws-logs'; +import * as cdk from '@aws-cdk/core'; +import { createLoggingOptions } from '../lib/private'; + +describe('createLoggingOptions', () => { + let stack: cdk.Stack; + let destinationRole: iam.IRole; + + beforeEach(() => { + stack = new cdk.Stack(); + destinationRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DestinationRole'); + }); + + test('creates resources and configuration by default', () => { + const loggingOptions = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); + + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); + expect(stack.resolve(loggingOptions)).toStrictEqual({ + enabled: true, + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupstreamId3B940622', + }, + }); + }); + test('does not create resources or configuration if disabled', () => { + const loggingOptions = createLoggingOptions(stack, { logging: false, streamId: 'streamId', role: destinationRole }); + + expect(stack.resolve(loggingOptions)).toBeUndefined(); + }); + + test('creates configuration if log group provided', () => { + const loggingOptions = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); + + expect(stack.resolve(loggingOptions)).toMatchObject({ + enabled: true, + }); + }); + + test('throws error if logging disabled but log group provided', () => { + expect(() => createLoggingOptions(stack, { logging: false, logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole })).toThrowError('logging cannot be set to false when logGroup is provided'); + }); + + test('uses provided log group', () => { + const loggingOptions = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); + + expect(stack).toCountResources('AWS::Logs::LogGroup', 1); + expect(stack.resolve(loggingOptions)).toMatchObject({ + enabled: true, + logGroupName: { + Ref: 'LogGroupD9735569', + }, + logStreamName: { + Ref: 'LogGroupstreamIdA1293DC2', + }, + }); + }); + + test('re-uses log group if called multiple times', () => { + const loggingOptions = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); + const anotherLoggingOptions = createLoggingOptions(stack, { streamId: 'anotherStreamId', role: destinationRole }); + + expect(stack).toCountResources('AWS::Logs::LogGroup', 1); + expect(stack.resolve(loggingOptions)).toMatchObject({ + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupstreamId3B940622', + }, + }); + expect(stack.resolve(anotherLoggingOptions)).toMatchObject({ + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupanotherStreamIdF2754481', + }, + }); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index c3e70b88c9a95..906c04aa01e2c 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -1,6 +1,4 @@ -import * as iam from '@aws-cdk/aws-iam'; -import * as logs from '@aws-cdk/aws-logs'; -import { Construct, Node } from 'constructs'; +import { Construct } from 'constructs'; import { CfnDeliveryStream } from './kinesisfirehose.generated'; /** @@ -32,64 +30,3 @@ export interface IDestination { */ bind(scope: Construct, options: DestinationBindOptions): DestinationConfig; } - -/** - * Generic properties for defining a delivery stream destination. - */ -export interface DestinationProps { - /** - * If true, log errors when data transformation or data delivery fails. - * - * If `logGroup` is provided, this will be implicitly set to `true`. - * - * @default true - errors are logged. - */ - readonly logging?: boolean; - - /** - * The CloudWatch log group where log streams will be created to hold error logs. - * - * @default - if `logging` is set to `true`, a log group will be created for you. - */ - readonly logGroup?: logs.ILogGroup; - - /** - * The IAM role associated with this destination. - * - * Assumed by Kinesis Data Firehose to invoke processors and write to destinations - * - * @default - a role will be created with default permissions. - */ - readonly role?: iam.IRole; -} - -/** - * Abstract base class that destination types can extend to benefit from methods that create generic configuration. - */ -export abstract class DestinationBase implements IDestination { - - constructor(protected readonly props: DestinationProps = {}) { } - - abstract bind(scope: Construct, options: DestinationBindOptions): DestinationConfig; - - protected createLoggingOptions( - scope: Construct, - role: iam.IRole, - streamId: string, - ): CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { - if (this.props.logging === false && this.props.logGroup) { - throw new Error('logging cannot be set to false when logGroup is provided'); - } - if (this.props.logging !== false || this.props.logGroup) { - const logGroup = Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? this.props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); - const logGroupGrant = logGroup.grantWrite(role); - Node.of(scope).addDependency(logGroupGrant); - return { - enabled: true, - logGroupName: logGroup.logGroupName, - logStreamName: logGroup.addStream(streamId).logStreamName, - }; - } - return undefined; - } -} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index 01afe8f53e2e6..1e445d179cde0 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -73,19 +73,18 @@ }, "license": "Apache-2.0", "devDependencies": { + "@aws-cdk/assert-internal": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", "@types/jest": "^26.0.24", "cdk-build-tools": "0.0.0", + "cdk-integ-tools": "0.0.0", "cfn2ts": "0.0.0", - "pkglint": "0.0.0", - "@aws-cdk/assert-internal": "0.0.0", - "cdk-integ-tools": "0.0.0" + "pkglint": "0.0.0" }, "dependencies": { "@aws-cdk/aws-cloudwatch": "0.0.0", "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", - "@aws-cdk/aws-logs": "0.0.0", - "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/custom-resources": "0.0.0", "@aws-cdk/region-info": "0.0.0", @@ -96,8 +95,6 @@ "@aws-cdk/aws-cloudwatch": "0.0.0", "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", - "@aws-cdk/aws-logs": "0.0.0", - "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/custom-resources": "0.0.0", "@aws-cdk/region-info": "0.0.0", diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts deleted file mode 100644 index c684dcb57e073..0000000000000 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/destination.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { anything } from '@aws-cdk/assert-internal'; -import '@aws-cdk/assert-internal/jest'; -import * as iam from '@aws-cdk/aws-iam'; -import * as logs from '@aws-cdk/aws-logs'; -import * as cdk from '@aws-cdk/core'; -import { Construct } from 'constructs'; -import * as firehose from '../lib'; - -describe('destination', () => { - let stack: cdk.Stack; - let destinationRole: iam.IRole; - - beforeEach(() => { - stack = new cdk.Stack(); - destinationRole = new iam.Role(stack, 'DeliveryStreamRole', { - assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), - }); - }); - - describe('createLoggingOptions', () => { - class LoggingDestination extends firehose.DestinationBase { - public bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { - return { - extendedS3DestinationConfiguration: { - bucketArn: 'arn:aws:s3:::destination-bucket', - roleArn: destinationRole.roleArn, - cloudWatchLoggingOptions: this.createLoggingOptions(scope, destinationRole, 'streamId'), - }, - - }; - } - } - - test('creates resources and configuration by default', () => { - new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new LoggingDestination()], - }); - - expect(stack).toHaveResource('AWS::Logs::LogGroup'); - expect(stack).toHaveResource('AWS::Logs::LogStream'); - expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { - ExtendedS3DestinationConfiguration: { - BucketARN: 'arn:aws:s3:::destination-bucket', - CloudWatchLoggingOptions: { - Enabled: true, - LogGroupName: anything(), - LogStreamName: anything(), - }, - RoleARN: stack.resolve(destinationRole.roleArn), - }, - }); - }); - - test('does not create resources or configuration if disabled', () => { - new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new LoggingDestination({ - logging: false, - })], - }); - expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { - ExtendedS3DestinationConfiguration: { - BucketARN: 'arn:aws:s3:::destination-bucket', - RoleARN: stack.resolve(destinationRole.roleArn), - }, - }); - }); - - test('creates configuration if log group provided', () => { - new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new LoggingDestination({ - logGroup: new logs.LogGroup(stack, 'LogGroup'), - })], - }); - - expect(stack).toHaveResource('AWS::Logs::LogGroup'); - expect(stack).toHaveResource('AWS::Logs::LogStream'); - expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { - ExtendedS3DestinationConfiguration: { - BucketARN: 'arn:aws:s3:::destination-bucket', - CloudWatchLoggingOptions: { - Enabled: true, - LogGroupName: anything(), - LogStreamName: anything(), - }, - RoleARN: stack.resolve(destinationRole.roleArn), - }, - }); - }); - - test('throws error if logging disabled but log group provided', () => { - const testDestination = new LoggingDestination({ logging: false, logGroup: new logs.LogGroup(stack, 'Log Group') }); - - expect(() => testDestination.bind(stack, { role: destinationRole })).toThrowError('logging cannot be set to false when logGroup is provided'); - }); - - test('uses provided log group', () => { - const providedLogGroup = new logs.LogGroup(stack, 'LogGroup'); - - new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new LoggingDestination({ - logGroup: providedLogGroup, - })], - }); - - expect(stack).toHaveResource('AWS::Logs::LogGroup'); - expect(stack).toHaveResource('AWS::Logs::LogStream'); - expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { - ExtendedS3DestinationConfiguration: { - BucketARN: 'arn:aws:s3:::destination-bucket', - CloudWatchLoggingOptions: { - Enabled: true, - LogGroupName: stack.resolve(providedLogGroup.logGroupName), - LogStreamName: anything(), - }, - RoleARN: stack.resolve(destinationRole.roleArn), - }, - }); - }); - - test('re-uses log group if called multiple times', () => { - class TestDestination extends firehose.DestinationBase { - public bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { - let loggingOptions = this.createLoggingOptions(scope, destinationRole, 'streamId'); - loggingOptions = this.createLoggingOptions(scope, destinationRole, 'anotherStreamId'); - return { - extendedS3DestinationConfiguration: { - bucketArn: 'arn:aws:s3:::destination-bucket', - roleArn: destinationRole.roleArn, - cloudWatchLoggingOptions: loggingOptions, - }, - }; - } - }; - - new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new TestDestination()], - }); - expect(stack).toCountResources('AWS::Logs::LogGroup', 1); - }); - }); -}); From 569ca01853c9c35843663664860abc7229b6083f Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Mon, 19 Jul 2021 16:49:13 -0700 Subject: [PATCH 37/52] don't pass role into bind() --- packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index e1ad4b1ec1ff3..ead862abf89b1 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -219,7 +219,7 @@ export class DeliveryStream extends DeliveryStreamBase { }); this.grantPrincipal = role; - const destinationConfig = props.destinations[0].bind(this, { role: role }); + const destinationConfig = props.destinations[0].bind(this, {}); const resource = new CfnDeliveryStream(this, 'Resource', { deliveryStreamName: props.deliveryStreamName, From a307108f0e185f053330a3b6af38a6d935fb7721 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 14:38:10 -0700 Subject: [PATCH 38/52] feat(kinesisfirehose): destinations can provide resources to be depended on by the delivery stream (#15676) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../lib/private/helpers.ts | 28 +++++--- .../lib/s3-bucket.ts | 8 ++- .../test/helpers.test.ts | 66 ++++++++++++++----- .../test/integ.s3-bucket.expected.json | 10 +-- .../test/s3-bucket.test.ts | 55 +++++++++++++++- .../lib/delivery-stream.ts | 1 + .../aws-kinesisfirehose/lib/destination.ts | 8 +++ .../test/delivery-stream.test.ts | 48 ++++++++++---- .../test/integ.delivery-stream.expected.json | 54 ++++++++++++++- .../test/integ.delivery-stream.ts | 2 + 10 files changed, 229 insertions(+), 51 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts index 93c0d937ec203..57550e7434843 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts @@ -1,6 +1,7 @@ import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as logs from '@aws-cdk/aws-logs'; +import * as cdk from '@aws-cdk/core'; import { Construct, Node } from 'constructs'; export interface DestinationLoggingProps { @@ -33,21 +34,32 @@ export interface DestinationLoggingProps { readonly streamId: string; } -export function createLoggingOptions( - scope: Construct, - props: DestinationLoggingProps, -): firehose.CfnDeliveryStream.CloudWatchLoggingOptionsProperty | undefined { +export interface DestinationLoggingOutput { + /** + * Logging options that will be injected into the destination configuration. + */ + readonly loggingOptions: firehose.CfnDeliveryStream.CloudWatchLoggingOptionsProperty; + + /** + * Resources that were created by the sub-config creator that must be deployed before the delivery stream is deployed. + */ + readonly dependables: cdk.IDependable[]; +} + +export function createLoggingOptions(scope: Construct, props: DestinationLoggingProps): DestinationLoggingOutput | undefined { if (props.logging === false && props.logGroup) { throw new Error('logging cannot be set to false when logGroup is provided'); } if (props.logging !== false || props.logGroup) { const logGroup = Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); const logGroupGrant = logGroup.grantWrite(props.role); - Node.of(scope).addDependency(logGroupGrant); return { - enabled: true, - logGroupName: logGroup.logGroupName, - logStreamName: logGroup.addStream(props.streamId).logStreamName, + loggingOptions: { + enabled: true, + logGroupName: logGroup.logGroupName, + logStreamName: logGroup.addStream(props.streamId).logStreamName, + }, + dependables: [logGroupGrant], }; } return undefined; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 257bf651fc02d..a37ca4f8dc0e2 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -1,7 +1,7 @@ import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as s3 from '@aws-cdk/aws-s3'; -import { Construct, Node } from 'constructs'; +import { Construct } from 'constructs'; import { DestinationProps } from './common'; import { createLoggingOptions } from './private/helpers'; @@ -22,14 +22,16 @@ export class S3Bucket implements firehose.IDestination { }); const bucketGrant = this.bucket.grantReadWrite(role); - Node.of(scope).addDependency(bucketGrant); + + const { loggingOptions, dependables: loggingDependables } = createLoggingOptions(scope, { logging: this.props.logging, logGroup: this.props.logGroup, role, streamId: 'S3Destination' }) ?? {}; return { extendedS3DestinationConfiguration: { - cloudWatchLoggingOptions: createLoggingOptions(scope, { logging: this.props.logging, logGroup: this.props.logGroup, role, streamId: 'S3Destination' }), + cloudWatchLoggingOptions: loggingOptions, roleArn: role.roleArn, bucketArn: this.bucket.bucketArn, }, + dependables: [bucketGrant, ...(loggingDependables ?? [])], }; } } diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts index 9d54c428d56d0..2700b1e7f6753 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts @@ -13,12 +13,12 @@ describe('createLoggingOptions', () => { destinationRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DestinationRole'); }); - test('creates resources and configuration by default', () => { - const loggingOptions = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); + it('creates resources and configuration by default', () => { + const loggingOutput = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); expect(stack).toHaveResource('AWS::Logs::LogGroup'); expect(stack).toHaveResource('AWS::Logs::LogStream'); - expect(stack.resolve(loggingOptions)).toStrictEqual({ + expect(stack.resolve(loggingOutput?.loggingOptions)).toStrictEqual({ enabled: true, logGroupName: { Ref: 'LogGroupF5B46931', @@ -28,29 +28,30 @@ describe('createLoggingOptions', () => { }, }); }); - test('does not create resources or configuration if disabled', () => { - const loggingOptions = createLoggingOptions(stack, { logging: false, streamId: 'streamId', role: destinationRole }); - expect(stack.resolve(loggingOptions)).toBeUndefined(); + it('does not create resources or configuration if disabled', () => { + const loggingOutput = createLoggingOptions(stack, { logging: false, streamId: 'streamId', role: destinationRole }); + + expect(stack.resolve(loggingOutput)).toBeUndefined(); }); - test('creates configuration if log group provided', () => { - const loggingOptions = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); + it('creates configuration if log group provided', () => { + const loggingOutput = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); - expect(stack.resolve(loggingOptions)).toMatchObject({ + expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ enabled: true, }); }); - test('throws error if logging disabled but log group provided', () => { + it('throws error if logging disabled but log group provided', () => { expect(() => createLoggingOptions(stack, { logging: false, logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole })).toThrowError('logging cannot be set to false when logGroup is provided'); }); - test('uses provided log group', () => { - const loggingOptions = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); + it('uses provided log group', () => { + const loggingOutput = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); - expect(stack.resolve(loggingOptions)).toMatchObject({ + expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ enabled: true, logGroupName: { Ref: 'LogGroupD9735569', @@ -61,12 +62,12 @@ describe('createLoggingOptions', () => { }); }); - test('re-uses log group if called multiple times', () => { - const loggingOptions = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); - const anotherLoggingOptions = createLoggingOptions(stack, { streamId: 'anotherStreamId', role: destinationRole }); + it('re-uses log group if called multiple times', () => { + const loggingOutput = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); + const anotherLoggingOutput = createLoggingOptions(stack, { streamId: 'anotherStreamId', role: destinationRole }); expect(stack).toCountResources('AWS::Logs::LogGroup', 1); - expect(stack.resolve(loggingOptions)).toMatchObject({ + expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ logGroupName: { Ref: 'LogGroupF5B46931', }, @@ -74,7 +75,7 @@ describe('createLoggingOptions', () => { Ref: 'LogGroupstreamId3B940622', }, }); - expect(stack.resolve(anotherLoggingOptions)).toMatchObject({ + expect(stack.resolve(anotherLoggingOutput?.loggingOptions)).toMatchObject({ logGroupName: { Ref: 'LogGroupF5B46931', }, @@ -83,4 +84,33 @@ describe('createLoggingOptions', () => { }, }); }); + + it('grants log group write permissions to destination role', () => { + const logGroup = new logs.LogGroup(stack, 'Log Group'); + + createLoggingOptions(stack, { logGroup, streamId: 'streamId', role: destinationRole }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + Roles: [stack.resolve(destinationRole.roleName)], + PolicyDocument: { + Statement: [ + { + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Effect: 'Allow', + Resource: stack.resolve(logGroup.logGroupArn), + }, + ], + }, + }); + }); + + it('returns log group grant as dependable', () => { + const loggingOutput = createLoggingOptions(stack, { logging: true, streamId: 'streamId', role: destinationRole }); + + expect(loggingOutput?.dependables?.length).toBe(1); + expect(loggingOutput?.dependables[0]).toBeInstanceOf(iam.Grant); + }); }); diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json index c3e33cb73f21b..3a5dfb8a72b7c 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json @@ -198,10 +198,7 @@ ], "Version": "2012-10-17" } - }, - "DependsOn": [ - "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7" - ] + } }, "DeliveryStreamS3DestinationRole500FC089": { "Type": "AWS::IAM::Role", @@ -218,10 +215,7 @@ ], "Version": "2012-10-17" } - }, - "DependsOn": [ - "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7" - ] + } }, "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7": { "Type": "AWS::IAM::Policy", diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts index 748b55bc8fb24..e443a510128d4 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -1,5 +1,5 @@ import '@aws-cdk/assert-internal/jest'; -import { ABSENT, Capture, anything, MatchStyle } from '@aws-cdk/assert-internal'; +import { ABSENT, Capture, MatchStyle, ResourcePart, anything } from '@aws-cdk/assert-internal'; import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as logs from '@aws-cdk/aws-logs'; @@ -134,4 +134,57 @@ describe('S3 destination', () => { }, }); }); + + test('bucket and log group grants are depended on by delivery stream', () => { + const logGroup = logs.LogGroup.fromLogGroupName(stack, 'Log Group', 'evergreen'); + const capturedPolicyId = Capture.aString(); + const destination = new firehosedestinations.S3Bucket(bucket, { role: destinationRole, logGroup }); + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [destination], + }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyName: capturedPolicyId.capture(), + Roles: [stack.resolve(destinationRole.roleName)], + PolicyDocument: { + Statement: [ + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + stack.resolve(bucket.bucketArn), + { + 'Fn::Join': [ + '', + [ + stack.resolve(bucket.bucketArn), + '/*', + ], + ], + }, + ], + }, + { + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Effect: 'Allow', + Resource: stack.resolve(logGroup.logGroupArn), + }, + ], + }, + }); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + DependsOn: [capturedPolicyId.capturedValue], + }, ResourcePart.CompleteDefinition); + }); }); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index ead862abf89b1..51b159664e437 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -226,6 +226,7 @@ export class DeliveryStream extends DeliveryStreamBase { deliveryStreamType: 'DirectPut', ...destinationConfig, }); + destinationConfig.dependables?.forEach(dependable => resource.node.addDependency(dependable)); this.deliveryStreamArn = this.getResourceArnAttribute(resource.attrArn, { service: 'kinesis', diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts index 906c04aa01e2c..eb277babcdebb 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -1,3 +1,4 @@ +import * as cdk from '@aws-cdk/core'; import { Construct } from 'constructs'; import { CfnDeliveryStream } from './kinesisfirehose.generated'; @@ -11,6 +12,13 @@ export interface DestinationConfig { * @default - S3 destination is not used. */ readonly extendedS3DestinationConfiguration?: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty; + + /** + * Any resources that were created by the destination when binding it to the stack that must be deployed before the delivery stream is deployed. + * + * @default [] + */ + readonly dependables?: cdk.IDependable[]; } /** diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts index 564ffc69e4877..6be168b64ae9b 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -1,29 +1,38 @@ import '@aws-cdk/assert-internal/jest'; -import { ABSENT } from '@aws-cdk/assert-internal'; +import { ABSENT, ResourcePart } from '@aws-cdk/assert-internal'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as cdk from '@aws-cdk/core'; -import { Construct } from 'constructs'; +import { Construct, Node } from 'constructs'; import * as firehose from '../lib'; describe('delivery stream', () => { let stack: cdk.Stack; + let dependable: Construct; + let mockS3Destination: firehose.IDestination; const bucketArn = 'arn:aws:s3:::my-bucket'; const roleArn = 'arn:aws:iam::111122223333:role/my-role'; - const mockS3Destination: firehose.IDestination = { - bind(_scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { - return { - extendedS3DestinationConfiguration: { - bucketArn: bucketArn, - roleArn: roleArn, - }, - }; - }, - }; beforeEach(() => { stack = new cdk.Stack(); + mockS3Destination = { + bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + dependable = new class extends cdk.Construct { + constructor(depScope: Construct, id: string) { + super(depScope, id); + new cdk.CfnResource(this, 'Resource', { type: 'CDK::Dummy' }); + } + }(scope, 'Dummy Dep'); + return { + extendedS3DestinationConfiguration: { + bucketArn: bucketArn, + roleArn: roleArn, + }, + dependables: [dependable], + }; + }, + }; }); test('creates stream with default values', () => { @@ -123,6 +132,21 @@ describe('delivery stream', () => { }); }); + test('dependables supplied from destination are depended on by just the CFN resource', () => { + const dependableId = stack.resolve((Node.of(dependable).defaultChild as cdk.CfnResource).logicalId); + + new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + DependsOn: [dependableId], + }, ResourcePart.CompleteDefinition); + expect(stack).toHaveResourceLike('AWS::IAM::Role', { + DependsOn: ABSENT, + }, ResourcePart.CompleteDefinition); + }); + test('supplying 0 or multiple destinations throws', () => { expect(() => new firehose.DeliveryStream(stack, 'No Destinations', { destinations: [], diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json index 27e11bba8edb4..21c29f8fee06f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json @@ -22,6 +22,55 @@ } } }, + "RoleDefaultPolicy5FFB7DAB": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "RoleDefaultPolicy5FFB7DAB", + "Roles": [ + { + "Ref": "Role1ABCC5F0" + } + ] + } + }, "DeliveryStreamServiceRole964EEBCC": { "Type": "AWS::IAM::Role", "Properties": { @@ -57,7 +106,10 @@ ] } } - } + }, + "DependsOn": [ + "RoleDefaultPolicy5FFB7DAB" + ] } }, "Mappings": { diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts index b11b75f52fc42..1adfd9bfc0221 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts @@ -19,11 +19,13 @@ const role = new iam.Role(stack, 'Role', { const mockS3Destination: firehose.IDestination = { bind(_scope: constructs.Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + const bucketGrant = bucket.grantReadWrite(role); return { extendedS3DestinationConfiguration: { bucketArn: bucket.bucketArn, roleArn: role.roleArn, }, + dependables: [bucketGrant], }; }, }; From 04d5ae74e3afc3ea613e1370e6e05653e52f59a4 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 14:46:02 -0700 Subject: [PATCH 39/52] remove unused custom-resources dependency --- packages/@aws-cdk/aws-kinesisfirehose/package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index 1e445d179cde0..d6651f8d08c62 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -86,7 +86,6 @@ "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/core": "0.0.0", - "@aws-cdk/custom-resources": "0.0.0", "@aws-cdk/region-info": "0.0.0", "constructs": "^3.3.69" }, @@ -96,7 +95,6 @@ "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/core": "0.0.0", - "@aws-cdk/custom-resources": "0.0.0", "@aws-cdk/region-info": "0.0.0", "constructs": "^3.3.69" }, From 93cc0e57397d0d8ff437ec59361ccfae14f51339 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 19:40:20 -0700 Subject: [PATCH 40/52] rewrite providing IAM role section (include separate destination role, update example) --- .../@aws-cdk/aws-kinesisfirehose/README.md | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 0e68ac9a586ce..27d8caa26ce58 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -154,24 +154,38 @@ in the *Kinesis Data Firehose Developer Guide*. ## Specifying an IAM role -The DeliveryStream class automatically creates an IAM role with all the minimum necessary -permissions for Kinesis Data Firehose to access the resources referenced by your delivery -stream. For example: an Elasticsearch domain, a Redshift cluster, a backup or destination -S3 bucket, a Lambda data transformer, an AWS Glue table schema, etc. If you wish, you may -specify your own IAM role. It must have the correct permissions, or delivery stream -creation or data delivery may fail. +The DeliveryStream class automatically creates IAM service roles with all the minimum +necessary permissions for Kinesis Data Firehose to access the resources referenced by your +delivery stream. One service role is created for the delivery stream that allows Kinesis +Data Firehose to read from a Kinesis data stream (if one is configured as the delivery +stream source) and for server-side encryption. Another service role is created for each +destination, which gives Kinesis Data Firehose write access to the destination resource, +as well as the ability to invoke data transformers and read schemas for record format +conversion. If you wish, you may specify your own IAM role for either the delivery stream +or the destination service role, or both. It must have the correct trust policy (it must +allow Kinesis Data Firehose to assume it) or delivery stream creation or data delivery +will fail. ```ts fixture=with-bucket import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; import * as iam from '@aws-cdk/aws-iam'; -const role = new iam.Role(this, 'Role', { - assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), +// Create service roles for the delivery stream and destination. +// These can be used for other purposes and granted access to different resources. +// They must include the Kinesis Data Firehose service principal in their trust policies. +// Two separate roles are shown below, but the same role can be used for both purposes. +const deliveryStreamRole = new iam.Role(this, 'Delivery Stream Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), +}); +const destinationRole = new iam.Role(this, 'Destination Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), }); -bucket.grantWrite(role); + +// Specify the roles created above when defining the destination and delivery stream. +const destination = new destinations.S3Bucket(bucket, { role: destinationRole }); new DeliveryStream(this, 'Delivery Stream', { - destinations: [new destinations.S3Bucket(bucket)], - role: role, + destinations: [destination], + role: deliveryStreamRole, }); ``` From 3aa77bdf79f66e9b1e416bd5b1b1574ffae78721 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 19:42:17 -0700 Subject: [PATCH 41/52] add more IAM role detail --- packages/@aws-cdk/aws-kinesisfirehose/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 27d8caa26ce58..5034ac4a0765f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -164,7 +164,8 @@ as well as the ability to invoke data transformers and read schemas for record f conversion. If you wish, you may specify your own IAM role for either the delivery stream or the destination service role, or both. It must have the correct trust policy (it must allow Kinesis Data Firehose to assume it) or delivery stream creation or data delivery -will fail. +will fail. Other required permissions to destination resources, encryption keys, etc., +will be provided automatically. ```ts fixture=with-bucket import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; From 78839298920cb128f0fb0502c2163823d9906064 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 19:44:54 -0700 Subject: [PATCH 42/52] snapshot spacing --- .../test/s3-bucket.test.ts | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts index e443a510128d4..a7d4e9ceb0827 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -119,15 +119,7 @@ describe('S3 destination', () => { Effect: 'Allow', Resource: [ stack.resolve(bucket.bucketArn), - { - 'Fn::Join': [ - '', - [ - stack.resolve(bucket.bucketArn), - '/*', - ], - ], - }, + { 'Fn::Join': ['', [stack.resolve(bucket.bucketArn), '/*']] }, ], }, ], @@ -161,15 +153,7 @@ describe('S3 destination', () => { Effect: 'Allow', Resource: [ stack.resolve(bucket.bucketArn), - { - 'Fn::Join': [ - '', - [ - stack.resolve(bucket.bucketArn), - '/*', - ], - ], - }, + { 'Fn::Join': ['', [stack.resolve(bucket.bucketArn), '/*']] }, ], }, { From 45fec813786958084c7ac0758d360f1a6ead4356 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 19:57:57 -0700 Subject: [PATCH 43/52] make s3 destination props readonly --- .../@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index a37ca4f8dc0e2..536c6f7b2107f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -14,7 +14,7 @@ export interface S3BucketProps extends DestinationProps { } * An S3 bucket destination for data from a Kinesis Data Firehose delivery stream. */ export class S3Bucket implements firehose.IDestination { - constructor(private readonly bucket: s3.IBucket, private props: S3BucketProps = {}) { } + constructor(private readonly bucket: s3.IBucket, private readonly props: S3BucketProps = {}) { } bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { const role = this.props.role ?? new iam.Role(scope, 'S3 Destination Role', { From 28c120fca0bf6696add742d13140b932af872f9e Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 19:58:14 -0700 Subject: [PATCH 44/52] line length linting --- .../aws-kinesisfirehose-destinations/lib/s3-bucket.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 536c6f7b2107f..d1a0bb3aed853 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -23,7 +23,15 @@ export class S3Bucket implements firehose.IDestination { const bucketGrant = this.bucket.grantReadWrite(role); - const { loggingOptions, dependables: loggingDependables } = createLoggingOptions(scope, { logging: this.props.logging, logGroup: this.props.logGroup, role, streamId: 'S3Destination' }) ?? {}; + const { loggingOptions, dependables: loggingDependables } = createLoggingOptions( + scope, + { + logging: this.props.logging, + logGroup: this.props.logGroup, + role, + streamId: 'S3Destination', + }, + ) ?? {}; return { extendedS3DestinationConfiguration: { From 005402c72a397059c8e5441ebc6bd5f3ce51ad20 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 20:19:18 -0700 Subject: [PATCH 45/52] do not re-use log group if log group provided in props --- .../lib/private/helpers.ts | 2 +- .../test/helpers.test.ts | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts index 57550e7434843..a2032c41914a0 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts @@ -51,7 +51,7 @@ export function createLoggingOptions(scope: Construct, props: DestinationLogging throw new Error('logging cannot be set to false when logGroup is provided'); } if (props.logging !== false || props.logGroup) { - const logGroup = Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? props.logGroup ?? new logs.LogGroup(scope, 'LogGroup'); + const logGroup = props.logGroup ?? Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? new logs.LogGroup(scope, 'LogGroup'); const logGroupGrant = logGroup.grantWrite(props.role); return { loggingOptions: { diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts index 2700b1e7f6753..b7e2a0b2b7cf0 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts @@ -85,6 +85,29 @@ describe('createLoggingOptions', () => { }); }); + it('does not re-use log group if log group provided', () => { + const loggingOutput = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); + const anotherLoggingOutput = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Another Log Group'), streamId: 'streamId', role: destinationRole }); + + expect(stack).toCountResources('AWS::Logs::LogGroup', 2); + expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ + logGroupName: { + Ref: 'LogGroupF5B46931', + }, + logStreamName: { + Ref: 'LogGroupstreamId3B940622', + }, + }); + expect(stack.resolve(anotherLoggingOutput?.loggingOptions)).toMatchObject({ + logGroupName: { + Ref: 'AnotherLogGroup561B0F7C', + }, + logStreamName: { + Ref: 'AnotherLogGroupstreamId15DC3997', + }, + }); + }); + it('grants log group write permissions to destination role', () => { const logGroup = new logs.LogGroup(stack, 'Log Group'); From 0f9d42d848e8ad12bbb9af3859eba9ceeb6af93b Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Tue, 20 Jul 2021 20:23:44 -0700 Subject: [PATCH 46/52] rename DestinationProps -> CommonDestinationProps --- .../@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts | 2 +- .../aws-kinesisfirehose-destinations/lib/s3-bucket.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts index 4f4dddcdffbb4..3a97970d1ddbb 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts @@ -4,7 +4,7 @@ import * as logs from '@aws-cdk/aws-logs'; /** * Generic properties for defining a delivery stream destination. */ -export interface DestinationProps { +export interface CommonDestinationProps { /** * If true, log errors when data transformation or data delivery fails. * diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index d1a0bb3aed853..6e2a4faaab88d 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -2,13 +2,13 @@ import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as s3 from '@aws-cdk/aws-s3'; import { Construct } from 'constructs'; -import { DestinationProps } from './common'; +import { CommonDestinationProps } from './common'; import { createLoggingOptions } from './private/helpers'; /** * Props for defining an S3 destination of a Kinesis Data Firehose delivery stream. */ -export interface S3BucketProps extends DestinationProps { } +export interface S3BucketProps extends CommonDestinationProps { } /** * An S3 bucket destination for data from a Kinesis Data Firehose delivery stream. From ed02d9fda3ed597e5ab9e0bc2e1cadd9c8fafc1f Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Wed, 21 Jul 2021 12:28:42 -0700 Subject: [PATCH 47/52] move logging tests to S3 bucket destination --- .../test/helpers.test.ts | 139 ------------------ .../test/s3-bucket.test.ts | 120 ++++++++++----- 2 files changed, 86 insertions(+), 173 deletions(-) delete mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts deleted file mode 100644 index b7e2a0b2b7cf0..0000000000000 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/helpers.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import '@aws-cdk/assert-internal/jest'; -import * as iam from '@aws-cdk/aws-iam'; -import * as logs from '@aws-cdk/aws-logs'; -import * as cdk from '@aws-cdk/core'; -import { createLoggingOptions } from '../lib/private'; - -describe('createLoggingOptions', () => { - let stack: cdk.Stack; - let destinationRole: iam.IRole; - - beforeEach(() => { - stack = new cdk.Stack(); - destinationRole = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DestinationRole'); - }); - - it('creates resources and configuration by default', () => { - const loggingOutput = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); - - expect(stack).toHaveResource('AWS::Logs::LogGroup'); - expect(stack).toHaveResource('AWS::Logs::LogStream'); - expect(stack.resolve(loggingOutput?.loggingOptions)).toStrictEqual({ - enabled: true, - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupstreamId3B940622', - }, - }); - }); - - it('does not create resources or configuration if disabled', () => { - const loggingOutput = createLoggingOptions(stack, { logging: false, streamId: 'streamId', role: destinationRole }); - - expect(stack.resolve(loggingOutput)).toBeUndefined(); - }); - - it('creates configuration if log group provided', () => { - const loggingOutput = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); - - expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ - enabled: true, - }); - }); - - it('throws error if logging disabled but log group provided', () => { - expect(() => createLoggingOptions(stack, { logging: false, logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole })).toThrowError('logging cannot be set to false when logGroup is provided'); - }); - - it('uses provided log group', () => { - const loggingOutput = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Log Group'), streamId: 'streamId', role: destinationRole }); - - expect(stack).toCountResources('AWS::Logs::LogGroup', 1); - expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ - enabled: true, - logGroupName: { - Ref: 'LogGroupD9735569', - }, - logStreamName: { - Ref: 'LogGroupstreamIdA1293DC2', - }, - }); - }); - - it('re-uses log group if called multiple times', () => { - const loggingOutput = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); - const anotherLoggingOutput = createLoggingOptions(stack, { streamId: 'anotherStreamId', role: destinationRole }); - - expect(stack).toCountResources('AWS::Logs::LogGroup', 1); - expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupstreamId3B940622', - }, - }); - expect(stack.resolve(anotherLoggingOutput?.loggingOptions)).toMatchObject({ - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupanotherStreamIdF2754481', - }, - }); - }); - - it('does not re-use log group if log group provided', () => { - const loggingOutput = createLoggingOptions(stack, { streamId: 'streamId', role: destinationRole }); - const anotherLoggingOutput = createLoggingOptions(stack, { logGroup: new logs.LogGroup(stack, 'Another Log Group'), streamId: 'streamId', role: destinationRole }); - - expect(stack).toCountResources('AWS::Logs::LogGroup', 2); - expect(stack.resolve(loggingOutput?.loggingOptions)).toMatchObject({ - logGroupName: { - Ref: 'LogGroupF5B46931', - }, - logStreamName: { - Ref: 'LogGroupstreamId3B940622', - }, - }); - expect(stack.resolve(anotherLoggingOutput?.loggingOptions)).toMatchObject({ - logGroupName: { - Ref: 'AnotherLogGroup561B0F7C', - }, - logStreamName: { - Ref: 'AnotherLogGroupstreamId15DC3997', - }, - }); - }); - - it('grants log group write permissions to destination role', () => { - const logGroup = new logs.LogGroup(stack, 'Log Group'); - - createLoggingOptions(stack, { logGroup, streamId: 'streamId', role: destinationRole }); - - expect(stack).toHaveResourceLike('AWS::IAM::Policy', { - Roles: [stack.resolve(destinationRole.roleName)], - PolicyDocument: { - Statement: [ - { - Action: [ - 'logs:CreateLogStream', - 'logs:PutLogEvents', - ], - Effect: 'Allow', - Resource: stack.resolve(logGroup.logGroupArn), - }, - ], - }, - }); - }); - - it('returns log group grant as dependable', () => { - const loggingOutput = createLoggingOptions(stack, { logging: true, streamId: 'streamId', role: destinationRole }); - - expect(loggingOutput?.dependables?.length).toBe(1); - expect(loggingOutput?.dependables[0]).toBeInstanceOf(iam.Grant); - }); -}); diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts index a7d4e9ceb0827..7453f5b91183b 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -1,5 +1,5 @@ import '@aws-cdk/assert-internal/jest'; -import { ABSENT, Capture, MatchStyle, ResourcePart, anything } from '@aws-cdk/assert-internal'; +import { ABSENT, Capture, MatchStyle, ResourcePart, anything, arrayWith } from '@aws-cdk/assert-internal'; import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as logs from '@aws-cdk/aws-logs'; @@ -40,38 +40,6 @@ describe('S3 destination', () => { expect(stack).toHaveResource('AWS::Logs::LogStream'); }); - it('allows disabling logging', () => { - new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new firehosedestinations.S3Bucket(bucket, { - logging: false, - })], - }); - - expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { - ExtendedS3DestinationConfiguration: { - CloudWatchLoggingOptions: ABSENT, - }, - }); - }); - - it('allows providing a log group', () => { - const logGroup = logs.LogGroup.fromLogGroupName(stack, 'Log Group', 'evergreen'); - - new firehose.DeliveryStream(stack, 'DeliveryStream', { - destinations: [new firehosedestinations.S3Bucket(bucket, { - logGroup, - })], - }); - - expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { - ExtendedS3DestinationConfiguration: { - CloudWatchLoggingOptions: { - LogGroupName: 'evergreen', - }, - }, - }); - }); - it('creates a role when none is provided', () => { const capturedRoleArn = Capture.aString(); @@ -127,7 +95,7 @@ describe('S3 destination', () => { }); }); - test('bucket and log group grants are depended on by delivery stream', () => { + it('bucket and log group grants are depended on by delivery stream', () => { const logGroup = logs.LogGroup.fromLogGroupName(stack, 'Log Group', 'evergreen'); const capturedPolicyId = Capture.aString(); const destination = new firehosedestinations.S3Bucket(bucket, { role: destinationRole, logGroup }); @@ -171,4 +139,88 @@ describe('S3 destination', () => { DependsOn: [capturedPolicyId.capturedValue], }, ResourcePart.CompleteDefinition); }); + + describe('logging', () => { + it('creates resources and configuration by default', () => { + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket)], + }); + + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: anything(), + LogStreamName: anything(), + }, + }, + }); + }); + + it('does not create resources or configuration if disabled', () => { + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { logging: false })], + }); + + expect(stack).not.toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: ABSENT, + }, + }); + }); + + it('uses provided log group', () => { + const logGroup = new logs.LogGroup(stack, 'Log Group'); + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { logGroup })], + }); + + expect(stack).toCountResources('AWS::Logs::LogGroup', 1); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: stack.resolve(logGroup.logGroupName), + LogStreamName: anything(), + }, + }, + }); + }); + + it('throws error if logging disabled but log group provided', () => { + const destination = new firehosedestinations.S3Bucket(bucket, { logging: false, logGroup: new logs.LogGroup(stack, 'Log Group') }); + + expect(() => new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [destination], + })).toThrowError('logging cannot be set to false when logGroup is provided'); + }); + + it('grants log group write permissions to destination role', () => { + const logGroup = new logs.LogGroup(stack, 'Log Group'); + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { logGroup, role: destinationRole })], + }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + Roles: [stack.resolve(destinationRole.roleName)], + PolicyDocument: { + Statement: arrayWith( + { + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Effect: 'Allow', + Resource: stack.resolve(logGroup.logGroupArn), + }, + ), + }, + }); + }); + }); }); From 98d6371c0cedc30c64242522658a7463ea57b6f6 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Thu, 22 Jul 2021 14:18:47 -0700 Subject: [PATCH 48/52] use new lazy mappings for CIDR blocks --- .../test/integ.s3-bucket.expected.json | 2 +- .../aws-kinesisfirehose/lib/delivery-stream.ts | 17 +++++++++++------ .../test/delivery-stream.test.ts | 15 +++++++++++++-- .../test/integ.delivery-stream.expected.json | 2 +- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json index 3a5dfb8a72b7c..00bc62879e11e 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json @@ -327,7 +327,7 @@ } }, "Mappings": { - "DeliveryStreamFirehoseCIDRMappingE9233479": { + "awscdkawskinesisfirehoseCidrBlocks": { "af-south-1": { "FirehoseCidrBlock": "13.244.121.224/27" }, diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts index 51b159664e437..4968930808be6 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -3,7 +3,7 @@ import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as cdk from '@aws-cdk/core'; import { RegionInfo } from '@aws-cdk/region-info'; -import { Construct } from 'constructs'; +import { Construct, Node } from 'constructs'; import { IDestination } from './destination'; import { CfnDeliveryStream } from './kinesisfirehose.generated'; @@ -238,9 +238,12 @@ export class DeliveryStream extends DeliveryStreamBase { } function setConnections(scope: Construct) { - const region = cdk.Stack.of(scope).region; - let cidrBlock = RegionInfo.get(region).firehoseCidrBlock; - if (!cidrBlock) { + const stack = cdk.Stack.of(scope); + + const mappingId = '@aws-cdk/aws-kinesisfirehose.CidrBlocks'; + let cfnMapping = Node.of(stack).tryFindChild(mappingId) as cdk.CfnMapping; + + if (!cfnMapping) { const mapping: {[region: string]: { FirehoseCidrBlock: string }} = {}; RegionInfo.regions.forEach((regionInfo) => { if (regionInfo.firehoseCidrBlock) { @@ -249,12 +252,14 @@ function setConnections(scope: Construct) { }; } }); - const cfnMapping = new cdk.CfnMapping(scope, 'Firehose CIDR Mapping', { + cfnMapping = new cdk.CfnMapping(stack, mappingId, { mapping, + lazy: true, }); - cidrBlock = cdk.Fn.findInMap(cfnMapping.logicalId, region, 'FirehoseCidrBlock'); } + const cidrBlock = cfnMapping.findInMap(stack.region, 'FirehoseCidrBlock'); + return new ec2.Connections({ peer: ec2.Peer.ipv4(cidrBlock), }); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts index 6be168b64ae9b..bb9c3ba744a2f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -1,5 +1,5 @@ import '@aws-cdk/assert-internal/jest'; -import { ABSENT, ResourcePart } from '@aws-cdk/assert-internal'; +import { ABSENT, ResourcePart, SynthUtils, anything } from '@aws-cdk/assert-internal'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as cdk from '@aws-cdk/core'; @@ -194,7 +194,7 @@ describe('delivery stream', () => { { CidrIp: { 'Fn::FindInMap': [ - 'DeliveryStreamFirehoseCIDRMappingE9233479', + anything(), { Ref: 'AWS::Region', }, @@ -225,6 +225,17 @@ describe('delivery stream', () => { }); }); + test('only adds one Firehose IP address mapping to stack even if multiple delivery streams defined', () => { + new firehose.DeliveryStream(stack, 'Delivery Stream 1', { + destinations: [mockS3Destination], + }); + new firehose.DeliveryStream(stack, 'Delivery Stream 2', { + destinations: [mockS3Destination], + }); + + expect(Object.keys(SynthUtils.toCloudFormation(stack).Mappings).length).toBe(1); + }); + test('can add tags', () => { const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { destinations: [mockS3Destination], diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json index 21c29f8fee06f..f9e785a3def9e 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json @@ -113,7 +113,7 @@ } }, "Mappings": { - "DeliveryStreamFirehoseCIDRMappingE9233479": { + "awscdkawskinesisfirehoseCidrBlocks": { "af-south-1": { "FirehoseCidrBlock": "13.244.121.224/27" }, From e0064417eb955c67373bcf72dd0d31b84d989f52 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 22 Jul 2021 14:40:45 -0700 Subject: [PATCH 49/52] update S3 -> S3Bucket class name in default.ts-fixture --- .../aws-kinesisfirehose-destinations/rosetta/default.ts-fixture | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture index 8c002eb1618f7..fe46e06908b34 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture @@ -1,6 +1,6 @@ // Fixture with packages imported, but nothing else import { Construct } from '@aws-cdk/core'; -import { S3 } from '@aws-cdk/aws-kinesisfirehose-destinations'; +import { S3Bucket } from '@aws-cdk/aws-kinesisfirehose-destinations'; class Fixture extends Construct { constructor(scope: Construct, id: string) { From 86c7cb75cfcfd4c03229a7d531b2ca76b72ef4c1 Mon Sep 17 00:00:00 2001 From: Madeline Kusters <80541297+madeline-k@users.noreply.github.com> Date: Thu, 22 Jul 2021 15:47:54 -0700 Subject: [PATCH 50/52] Apply suggestions from code review Co-authored-by: Adam Ruka --- .../lib/s3-bucket.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts index 6e2a4faaab88d..ad3c0313ff061 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -23,15 +23,12 @@ export class S3Bucket implements firehose.IDestination { const bucketGrant = this.bucket.grantReadWrite(role); - const { loggingOptions, dependables: loggingDependables } = createLoggingOptions( - scope, - { - logging: this.props.logging, - logGroup: this.props.logGroup, - role, - streamId: 'S3Destination', - }, - ) ?? {}; + const { loggingOptions, dependables: loggingDependables } = createLoggingOptions(scope, { + logging: this.props.logging, + logGroup: this.props.logGroup, + role, + streamId: 'S3Destination', + }) ?? {}; return { extendedS3DestinationConfiguration: { From f8b03c41413e63479a4e18b83ab5773c57890007 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 22 Jul 2021 15:52:01 -0700 Subject: [PATCH 51/52] remove private index.ts --- .../aws-kinesisfirehose-destinations/lib/private/index.ts | 1 - 1 file changed, 1 deletion(-) delete mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts deleted file mode 100644 index c5f595cf9d428..0000000000000 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './helpers'; From 8c2903e9aabf5c3fc2b7b508f848499fe3fa5e40 Mon Sep 17 00:00:00 2001 From: Madeline Kusters Date: Thu, 22 Jul 2021 16:15:20 -0700 Subject: [PATCH 52/52] hardcode logical IDs in unit tests --- .../test/s3-bucket.test.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts index 7453f5b91183b..50e891b3091d1 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -1,5 +1,5 @@ import '@aws-cdk/assert-internal/jest'; -import { ABSENT, Capture, MatchStyle, ResourcePart, anything, arrayWith } from '@aws-cdk/assert-internal'; +import { ABSENT, MatchStyle, ResourcePart, anything, arrayWith } from '@aws-cdk/assert-internal'; import * as iam from '@aws-cdk/aws-iam'; import * as firehose from '@aws-cdk/aws-kinesisfirehose'; import * as logs from '@aws-cdk/aws-logs'; @@ -41,7 +41,6 @@ describe('S3 destination', () => { }); it('creates a role when none is provided', () => { - const capturedRoleArn = Capture.aString(); new firehose.DeliveryStream(stack, 'DeliveryStream', { destinations: [new firehosedestinations.S3Bucket(bucket)], @@ -51,14 +50,14 @@ describe('S3 destination', () => { ExtendedS3DestinationConfiguration: { RoleARN: { 'Fn::GetAtt': [ - capturedRoleArn.capture(), + 'DeliveryStreamS3DestinationRoleD96B8345', 'Arn', ], }, }, }); expect(stack).toMatchTemplate({ - [capturedRoleArn.capturedValue]: { + ['DeliveryStreamS3DestinationRoleD96B8345']: { Type: 'AWS::IAM::Role', }, }, MatchStyle.SUPERSET); @@ -97,15 +96,13 @@ describe('S3 destination', () => { it('bucket and log group grants are depended on by delivery stream', () => { const logGroup = logs.LogGroup.fromLogGroupName(stack, 'Log Group', 'evergreen'); - const capturedPolicyId = Capture.aString(); const destination = new firehosedestinations.S3Bucket(bucket, { role: destinationRole, logGroup }); - new firehose.DeliveryStream(stack, 'DeliveryStream', { destinations: [destination], }); expect(stack).toHaveResourceLike('AWS::IAM::Policy', { - PolicyName: capturedPolicyId.capture(), + PolicyName: 'DestinationRoleDefaultPolicy1185C75D', Roles: [stack.resolve(destinationRole.roleName)], PolicyDocument: { Statement: [ @@ -136,7 +133,7 @@ describe('S3 destination', () => { }, }); expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { - DependsOn: [capturedPolicyId.capturedValue], + DependsOn: ['DestinationRoleDefaultPolicy1185C75D'], }, ResourcePart.CompleteDefinition); });