From fd6c7bee89ccbdef68afe4b2e4b645628c777cda Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Tue, 8 Aug 2023 13:59:17 -0400 Subject: [PATCH 01/13] Remove legacy playbooks Signed-off-by: Webster Mudge --- README.md | 10 ++ main.yml | 101 ------------- pbc_setup.yml | 86 ------------ pbc_teardown.yml | 67 --------- pvc_base_postfix.yml | 205 --------------------------- pvc_base_prereqs_ext.yml | 296 --------------------------------------- pvc_base_prereqs_int.yml | 247 -------------------------------- pvc_base_setup.yml | 68 --------- pvc_base_teardown.yml | 85 ----------- quickstart.sh | 166 ---------------------- 10 files changed, 10 insertions(+), 1321 deletions(-) create mode 100644 README.md delete mode 100644 main.yml delete mode 100644 pbc_setup.yml delete mode 100644 pbc_teardown.yml delete mode 100644 pvc_base_postfix.yml delete mode 100644 pvc_base_prereqs_ext.yml delete mode 100644 pvc_base_prereqs_int.yml delete mode 100644 pvc_base_setup.yml delete mode 100644 pvc_base_teardown.yml delete mode 100755 quickstart.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..7b80e8d --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# Cloudera Deploy + +## cdp-navigator + +`cloudera-deploy` uses `ansible-navigator` to manage and execute the deployment definitions. Setting up `ansible-navigator` is straightforward; create and activate a new `virtualenv` and install the latest `ansible-core` and `ansible-navigator`. + +```bash +python -m venv ~/cdp-navigator; source ~/cdp-navigator/bin/activate; pip install ansible-core ansible-navigator +``` + diff --git a/main.yml b/main.yml deleted file mode 100644 index 979499d..0000000 --- a/main.yml +++ /dev/null @@ -1,101 +0,0 @@ ---- - -# Copyright 2022 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Primary entrypoint for all CDP installations (Public Cloud, Private Cloud) -# -# There are five "core" runlevels: -# - validate -# - initialize -# - infra -# - plat -# - run -# - teardown -# -# In this context, 'run' means 'runtime' or 'application'. The platform level -# is now both CDP Public Cloud SDX and Data Services, which is a change -# from before where 'plat' referred to the DL and SDX and 'run' referred to the -# Data Services. -# -# Individual Data Services (and Datahubs) should be tagged _within_ the -# 'cloudera.exe.*' roles themselves: -# - de -# - df -# - dh -# - dw -# - ml -# - opdb - -- name: Initialize Cloudera Deploy Run - hosts: localhost - connection: local - gather_facts: yes - tags: always - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - -- ansible.builtin.import_playbook: "{{ [definition_path, 'pre_teardown.yml'] | path_join }}" - when: - - hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_teardown - -- ansible.builtin.import_playbook: pbc_teardown.yml - when: - - hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown - - hostvars.localhost.init__call_cdp_pbc == True - -- ansible.builtin.import_playbook: pvc_base_teardown.yml - when: - - hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown - - hostvars.localhost.init__call_cdp_pvc == True - -- ansible.builtin.import_playbook: "{{ [definition_path, 'pre_setup.yml'] | path_join }}" - when: - - hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_runtime - -- ansible.builtin.import_playbook: pbc_setup.yml - when: - - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - - hostvars.localhost.init__call_cdp_pbc == True - -- ansible.builtin.import_playbook: pvc_base_prereqs_ext.yml - when: - - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - - hostvars.localhost.init__call_cdp_pvc == True - -- ansible.builtin.import_playbook: pvc_base_prereqs_int.yml - when: - - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - - hostvars.localhost.init__call_cdp_pvc == True - -- ansible.builtin.import_playbook: pvc_base_setup.yml - when: - - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - - hostvars.localhost.init__call_cdp_pvc == True - -- ansible.builtin.import_playbook: pvc_base_postfix.yml - when: - - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - - hostvars.localhost.init__call_cdp_pvc == True - -- ansible.builtin.import_playbook: "{{ [definition_path, 'post_setup.yml'] | path_join }}" - when: - - hostvars.localhost.run_runtime - -- ansible.builtin.import_playbook: "{{ [definition_path, 'post_teardown.yml'] | path_join }}" - when: - - hostvars.localhost.run_teardown diff --git a/pbc_setup.yml b/pbc_setup.yml deleted file mode 100644 index 70d521e..0000000 --- a/pbc_setup.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Init run tasks for Ansible Controller - hosts: localhost - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Set up CDP Public Cloud - hosts: localhost - connection: local - environment: "{{ globals.env_vars }}" - gather_facts: yes - tasks: - -# Inherits 'plat' and 'run' from playbook import -# Applies 'validate', and 'initialize' via the role imports -# By using imports vs includes for roles, we can propagate -# tags via --list-tags - -# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, -# including those that would otherwise be 'always' - in this context; 'always' -# should be reserved for the initialization of cloudera-deploy - - - name: Validate Platform configuration - ansible.builtin.import_role: - name: cloudera.exe.platform - tasks_from: validate - tags: - - validate - - - name: Validate Data Services configuration - ansible.builtin.import_role: - name: cloudera.exe.runtime - tasks_from: validate - tags: - - validate - - - name: Initialize Platform setup - ansible.builtin.import_role: - name: cloudera.exe.platform - tasks_from: initialize_setup - tags: - - initialize - - - name: Set up Platform - ansible.builtin.import_role: - name: cloudera.exe.platform - tasks_from: setup - - - name: Initialize Data Services setup - ansible.builtin.import_role: - name: cloudera.exe.runtime - tasks_from: initialize_setup - tags: - - initialize - - - name: Set up Data Services - ansible.builtin.import_role: - name: cloudera.exe.runtime - tasks_from: setup diff --git a/pbc_teardown.yml b/pbc_teardown.yml deleted file mode 100644 index a26acee..0000000 --- a/pbc_teardown.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Init run tasks for Ansible Controller - hosts: localhost - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Tear down CDP Public Cloud - hosts: localhost - environment: "{{ globals.env_vars }}" - gather_facts: yes - tasks: - -# Inherits 'infra', 'teardown', and 'never' from the playbook import -# Does not apply other tags via the role imports -# By using imports vs includes for roles, we can propagate -# tags via --list-tags - -# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, -# including those that would otherwise be 'always' - in this context; 'always' -# should be reserved for the initialization of cloudera-deploy - - - name: Initialize Data Services teardown - ansible.builtin.import_role: - name: cloudera.exe.runtime - tasks_from: initialize_teardown - - - name: Tear down Data Services - ansible.builtin.import_role: - name: cloudera.exe.runtime - tasks_from: teardown - - - name: Initialize Platform teardown - ansible.builtin.import_role: - name: cloudera.exe.platform - tasks_from: initialize_teardown - - - name: Tear down Platform - ansible.builtin.import_role: - name: cloudera.exe.platform - tasks_from: teardown diff --git a/pvc_base_postfix.yml b/pvc_base_postfix.yml deleted file mode 100644 index d670c31..0000000 --- a/pvc_base_postfix.yml +++ /dev/null @@ -1,205 +0,0 @@ ---- - -# Copyright 2022 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This Playbook is specifically for Deploying Cloudera Clusters -# Edit with extreme caution -# This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml - -# STARTBLOCK # Init run -- name: Init run tasks for Ansible Controller - hosts: localhost - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always -# ENDBLOCK # Init run -# STARTBLOCK # Fix Auto-TLS - -- name: Auto-TLS Services Setup - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.cms_tls - when: autotls is defined and autotls == True - tags: - - autotls - - never - -# ENDBLOCK # Fix Auto-TLS -# STARTBLOCK # Setup HDFS Encryption - -- name: Setup KTS HA - hosts: localhost - become: yes - gather_facts: no - tasks: - - ansible.builtin.include_role: - name: cloudera.cluster.deployment.services.kts_high_availability - public: yes - apply: - tags: - - kts - - full_cluster - when: - - "'kts_active' in groups" - - "'kts_passive' in groups" - tags: - - kts - - full_cluster - -- name: Handle KMS services - hosts: localhost - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.deployment.services.kms - when: "'kms_servers' in groups" - - role: cloudera.cluster.deployment.services.kms_ha - when: "'kms_servers' in groups" - tags: - - kms - - full_cluster - -- name: Handle KMS services - hosts: cloudera_manager - gather_facts: no - become: no - roles: - - role: cloudera.cluster.operations.refresh_ranger_kms_repo - when: "'kms_servers' in groups" - tags: - - kms - - full_cluster - -- name: Restart and re-deploy stale client configs - hosts: localhost - gather_facts: no - roles: - - role: cloudera.cluster.operations.restart_stale - when: "'kms_servers' in groups" - vars: - client_config_timeout: "{{ restart_client_config_timeout | default(300) }}" - tags: - - kms - - restart_stale - - full_cluster - - # ENDBLOCK # Setup HDFS Encryption - # STARTBLOCK # WXM Setup - -- name: Handle WXM Setup - hosts: "{{ tp_host | default('cluster_master_nodes[0]') }}" - gather_facts: yes - tags: - - wxm - - full_cluster - tasks: - - name: Setup WXM - when: - - use_wxm | default(False) - - altus_key_id | length > 0 - - altus_private_key | length > 0 - import_role: - name: cloudera.cluster.deployment.services.wxm - -# ENDBLOCK # WXM Setup - -- name: Post-Install for PvC on all cluster hosts - hosts: cloudera_manager, cluster, ecs_nodes - gather_facts: yes - tags: - - pvc - tasks: - - name: Add missing ExtJS for Oozie UI - include_role: - name: cloudera.cluster.config.services.oozie_ui - when: oozie_service_exists | default(false) - -- name: Post-Install Cloudera Manager and Cluster - hosts: cloudera_manager - gather_facts: yes - tags: - - pvc - tasks: - - name: Refresh CM Services Info - include_role: - name: cloudera.cluster.cloudera_manager.services_info - public: yes - vars: - cluster_name: "{{ cluster_name_base }}" - - - name: Fix Hue ticket lifetime for Free IPA - include_role: - name: cloudera.cluster.config.services.hue_ticket_lifetime - when: - - hue_service_exists | default(false) - - krb5_kdc_type == 'Red Hat IPA' - - - name: Set Cloudera Manager session timeout to 30d - include_role: - name: cloudera.cluster.cloudera_manager.session_timeout - -## TODO Believe this is version specific, in 7.1.7Spx solr plugin is NOT missing, needs work -# - name: Create missing Solr plugin for Ranger -# include_role: -# name: cloudera.cluster.config.services.solr_ranger_plugin -# when: -# - ranger_service_exists | default(false) -# - solr_service_exists | default(false) - - - name: Add Solr urls to Knox - include_role: - name: cloudera.cluster.config.services.solr_knox - when: knox_service_exists | default(false) and solr_service_exists | default(false) - - - name: Add missing TLS values for KMS - when: - - kms_service_exists | default(False) - - (autotls | default(False)) - include_role: - name: cloudera.cluster.config.services.kms_tls - -## TODO Believe this is version specific, in 7.1.7SPx Ranger policies are duplicated, needs work -# - name: Ranger extra policies to have enough rights with basic accounts -# when: ranger_service_exists | default(false) -# include_role: -# name: cloudera.cluster.config.services.ranger_pvc_default_policies - -# End run -### \ No newline at end of file diff --git a/pvc_base_prereqs_ext.yml b/pvc_base_prereqs_ext.yml deleted file mode 100644 index d4ddbf6..0000000 --- a/pvc_base_prereqs_ext.yml +++ /dev/null @@ -1,296 +0,0 @@ ---- - -# Copyright 2022 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This Playbook is specifically for Deploying Cloudera Clusters -# Edit with extreme caution - -# STARTBLOCK # Init run -- name: Init run tasks for Ansible Controller - hosts: localhost - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always -# ENDBLOCK # Init run - -# STARTBLOCK # Verify Inventory and Definition -- name: Verify inventory [verify_inventory] - hosts: localhost - gather_facts: no - roles: - - cloudera.cluster.verify.inventory - tags: - - verify - - verify_inventory - - default_cluster - - full_cluster - -- name: Verify definition [verify_definition] - hosts: cloudera_manager - gather_facts: yes - roles: - - cloudera.cluster.verify.definition - tags: - - verify - - verify_definition - - default_cluster - - full_cluster - -# Moved before parcel verification to allow rehosting -- name: Install custom parcel repository - hosts: custom_repo - become: yes - roles: - - cloudera.cluster.infrastructure.custom_repo - tags: - - custom_repo - - default_cluster - - full_cluster - -# Moved from verify_parcels to reduce duplication -- name: Verify definition [verify_parcels_and_roles] - hosts: cloudera_manager - gather_facts: yes - roles: - - cloudera.cluster.verify.parcels_and_roles - tags: - - verify - - verify_parcels - - default_cluster - - full_cluster - -# ENDBLOCK # Verify Inventory and Definition -# STARTBLOCK # Prepare Nodes - -- name: Apply OS pre-requisite configurations - hosts: cloudera_manager, cluster, ca_server, ecs_nodes - become: yes - roles: - - cloudera.cluster.prereqs.os - tags: - - os - - default_cluster - - full_cluster - -- name: Apply OS Prereqs to ECS Nodes - hosts: ecs_nodes - gather_facts: yes - become: yes - tags: - - pvc - - os - - default_cluster - - full_cluster - tasks: - - name: Setup OS Prereqs for ECS Nodes - ansible.builtin.include_role: - name: cloudera.cluster.prereqs.pvc_ecs - -- name: Create local user accounts - hosts: cloudera_manager, cluster - become: yes - gather_facts: no - roles: - - cloudera.cluster.prereqs.user_accounts - tags: - - users - - default_cluster - - full_cluster - -- name: Create local users on ECS Nodes - hosts: ecs_nodes - become: yes - gather_facts: yes - tags: - - pvc - - users - - default_cluster - - full_cluster - tasks: - - name: Create user accounts for ECS nodes - ansible.builtin.include_role: - name: cloudera.cluster.prereqs.user_accounts_ecs - -- name: Install JDK - hosts: cloudera_manager, cluster, tls, krb5_server, ecs_nodes - become: yes - roles: - - cloudera.cluster.prereqs.jdk - tags: - - jdk - - security - - free_ipa - - kerberos - - tls - - default_cluster - - full_cluster - -# DB Connectors -- name: Install MySQL Connector - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.prereqs.mysql_connector - when: database_type == 'mysql' or database_type == 'mariadb' - tags: - - mysql_connector - - full_cluster - -- name: Install Oracle Connector - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.prereqs.oracle_connector - when: database_type == 'oracle' - tags: - - oracle_connector - - full_cluster - -# ENDBLOCK # Prepare Nodes -# STARTBLOCK # Create Cluster Service Infrastructure - -- name: Install Kerberos Server - hosts: krb5_server - gather_facts: yes - become: yes - roles: - - role: cloudera.cluster.infrastructure.krb5_server - tags: - - security - - kerberos - - free_ipa - - tls - - full_cluster - -- name: Setup KRB5 clients - hosts: cloudera_manager, cluster, ecs_nodes - gather_facts: yes - become: yes - roles: - - role: cloudera.cluster.infrastructure.krb5_client - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - security - - kerberos - - free_ipa - - tls - - full_cluster - -- name: Install CA server - hosts: ca_server - become: yes - roles: - - cloudera.cluster.infrastructure.ca_server - tags: - - security - - tls - - full_cluster - -- name: Install HAProxy - hosts: haproxy - become: yes - roles: - - cloudera.cluster.infrastructure.haproxy - tags: - - ha - - full_cluster - -# ENDBLOCK # Create Cluster Service Infrastructure -# STARTBLOCK # Prepare TLS - -- name: Build TLS keystores and truststores - hosts: tls - become: yes - gather_facts: no - roles: - - cloudera.cluster.security.tls_generate_csr - - cloudera.cluster.security.tls_signing - - cloudera.cluster.security.tls_install_certs - vars: - local_certs_dir: "{{ local_temp_dir }}/certs" - local_csrs_dir: "{{ local_temp_dir }}/csrs" - tags: - - tls - - full_cluster - -- name: Delete temp directories - hosts: localhost - become: no - gather_facts: no - tasks: - - file: - path: "{{ [local_temp_dir, dir] | path_join }}" - state: absent - loop: - - csrs - - certs - loop_control: - loop_var: dir - tags: - - always - -# ENDBLOCK # Prepare TLS -# STARTBLOCK # NiFi TLS - -- name: Setup symlinks for NiFi TLS keystore and truststore - hosts: cluster - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.security.tls_nifi - when: > - (tls | default(False) - or manual_tls_cert_distribution | default(False)) - and not (autotls | default(False)) -# ENDBLOCK # NiFi TLS -# STARTBLOCK # Install Cluster Service Infrastructure II - -- name: Install RDBMS - hosts: db_server - become: yes - roles: - - cloudera.cluster.infrastructure.rdbms - tags: - - database - - default_cluster - - full_cluster - -# ENDBLOCK # Install Cluster Service Infrastructure II \ No newline at end of file diff --git a/pvc_base_prereqs_int.yml b/pvc_base_prereqs_int.yml deleted file mode 100644 index 5f892cc..0000000 --- a/pvc_base_prereqs_int.yml +++ /dev/null @@ -1,247 +0,0 @@ ---- - -# Copyright 2022 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This Playbook is specifically for Deploying Cloudera Clusters -# Edit with extreme caution - -# STARTBLOCK # Init run -- name: Init run tasks for Ansible Controller - hosts: localhost - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always -# ENDBLOCK # Init run - -# STARTBLOCK # Install Cloudera Manager - -- name: Install Cloudera Manager daemons - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - any_errors_fatal: true - roles: - - role: cloudera.cluster.cloudera_manager.daemons - tags: - - cm - - default_cluster - - full_cluster - -- name: Install Cloudera Manager server - hosts: cloudera_manager - become: yes - roles: - - role: cloudera.cluster.cloudera_manager.server - tags: - - cm - - default_cluster - - full_cluster - -- name: Install Cloudera Manager License - hosts: cloudera_manager - become: yes - roles: - - role: cloudera.cluster.cloudera_manager.license - tags: - - cm - - license - - default_cluster - - full_cluster - -- name: Install Cloudera Manager agents - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - any_errors_fatal: true - roles: - - role: cloudera.cluster.cloudera_manager.agent - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager server for TLS - hosts: cloudera_manager - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.server_tls - when: tls | default(False) or manual_tls_cert_distribution | default(False) - tags: - - tls - - cm - - full_cluster - -- name: Configure Cloudera Manager agents - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - any_errors_fatal: true - roles: - - cloudera.cluster.cloudera_manager.agent_config - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager server - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.cloudera_manager.config - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager auth and accounts - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.external_auth - - role: cloudera.cluster.cloudera_manager.external_account - tags: - - cm - - full_cluster - -# ENDBLOCK # Install Cloudera Manager -# STARTBLOCK # Cloudera Manager Password - -- name: Configure Cloudera Manager Password - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.admin_password.set - tags: - - cm - - default_cluster - - full_cluster - -- name: Check Cloudera Manager admin password - hosts: cloudera_manager, cluster - gather_facts: no - roles: - - cloudera.cluster.cloudera_manager.api_client - tags: - - cm - - default_cluster - - full_cluster - -# ENDBLOCK # Cloudera Manager Password -# STARTBLOCK # Prepare Security - -- name: Enable Auto-TLS - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.autotls - when: autotls is defined and autotls == True - tags: - - autotls - - full_cluster - -- name: Install pre-requisite packages for Kerberos - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - roles: - - role: cloudera.cluster.prereqs.kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - prereqs - - full_cluster - -- name: Configure Cloudera Manager server for Kerberos - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - full_cluster - -# ENDBLOCK # Prepare Security -# STARTBLOCK # Configure CM - -- name: Restart Cloudera Manager Agents - hosts: cloudera_manager, cluster, ecs_nodes - gather_facts: no - become: yes - tasks: - - name: Restart Cloudera Manager Agents - meta: noop - notify: - - restart cloudera-scm-agent - tags: - - never - - restart_agents - -- name: Ensure that the agents are heartbeating - hosts: cloudera_manager, cluster, ecs_nodes - gather_facts: yes - any_errors_fatal: yes - roles: - - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat - when: cloudera_manager_agent_wait_for_heartbeat | default(True) - tags: - - heartbeat - - default_cluster - - full_cluster - -- name: Deploy Cloudera Management Service - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.deployment.services.mgmt - tags: - - mgmt - - default_cluster - - full_cluster - -- name: Preload parcels from custom repo to Cloudera Manager - hosts: cloudera_manager - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.preload_parcels - when: "'custom_repo' in groups" - tags: - - preload_parcels - - default_cluster - - full_cluster - -# ENDBLOCK # Configure CM diff --git a/pvc_base_setup.yml b/pvc_base_setup.yml deleted file mode 100644 index bcd2567..0000000 --- a/pvc_base_setup.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- - -# Copyright 2022 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This Playbook is specifically for Deploying Cloudera Clusters -# Edit with extreme caution -# This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml - -# STARTBLOCK # Init run -- name: Init run tasks for Ansible Controller - hosts: localhost - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always -# ENDBLOCK # Init run -# STARTBLOCK # Install Cluster - -- name: Deploy clusters - hosts: cloudera_manager - gather_facts: yes - roles: - - cloudera.cluster.deployment.cluster - tags: - - cluster - - pvc - - default_cluster - - full_cluster - -# ENDBLOCK # Install Cluster - -# End run -### \ No newline at end of file diff --git a/pvc_base_teardown.yml b/pvc_base_teardown.yml deleted file mode 100644 index 7e997eb..0000000 --- a/pvc_base_teardown.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- - -# Copyright 2022 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# STARTBLOCK # Init run -- name: Init run tasks for Ansible Controller - hosts: localhost - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always -# ENDBLOCK # Init run - -# STARTBLOCK # Teardown -# Teardown CA - -- name: Teardown CA server - hosts: ca_server - gather_facts: no - become: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.cluster.infrastructure.ca_certs - tasks_from: clean.yml - tags: - - teardown_ca - -- name: Teardown security artifact directories - hosts: tls - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.security.tls_clean - when: "'tls' in groups" - tags: - - teardown_tls - -# Teardown Cluster - -- name: Teardown ECS - hosts: ecs_nodes - gather_facts: no - become: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.cluster.teardown - tasks_from: teardown_ecs.yml - vars: - cluster: "{{ definition.clusters | selectattr('type', 'defined') | selectattr('type', 'search', 'ecs') | first }}" - tags: - - teardown_ecs - -- name: Cluster Teardown Process - hosts: all - gather_facts: no - become: yes - any_errors_fatal: true - roles: - - cloudera.cluster.teardown - tags: - - teardown_cluster - -# ENDBLOCK # Teardown \ No newline at end of file diff --git a/quickstart.sh b/quickstart.sh deleted file mode 100755 index 38a95b3..0000000 --- a/quickstart.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -IMAGE_NAME="${image_name:-ghcr.io/cloudera-labs/cldr-runner}" -PROVIDER="${provider:-full}" -IMAGE_VER="${image_ver:-latest}" -IMAGE_NO_PULL="${no_pull:+true}" -CONTAINER_NAME="${container:-cloudera-deploy}" - -IMAGE_TAG="${PROVIDER}-${IMAGE_VER}" -IMAGE_FULL_NAME="${IMAGE_NAME}:${IMAGE_TAG}" - -# dir of script -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; -# parent dir of that dir -PARENT_DIRECTORY="${DIR%/*}" - -PROJECT_DIR="${1:-${PARENT_DIRECTORY}}" - -echo "Checking if Docker is running..." -{ docker info >/dev/null 2>&1; echo "Docker OK"; } || { echo "Docker is required and does not seem to be running - please start Docker and retry" ; exit 1; } - -if [ "${IMAGE_NO_PULL}" != true ]; then - echo "Checking for updated execution container image '${IMAGE_FULL_NAME}'" - docker pull "${IMAGE_FULL_NAME}" -fi - -echo "Ensuring default credential paths are available in calling using profile for mounting to execution environment" -for thisdir in ".aws" ".ssh" ".cdp" ".azure" ".kube" ".config" ".config/cloudera-deploy/log" ".config/cloudera-deploy/profiles" -do - mkdir -p "${HOME}"/$thisdir -done - -echo "Ensure Default profile is present" -if [ ! -f "${HOME}"/.config/cloudera-deploy/profiles/default ]; then - if [ ! -f "${DIR}/profile.yml" ]; then - curl "https://raw.githubusercontent.com/cloudera-labs/cloudera-deploy/main/profile.yml" -o "${HOME}"/.config/cloudera-deploy/profiles/default - else - cp "${DIR}/profile.yml" "${HOME}"/.config/cloudera-deploy/profiles/default - fi -fi - -# If CLDR_COLLECTION_PATH is set, the default version in the container will be removed and this path added to the Ansible Collection path -# The path supplied must be relative to PROJECT_DIR, e.g. ansible_dev/collections -if [ -n "${CLDR_COLLECTION_PATH}" ]; then - echo "Path to custom Cloudera Collection supplied as ${CLDR_COLLECTION_PATH}, adding to Ansible Collection path" - ANSIBLE_COLLECTIONS_PATH="/opt/cldr-runner/collections:/runner/project/${CLDR_COLLECTION_PATH}" - QUICKSTART_PROMPT='Quickstart? Run this command -- ansible-playbook project/cloudera-deploy/main.yml -e "definition_path=examples/sandbox" -t run,default_cluster' -else - echo "Custom Cloudera Collection path not found" - ANSIBLE_COLLECTIONS_PATH="/opt/cldr-runner/collections" - QUICKSTART_PROMPT='Quickstart? Run this command -- ansible-playbook /opt/cloudera-deploy/main.yml -e "definition_path=examples/sandbox" -t run,default_cluster' -fi - -# If CLDR_PYTHON_PATH is set, that will be set as the system PYTHONPATH variable in the container -# This is a good way to point at any custom python source code in your /runner/project mount, including CDPY -# The path supplied must be a full path to the source root for each source included, e.g /runner/project/cdpy/src -if [ -n "${CLDR_PYTHON_PATH}" ]; then - echo "Path to custom Python sourcecode supplied as ${CLDR_PYTHON_PATH}, setting as System PYTHONPATH" - PYTHONPATH="${CLDR_PYTHON_PATH}" -else - echo "'CLDR_PYTHON_PATH' is not set, skipping setup of PYTHONPATH in execution container" -fi - -echo "Checking if ssh-agent is running..." -if pgrep -x "ssh-agent" >/dev/null -then - echo "ssh-agent OK" -else - echo "ssh-agent is stopped, please start it by running: eval \`ssh-agent -s\` " - exit 1 -fi - -echo "Checking OS" -if [ ! -f "/run/host-services/ssh-auth.sock" ]; -then - if [ -n "${SSH_AUTH_SOCK}" ]; - then - SSH_AUTH_SOCK=${SSH_AUTH_SOCK} - else - echo "ERROR: SSH_AUTH_SOCK is empty or not set, unable to proceed. Exiting" - exit 1 - fi -else - SSH_AUTH_SOCK=${SSH_AUTH_SOCK} -fi - -echo "SSH authentication for container taken from ${SSH_AUTH_SOCK}" - -if [ ! "$(docker ps -q -f name=${CONTAINER_NAME})" ]; then - if [ "$(docker ps -aq -f status=exited -f name=${CONTAINER_NAME})" ]; then - # cleanup if exited - echo "Attempting removal of exited execution container named '${CONTAINER_NAME}'" - docker rm "${CONTAINER_NAME}" >/dev/null 2>&1 || echo "Execution container '${CONTAINER_NAME}' already removed, continuing..." - fi - # create new container if not running - echo "Creating new execution container named '${CONTAINER_NAME}' with '${PROJECT_DIR}' mounted to /runner/project from image '${IMAGE_FULL_NAME}'" - docker run -td \ - --detach-keys="ctrl-@" \ - -v "${PROJECT_DIR}":/runner/project \ - --mount type=bind,src="${SSH_AUTH_SOCK}",target=/run/host-services/ssh-auth.sock \ - -e SSH_AUTH_SOCK="/run/host-services/ssh-auth.sock" \ - -e ANSIBLE_LOG_PATH="/home/runner/.config/cloudera-deploy/log/${CLDR_BUILD_VER:-latest}-$(date +%F_%H%M%S)" \ - -e ANSIBLE_INVENTORY="inventory" \ - -e ANSIBLE_CALLBACK_WHITELIST="ansible.posix.profile_tasks" \ - -e ANSIBLE_GATHERING="smart" \ - -e ANSIBLE_DEPRECATION_WARNINGS=false \ - -e ANSIBLE_HOST_KEY_CHECKING=false \ - -e ANSIBLE_SSH_RETRIES=10 \ - -e ANSIBLE_COLLECTIONS_PATH="${ANSIBLE_COLLECTIONS_PATH}" \ - -e PYTHONPATH="${PYTHONPATH}" \ - -e ANSIBLE_ROLES_PATH="/opt/cldr-runner/roles" \ - -e AWS_DEFAULT_OUTPUT="json" \ - --mount "type=bind,source=${HOME}/.aws,target=/home/runner/.aws" \ - --mount "type=bind,source=${HOME}/.config,target=/home/runner/.config" \ - --mount "type=bind,source=${HOME}/.ssh,target=/home/runner/.ssh" \ - --mount "type=bind,source=${HOME}/.cdp,target=/home/runner/.cdp" \ - --mount "type=bind,source=${HOME}/.azure,target=/home/runner/.azure" \ - --mount "type=bind,source=${HOME}/.kube,target=/home/runner/.kube" \ - --network="host" \ - --name "${CONTAINER_NAME}" \ - "${IMAGE_FULL_NAME}" \ - /usr/bin/env bash - - echo "Installing the cloudera-deploy project to the execution container '${CONTAINER_NAME}'" - docker exec -td "${CONTAINER_NAME}" /usr/bin/env git clone https://github.com/cloudera-labs/cloudera-deploy.git /opt/cloudera-deploy --depth 1 - - if [ -n "${CLDR_COLLECTION_PATH}" ]; then - docker exec -td "${CONTAINER_NAME}" /usr/bin/env rm -rf /opt/cldr-runner/collections/ansible_collections/cloudera - fi - if [ -n "${CLDR_PYTHON_PATH}" ]; then - docker exec -td "${CONTAINER_NAME}" pip uninstall -y cdpy - fi -fi - -cat < Date: Tue, 8 Aug 2023 14:00:04 -0400 Subject: [PATCH 02/13] Update CML definition to use ansible-navigator Signed-off-by: Webster Mudge --- .../aws/cml/.gitignore | 16 +--- public-cloud/aws/cml/README.md | 76 +++++++++++++++++++ public-cloud/aws/cml/ansible-navigator.yml | 65 ++++++++++++++++ .../aws}/cml/definition.yml | 29 +++++-- public-cloud/aws/cml/inventory.ini | 15 ++++ public-cloud/aws/cml/main.yml | 34 +++++++++ public-cloud/aws/cml/teardown.yml | 34 +++++++++ 7 files changed, 252 insertions(+), 17 deletions(-) rename examples/cml/application.yml => public-cloud/aws/cml/.gitignore (71%) create mode 100644 public-cloud/aws/cml/README.md create mode 100644 public-cloud/aws/cml/ansible-navigator.yml rename {examples => public-cloud/aws}/cml/definition.yml (55%) create mode 100644 public-cloud/aws/cml/inventory.ini create mode 100644 public-cloud/aws/cml/main.yml create mode 100644 public-cloud/aws/cml/teardown.yml diff --git a/examples/cml/application.yml b/public-cloud/aws/cml/.gitignore similarity index 71% rename from examples/cml/application.yml rename to public-cloud/aws/cml/.gitignore index cc80908..dbf25ad 100644 --- a/examples/cml/application.yml +++ b/public-cloud/aws/cml/.gitignore @@ -1,6 +1,4 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,12 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Coda - hosts: localhost - connection: local - gather_facts: no - become: no - tasks: - - name: Deployment results - debug: - msg: Success! \ No newline at end of file +ansible-navigator.log +runs +context diff --git a/public-cloud/aws/cml/README.md b/public-cloud/aws/cml/README.md new file mode 100644 index 0000000..f5bcb6f --- /dev/null +++ b/public-cloud/aws/cml/README.md @@ -0,0 +1,76 @@ +# CDP Public Cloud - Cloudera Machine Learning (CML) + +> Constructs a set of Cloudera Machine Learning (CML) workspaces within their own CDP Public Cloud Environment and Datalake. Generates via Ansible the AWS infrastructure and CDP artifacts, including SSH key, cross-account credentials, S3 buckets, etc. + +## Requirements + +To run, you need: + +* Docker (or a Docker clone[^1]) +* AWS credentials (set via `AWS_PROFILE`) +* CDP credentials (set via `CDP_PROFILE`) + +[^1]: [OrbStack](https://orbstack.dev) works well on OSX. + +## Set Up + +First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). + +Then, clone this project and change your working directory. + +```bash +git clone https://github.com/cloudera-labs/cloudera-deploy.git; cd cloudera-deploy/public-cloud/aws/cml +``` + +## Configure + +Set the required environment variables: + +```bash +export AWS_PROFILE=your-aws-profile +export CDP_PROFILE=your-cdp-profile +``` + +Tweak the `definition.yml` parameters to your liking. Notably, you should add and/or change: + +```yaml +name_prefix: ex01 # Keep this short (4-7 characters) +admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. +infra_region: us-east-2 +``` + +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. + +### SSH Keys + +This definition will create a new SSH keypair on the host in your `~/.ssh` directory if you do not specify a SSH public key. If you wish to use an existing SSH key already loaded into AWS, set `public_key_id` to the key's label. If you wish to use an existing SSH key, but need to have it loaded into AWS, then set `public_key_file` to the key's path. + +## Execute + +Then set up the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run main.yml +``` + +## Tear Down + +Tear down the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run teardown.yml +``` + +## FAQ + +### Using tags + +If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. + +### Using _extra variables_ + +Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. + +### Using Ansible collection and role paths + +Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. diff --git a/public-cloud/aws/cml/ansible-navigator.yml b/public-cloud/aws/cml/ansible-navigator.yml new file mode 100644 index 0000000..a0dc3ef --- /dev/null +++ b/public-cloud/aws/cml/ansible-navigator.yml @@ -0,0 +1,65 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + ansible: + inventory: + entries: + - inventory.ini + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - AWS_PROFILE + - CDP_PROFILE + set: + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + image: ghcr.io/cloudera-labs/cldr-runner:aws-devel + pull: + arguments: + - "--tls-verify=false" + volume-mounts: + - src: "${ANSIBLE_COLLECTIONS_PATH}" + dest: "${ANSIBLE_COLLECTIONS_PATH}" + options: "Z" + - src: "~/.aws" + dest: "/runner/.aws" + options: "Z" + - src: "~/.cdp" + dest: "/runner/.cdp" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/examples/cml/definition.yml b/public-cloud/aws/cml/definition.yml similarity index 55% rename from examples/cml/definition.yml rename to public-cloud/aws/cml/definition.yml index 8db362a..eb95ae4 100644 --- a/examples/cml/definition.yml +++ b/public-cloud/aws/cml/definition.yml @@ -1,6 +1,6 @@ --- -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +################################################################################ +# Global variables +################################################################################ +name_prefix: cmlex # name prefix for everything +infra_region: us-east-2 # CSP region for infra +infra_type: aws # CSP + +# globals: +# force_teardown: yes + +################################################################################ +# CDP Environment and Datalake variables +################################################################################ +env: + public_endpoint_access: yes + tunnel: yes + +################################################################################ +# Machine Learning Configurations +################################################################################ ml: definitions: - tls: yes @@ -21,6 +41,7 @@ ml: governance: yes metrics: yes public_loadbalancer: yes + ip_addresses: [ "0.0.0.0/0" ] # Unrestricted instance_groups: - name: cpu_settings autoscaling: @@ -29,9 +50,8 @@ ml: instanceCount: 0 instanceTier: "ON_DEMAND" instanceType: m5.4xlarge # AWS -# instanceType: Standard_D8_v3 # Azure rootVolume: - size: 100 + size: 300 - name: gpu_settings autoscaling: maxInstances: 1 @@ -39,6 +59,5 @@ ml: instanceCount: 0 instanceTier: "ON_DEMAND" instanceType: "p2.8xlarge" # AWS -# instanceType: Standard_D8_v3 # Azure rootVolume: - size: 100 \ No newline at end of file + size: 300 diff --git a/public-cloud/aws/cml/inventory.ini b/public-cloud/aws/cml/inventory.ini new file mode 100644 index 0000000..fef0f4a --- /dev/null +++ b/public-cloud/aws/cml/inventory.ini @@ -0,0 +1,15 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" \ No newline at end of file diff --git a/public-cloud/aws/cml/main.yml b/public-cloud/aws/cml/main.yml new file mode 100644 index 0000000..7d88620 --- /dev/null +++ b/public-cloud/aws/cml/main.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Set up CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_setup.yml + +- name: Set up CDP Public Cloud (CML example) + ansible.builtin.import_playbook: cloudera.exe.pbc_setup.yml diff --git a/public-cloud/aws/cml/teardown.yml b/public-cloud/aws/cml/teardown.yml new file mode 100644 index 0000000..d3b2e5f --- /dev/null +++ b/public-cloud/aws/cml/teardown.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Tear down CDP Public Cloud (CML example) + ansible.builtin.import_playbook: cloudera.exe.pbc_teardown.yml + +- name: Tear down CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_teardown.yml From 5bb6f6b78364f3c198e3acb160742d34438d3346 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Tue, 8 Aug 2023 16:23:31 -0400 Subject: [PATCH 03/13] Add AWS base definition for Env and DL only Signed-off-by: Webster Mudge --- public-cloud/aws/base/.gitignore | 17 +++++ public-cloud/aws/base/README.md | 76 +++++++++++++++++++++ public-cloud/aws/base/ansible-navigator.yml | 65 ++++++++++++++++++ public-cloud/aws/base/definition.yml | 32 +++++++++ public-cloud/aws/base/inventory.ini | 15 ++++ public-cloud/aws/base/main.yml | 34 +++++++++ public-cloud/aws/base/teardown.yml | 34 +++++++++ 7 files changed, 273 insertions(+) create mode 100644 public-cloud/aws/base/.gitignore create mode 100644 public-cloud/aws/base/README.md create mode 100644 public-cloud/aws/base/ansible-navigator.yml create mode 100644 public-cloud/aws/base/definition.yml create mode 100644 public-cloud/aws/base/inventory.ini create mode 100644 public-cloud/aws/base/main.yml create mode 100644 public-cloud/aws/base/teardown.yml diff --git a/public-cloud/aws/base/.gitignore b/public-cloud/aws/base/.gitignore new file mode 100644 index 0000000..dbf25ad --- /dev/null +++ b/public-cloud/aws/base/.gitignore @@ -0,0 +1,17 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator.log +runs +context diff --git a/public-cloud/aws/base/README.md b/public-cloud/aws/base/README.md new file mode 100644 index 0000000..faaf2e8 --- /dev/null +++ b/public-cloud/aws/base/README.md @@ -0,0 +1,76 @@ +# CDP Public Cloud - Environment and Datalake Base Example + +> Constructs a CDP Public Cloud Environment and Datalake. Generates via Ansible the AWS infrastructure and CDP artifacts, including SSH key, cross-account credentials, S3 buckets, etc. + +## Requirements + +To run, you need: + +* Docker (or a Docker clone[^1]) +* AWS credentials (set via `AWS_PROFILE`) +* CDP credentials (set via `CDP_PROFILE`) + +[^1]: [OrbStack](https://orbstack.dev) works well on OSX. + +## Set Up + +First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). + +Then, clone this project and change your working directory. + +```bash +git clone https://github.com/cloudera-labs/cloudera-deploy.git; cd cloudera-deploy/public-cloud/aws/base +``` + +## Configure + +Set the required environment variables: + +```bash +export AWS_PROFILE=your-aws-profile +export CDP_PROFILE=your-cdp-profile +``` + +Tweak the `definition.yml` parameters to your liking. Notably, you should add and/or change: + +```yaml +name_prefix: ex01 # Keep this short (4-7 characters) +admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. +infra_region: us-east-2 +``` + +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. + +### SSH Keys + +This definition will create a new SSH keypair on the host in your `~/.ssh` directory if you do not specify a SSH public key. If you wish to use an existing SSH key already loaded into AWS, set `public_key_id` to the key's label. If you wish to use an existing SSH key, but need to have it loaded into AWS, then set `public_key_file` to the key's path. + +## Execute + +Then set up the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run main.yml +``` + +## Tear Down + +Tear down the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run teardown.yml +``` + +## FAQ + +### Using tags + +If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. + +### Using _extra variables_ + +Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. + +### Using Ansible collection and role paths + +Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. diff --git a/public-cloud/aws/base/ansible-navigator.yml b/public-cloud/aws/base/ansible-navigator.yml new file mode 100644 index 0000000..a0dc3ef --- /dev/null +++ b/public-cloud/aws/base/ansible-navigator.yml @@ -0,0 +1,65 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + ansible: + inventory: + entries: + - inventory.ini + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - AWS_PROFILE + - CDP_PROFILE + set: + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + image: ghcr.io/cloudera-labs/cldr-runner:aws-devel + pull: + arguments: + - "--tls-verify=false" + volume-mounts: + - src: "${ANSIBLE_COLLECTIONS_PATH}" + dest: "${ANSIBLE_COLLECTIONS_PATH}" + options: "Z" + - src: "~/.aws" + dest: "/runner/.aws" + options: "Z" + - src: "~/.cdp" + dest: "/runner/.cdp" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/public-cloud/aws/base/definition.yml b/public-cloud/aws/base/definition.yml new file mode 100644 index 0000000..6b06a97 --- /dev/null +++ b/public-cloud/aws/base/definition.yml @@ -0,0 +1,32 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################ +# Global variables +################################################################################ +name_prefix: basex # name prefix for everything +infra_region: us-east-2 # CSP region for infra +infra_type: aws # CSP + +# globals: +# force_teardown: yes + +################################################################################ +# CDP Environment and Datalake variables +################################################################################ +env: + public_endpoint_access: yes + tunnel: yes diff --git a/public-cloud/aws/base/inventory.ini b/public-cloud/aws/base/inventory.ini new file mode 100644 index 0000000..fef0f4a --- /dev/null +++ b/public-cloud/aws/base/inventory.ini @@ -0,0 +1,15 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" \ No newline at end of file diff --git a/public-cloud/aws/base/main.yml b/public-cloud/aws/base/main.yml new file mode 100644 index 0000000..b717158 --- /dev/null +++ b/public-cloud/aws/base/main.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Set up CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_setup.yml + +- name: Set up CDP Public Cloud (Env and DL example) + ansible.builtin.import_playbook: cloudera.exe.pbc_setup.yml diff --git a/public-cloud/aws/base/teardown.yml b/public-cloud/aws/base/teardown.yml new file mode 100644 index 0000000..64debea --- /dev/null +++ b/public-cloud/aws/base/teardown.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Tear down CDP Public Cloud (Env and DL example) + ansible.builtin.import_playbook: cloudera.exe.pbc_teardown.yml + +- name: Tear down CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_teardown.yml From 6c3a18e6239bc9fe70b0cb7bf93c3cd1961d2e83 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Tue, 8 Aug 2023 16:24:19 -0400 Subject: [PATCH 04/13] Remove obsolete sandbox definitions Signed-off-by: Webster Mudge --- examples/sandbox-aws/application.yml | 25 --- examples/sandbox-aws/definition.yml | 196 ---------------------- examples/sandbox/application.yml | 25 --- examples/sandbox/definition.yml | 23 --- examples/sandbox/inventory_static.example | 41 ----- examples/sandbox/inventory_template.ini | 44 ----- 6 files changed, 354 deletions(-) delete mode 100644 examples/sandbox-aws/application.yml delete mode 100644 examples/sandbox-aws/definition.yml delete mode 100644 examples/sandbox/application.yml delete mode 100644 examples/sandbox/definition.yml delete mode 100644 examples/sandbox/inventory_static.example delete mode 100644 examples/sandbox/inventory_template.ini diff --git a/examples/sandbox-aws/application.yml b/examples/sandbox-aws/application.yml deleted file mode 100644 index cc80908..0000000 --- a/examples/sandbox-aws/application.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Coda - hosts: localhost - connection: local - gather_facts: no - become: no - tasks: - - name: Deployment results - debug: - msg: Success! \ No newline at end of file diff --git a/examples/sandbox-aws/definition.yml b/examples/sandbox-aws/definition.yml deleted file mode 100644 index b5e6fbe..0000000 --- a/examples/sandbox-aws/definition.yml +++ /dev/null @@ -1,196 +0,0 @@ ---- -############################################################################### -#Global variables - these can also be set in your profile -#These will take precedence over those set in your profile -############################################################################### -name_prefix: chpe #name prefix for everything, infra, plat, and run -infra_region: us-east-1 #CSP region for infra -infra_type: aws #CSP -cdp_profile: default #CDP Profile to use -aws_profile: default #AWS Profile to use -################################################################################ -#Infra Level Configurations -################################################################################ -infra: - teardown: - delete_data: yes #Whether to delete the cloud storage (s3) for the env - delete_ssh_key: no #Whether to delete the env ssh key - vpc: - #An additional CIDR to add to the env IP - your personal IP is automatic - extra_cidr: "74.217.76.0/24" -################################################################################ -#Platform/Environment Level Configurations -################################################################################ -env: - #Override the environment name - the default is --env - name: "chpe-aw-env" - datalake: - #Override the datalake name - the default is --dl - name: "chpe-aw-dl" - teardown: - delete_admin_group: yes #Whether to delete the admin group during teardown - delete_user_group: yes #Whether to delete the user group during teardown - delete_cross_account: yes #Whether to delete cross acct role during teardown - delete_credential: yes #Whether to delete the CDP credential during teardown - cdp: - #Admin Group Configuration - Name/Roles/Resource Roles - admin_group: - name: "chpe_demo-aws-prim" - roles: - - PowerUser - #User Group Configuration - Name/Roles/Resource Roles - user_group: - name: "chpe_demos_workers_ww" - roles: - - PowerUser - resource_roles: - - EnvironmentUser - - DEUser - - DWUser - - MLUser - workload_analytics: yes #Where to enable WXM during Environment creation -################################################################################ -#Datahub Configurations -################################################################################ -datahub: - #List of Datahub Definitions to build - definitions: - #Example of Datahub Creation by Cluster Definition - use CDP defaults for - #instance groups - - definition: "Data Engineering for AWS" - #Override the name of the datahub - default is -dhub - name: "def-only-dh" - #Example of Datahub Creation by including a bespoke jinja template - #Example can be found in cloudera.exe/roles/runtime/templates/datahub_streams_messaging_light.j2 - - include: "datahub_streams_messaging_light.j2" - #Example of Datahub Creation by instance group specification - - template: Flow Management Light Duty - #Provide a custom suffix on the default naming convention - default suffix is dhub - suffix: "in-depth" - instance_groups: - - instanceGroupName: management - - instanceGroupName: nifi_scaling - instanceGroupType: CORE - #Instance Group Name - - instanceGroupName: nifi - #Instance Group Node Count - nodeCount: 3 - #Instance Group Type - CORE/GATEWAY - instanceGroupType: CORE - #EBS Volume Configuration - attachedVolumeConfiguration: - #EBS Volume size per node - - volumeSize: 500 - #EBS Volume Count per node - volumeCount: 4 - #EBS Volume Type - st1/gp2/standard/ephemeral - #ephemeral = ephemeral disk - not EBS - #standard = magnetic - #st1 = througput optimized HDD - #gp2 = general purpose ssd - volumeType: st1 -################################################################################ -#Operational Database Configurations -################################################################################ -opdb: - definitions: - - name: chpe-od #Override OpDB name - default is -od - - name: chpe-od-2 #Override OpDB name - default is -od -################################################################################ -#Machine Learning Configurations -################################################################################ -ml: - #List of ML Workspace Definitions to provision - definitions: - #Workspace Number 1 - - tls: yes #Enable TLS? - #Enable Model Monitoring - monitoring: yes - #Enable a public-facing load balancer? Applicable for L0/L1 networks - public_loadbalancer: yes - #Override the default workspace name - default is -wksp - name: chpe-wksp - #Whether to remove/delete workspace storage (EFS) upon termination - storage: yes - #CPU and GPU Instance Group Definitions - instance_groups: - #CPU Instance Group - - name: cpu_settings - #CPU Instance Type - instanceType: "m5.4xlarge" - #Default Instance Count - instanceCount: 0 - #Root Volume Size for Each Instance - rootVolume: - size: 100 - #Autoscaling Properties (min/max/enabled) - autoscaling: - minInstances: 0 - maxInstances: 10 - enabled: true - #GPU Instance Group - - name: gpu_settings - #Default Instance Count - instanceCount: 0 - #GPU Instance Type - instanceType: "p2.8xlarge" - #Root Volume Size for Each Instance - rootVolume: - size: 100 - #Autoscaling Properties (min/max/enabled) - autoscaling: - minInstances: 0 - maxInstances: 2 - enabled: true - #List of IP CIDRs to whitelist for public load balancers AND k8s management API - ip_addresses: - - "108.5.26.129/32" - - "1.2.3.4/32" - #Workspace Number 2 - - tls: yes - monitoring: yes - public_loadbalancer: yes - #Override the default workspace name - default is -wksp - name: "my-cml-space" - storage: yes - instance_groups: - - name: cpu_settings - instanceType: "m5.4xlarge" - instanceCount: 1 - rootVolume: - size: 100 - autoscaling: - minInstances: 1 - maxInstances: 5 - enabled: true - - name: gpu_settings - instanceCount: 0 - instanceType: "p2.8xlarge" - rootVolume: - size: 100 - autoscaling: - minInstances: 0 - maxInstances: 2 - enabled: true - ip_addresses: - - "108.5.26.129/32" - - "1.2.3.4/32" -################################################################################ -#Data Warehouse Configurations -################################################################################ -dw: #No current configs can be set here (yet) -################################################################################ -#Data Flow Configurations -################################################################################ -df: - min_k8s_nodes: 3 #Minumum number of k8s nodes for autoscaling - max_k8s_nodes: 5 #Minumum number of k8s nodes for autoscaling - #Enable a public-facing load balancer? Applicable for L0/L1 networks - public_loadbalancer: true - #List of IP CIDRs to whitelist for public load balancers AND k8s management API - ip_ranges: - - "108.5.26.129/32" - - "1.2.3.4/32" - #Whether to remove/delete experience storage upon termination - teardown: - persist: false \ No newline at end of file diff --git a/examples/sandbox/application.yml b/examples/sandbox/application.yml deleted file mode 100644 index cc80908..0000000 --- a/examples/sandbox/application.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Coda - hosts: localhost - connection: local - gather_facts: no - become: no - tasks: - - name: Deployment results - debug: - msg: Success! \ No newline at end of file diff --git a/examples/sandbox/definition.yml b/examples/sandbox/definition.yml deleted file mode 100644 index ebc392e..0000000 --- a/examples/sandbox/definition.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -datahub: - definitions: - - include: "datahub_streams_messaging_light.j2" - -use_default_cluster_definition: yes -use_auto_repo_mirror: yes -preload_cm_parcel_repo: yes \ No newline at end of file diff --git a/examples/sandbox/inventory_static.example b/examples/sandbox/inventory_static.example deleted file mode 100644 index 5a0c788..0000000 --- a/examples/sandbox/inventory_static.example +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[cloudera_manager] -ip-10-10-11-191.eu-west-1.compute.internal ansible_host=52.211.97.38 ansible_user=centos ansible_ssh_private_key_file=~/.ssh/mykey.pem - -[cluster_master_nodes] -ip-10-10-11-191.eu-west-1.compute.internal ansible_host=52.211.97.38 ansible_user=centos ansible_ssh_private_key_file=~/.ssh/mykey.pem host_template=Master1 - -[cluster_worker_nodes] -ip-10-10-2-62.eu-west-1.compute.internal ansible_host=63.35.213.171 ansible_user=centos ansible_ssh_private_key_file=~/.ssh/mykey.pem -ip-10-10-15-242.eu-west-1.compute.internal ansible_host=34.245.231.13 ansible_user=centos ansible_ssh_private_key_file=~/.ssh/mykey.pem -ip-10-10-12-158.eu-west-1.compute.internal ansible_host=3.249.133.103 ansible_user=centos ansible_ssh_private_key_file=~/.ssh/mykey.pem - -[cluster_worker_nodes:vars] -host_template=Workers - -[cluster:children] -cluster_master_nodes -cluster_worker_nodes - -[db_server] -ip-10-10-11-191.eu-west-1.compute.internal ansible_host=52.211.97.38 ansible_user=centos ansible_ssh_private_key_file=~/.ssh/mykey.pem - -[custom_repo] -ip-10-10-11-191.eu-west-1.compute.internal ansible_host=52.211.97.38 ansible_user=centos ansible_ssh_private_key_file=~/.ssh/mykey.pem - -[deployment:children] -cluster -cloudera_manager diff --git a/examples/sandbox/inventory_template.ini b/examples/sandbox/inventory_template.ini deleted file mode 100644 index 2ce3d9f..0000000 --- a/examples/sandbox/inventory_template.ini +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[cloudera_manager] -host-1.example.com - -[cluster_master_nodes] -host-1.example.com host_template=Master1 - -[cluster_worker_nodes] -host-2.example.com -host-3.example.com -host-4.example.com - -[cluster_worker_nodes:vars] -host_template=Workers - -[cluster:children] -cluster_master_nodes -cluster_worker_nodes - -[db_server] -host-1.example.com - -[custom_repo] -host-1.example.com - -[deployment:children] -cluster -cloudera_manager - -[all:vars] -ansible_python_interpreter=auto \ No newline at end of file From cf6e8e6ae43278104d123139f269e6acde70b3b8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 9 Aug 2023 17:16:35 -0400 Subject: [PATCH 05/13] Convert CDE example to ansible-navigator and collection playbooks Signed-off-by: Webster Mudge --- examples/cde/definition.yml | 41 ---------- .../aws/cde/.gitignore | 16 +--- public-cloud/aws/cde/README.md | 76 +++++++++++++++++++ public-cloud/aws/cde/ansible-navigator.yml | 65 ++++++++++++++++ public-cloud/aws/cde/definition.yml | 76 +++++++++++++++++++ public-cloud/aws/cde/inventory.ini | 15 ++++ public-cloud/aws/cde/main.yml | 34 +++++++++ public-cloud/aws/cde/teardown.yml | 34 +++++++++ 8 files changed, 304 insertions(+), 53 deletions(-) delete mode 100644 examples/cde/definition.yml rename examples/cde/application.yml => public-cloud/aws/cde/.gitignore (71%) create mode 100644 public-cloud/aws/cde/README.md create mode 100644 public-cloud/aws/cde/ansible-navigator.yml create mode 100644 public-cloud/aws/cde/definition.yml create mode 100644 public-cloud/aws/cde/inventory.ini create mode 100644 public-cloud/aws/cde/main.yml create mode 100644 public-cloud/aws/cde/teardown.yml diff --git a/examples/cde/definition.yml b/examples/cde/definition.yml deleted file mode 100644 index dfc1505..0000000 --- a/examples/cde/definition.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -de: - definitions: - - name: cde-cloudera-deploy-example - instance_type: 'm5.2xlarge' - minimum_instances: 1 - maximum_instances: 4 - minimum_spot_instances: 0 - maximum_spot_instances: 0 - enable_public_endpoint: yes - enable_workload_analytics: yes - initial_instances: 1 - initial_spot_instances: 0 - root_volume_size: 100 - chart_value_overrides: [{"chartName":"dex-app", "overrides":"dexapp.api.gangScheduling.enabled:true"}] - skip_validation: yes - tags: {'cde-cloudera-deploy-example':'v0.0.1'} - use_ssd: yes - virtual_clusters: - - name: cloudera-deployed-vc-1 - cpu_requests: 32 - memory_requests: '128Gi' - spark_version: 'SPARK2' - acl_users: '*' - runtime_spot_component: 'NONE' - chart_value_overrides: [{"chartName":"dex-app", "overrides":"pipelines.enabled:true"}] diff --git a/examples/cde/application.yml b/public-cloud/aws/cde/.gitignore similarity index 71% rename from examples/cde/application.yml rename to public-cloud/aws/cde/.gitignore index 9d77e34..dbf25ad 100644 --- a/examples/cde/application.yml +++ b/public-cloud/aws/cde/.gitignore @@ -1,6 +1,4 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,12 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Coda - hosts: localhost - connection: local - gather_facts: no - become: no - tasks: - - name: Deployment results - debug: - msg: Success! +ansible-navigator.log +runs +context diff --git a/public-cloud/aws/cde/README.md b/public-cloud/aws/cde/README.md new file mode 100644 index 0000000..958e8cd --- /dev/null +++ b/public-cloud/aws/cde/README.md @@ -0,0 +1,76 @@ +# CDP Public Cloud - Cloudera Data Engineering (CDE) + +> Constructs a set of Cloudera Data Engineering (CDE) workspaces within their own CDP Public Cloud Environment and Datalake. Generates via Ansible the AWS infrastructure and CDP artifacts, including SSH key, cross-account credentials, S3 buckets, etc. + +## Requirements + +To run, you need: + +* Docker (or a Docker clone[^1]) +* AWS credentials (set via `AWS_PROFILE`) +* CDP credentials (set via `CDP_PROFILE`) + +[^1]: [OrbStack](https://orbstack.dev) works well on OSX. + +## Set Up + +First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). + +Then, clone this project and change your working directory. + +```bash +git clone https://github.com/cloudera-labs/cloudera-deploy.git; cd cloudera-deploy/public-cloud/aws/cde +``` + +## Configure + +Set the required environment variables: + +```bash +export AWS_PROFILE=your-aws-profile +export CDP_PROFILE=your-cdp-profile +``` + +Tweak the `definition.yml` parameters to your liking. Notably, you should add and/or change: + +```yaml +name_prefix: ex01 # Keep this short (4-7 characters) +admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. +infra_region: us-east-2 +``` + +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. + +### SSH Keys + +This definition will create a new SSH keypair on the host in your `~/.ssh` directory if you do not specify a SSH public key. If you wish to use an existing SSH key already loaded into AWS, set `public_key_id` to the key's label. If you wish to use an existing SSH key, but need to have it loaded into AWS, then set `public_key_file` to the key's path. + +## Execute + +Then set up the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run main.yml +``` + +## Tear Down + +Tear down the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run teardown.yml +``` + +## FAQ + +### Using tags + +If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. + +### Using _extra variables_ + +Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. + +### Using Ansible collection and role paths + +Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. diff --git a/public-cloud/aws/cde/ansible-navigator.yml b/public-cloud/aws/cde/ansible-navigator.yml new file mode 100644 index 0000000..81bb177 --- /dev/null +++ b/public-cloud/aws/cde/ansible-navigator.yml @@ -0,0 +1,65 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + ansible: + inventory: + entries: + - inventory.ini + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - AWS_PROFILE + - CDP_PROFILE + set: + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + image: ghcr.io/cloudera-labs/cldr-runner:aws-devel02 + pull: + arguments: + - "--tls-verify=false" + volume-mounts: + - src: "${ANSIBLE_COLLECTIONS_PATH}" + dest: "${ANSIBLE_COLLECTIONS_PATH}" + options: "Z" + - src: "~/.aws" + dest: "/runner/.aws" + options: "Z" + - src: "~/.cdp" + dest: "/runner/.cdp" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/public-cloud/aws/cde/definition.yml b/public-cloud/aws/cde/definition.yml new file mode 100644 index 0000000..fecb95b --- /dev/null +++ b/public-cloud/aws/cde/definition.yml @@ -0,0 +1,76 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################ +# Global variables +################################################################################ +# Either define here or override using _extra variables_ in the CLI or AWX. +# For example, '-e name_prefix=cdeex' +# name_prefix: # You must specify a name prefix +# admin_password: # You must specify an admin password +infra_region: us-east-2 # CSP region for infra +infra_type: aws # CSP + +# Limit to the caller/controller +allowed_cidrs: "{{ lookup('ansible.builtin.url', 'https://api.ipify.org', wantlist=True) | product(['32']) | map('join', '/') | list }}" + +################################################################################ +# CDP Environment and Datalake variables +################################################################################ +env: + public_endpoint_access: yes + tunnel: yes + +infra: + vpc: + extra_cidr: "{{ allowed_cidrs }}" + extra_ports: [22, 443] + +################################################################################ +# Data Engineering Configurations +################################################################################ +de: + definitions: + - name: "{{ name_prefix }}-aws-example" + instance_type: 'm5.2xlarge' + minimum_instances: 1 + maximum_instances: 4 + minimum_spot_instances: 0 + maximum_spot_instances: 0 + enable_public_endpoint: yes + loadbalancer_ips: "{{ allowed_cidrs }}" + whitelist_ips: "{{ allowed_cidrs }}" + enable_workload_analytics: yes + initial_instances: 1 + initial_spot_instances: 0 + root_volume_size: 100 + chart_value_overrides: + - chartName: "dex-app" + overrides: "dexapp.api.gangScheduling.enabled:true" + skip_validation: yes + tags: + cloudera-deploy-example : 'v2.0.0' + use_ssd: yes + virtual_clusters: + - name: "{{ name_prefix }}-aws-vc-spark2" + cpu_requests: 32 + memory_requests: '128Gi' + spark_version: 'SPARK2' + acl_users: '*' + runtime_spot_component: 'NONE' + chart_value_overrides: + - chartName: "dex-app" + overrides: "pipelines.enabled:true" diff --git a/public-cloud/aws/cde/inventory.ini b/public-cloud/aws/cde/inventory.ini new file mode 100644 index 0000000..fef0f4a --- /dev/null +++ b/public-cloud/aws/cde/inventory.ini @@ -0,0 +1,15 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" \ No newline at end of file diff --git a/public-cloud/aws/cde/main.yml b/public-cloud/aws/cde/main.yml new file mode 100644 index 0000000..f07c5b7 --- /dev/null +++ b/public-cloud/aws/cde/main.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Set up CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_setup.yml + +- name: Set up CDP Public Cloud (CDE example) + ansible.builtin.import_playbook: cloudera.exe.pbc_setup.yml diff --git a/public-cloud/aws/cde/teardown.yml b/public-cloud/aws/cde/teardown.yml new file mode 100644 index 0000000..d04d851 --- /dev/null +++ b/public-cloud/aws/cde/teardown.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Tear down CDP Public Cloud (CDE example) + ansible.builtin.import_playbook: cloudera.exe.pbc_teardown.yml + +- name: Tear down CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_teardown.yml From e299b9a054c0a8ee56a88d93d6b5da449e74653a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 9 Aug 2023 17:17:27 -0400 Subject: [PATCH 06/13] Update base CDP Env and DL example for allowed CIDRs Signed-off-by: Webster Mudge --- public-cloud/aws/base/definition.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/public-cloud/aws/base/definition.yml b/public-cloud/aws/base/definition.yml index 6b06a97..cdc301c 100644 --- a/public-cloud/aws/base/definition.yml +++ b/public-cloud/aws/base/definition.yml @@ -17,12 +17,15 @@ ################################################################################ # Global variables ################################################################################ -name_prefix: basex # name prefix for everything -infra_region: us-east-2 # CSP region for infra -infra_type: aws # CSP +# Either define here or override using _extra variables_ in the CLI or AWX. +# For example, '-e name_prefix=basex' +# name_prefix: # You must specify a name prefix +# admin_password: # You must specify an admin password +infra_region: us-east-2 # CSP region for infra +infra_type: aws # CSP -# globals: -# force_teardown: yes +# Limit to the caller/controller +allowed_cidrs: "{{ lookup('ansible.builtin.url', 'https://api.ipify.org', wantlist=True) | product(['32']) | map('join', '/') | list }}" ################################################################################ # CDP Environment and Datalake variables @@ -30,3 +33,8 @@ infra_type: aws # CSP env: public_endpoint_access: yes tunnel: yes + +infra: + vpc: + extra_cidr: "{{ allowed_cidrs }}" + extra_ports: [22, 443] From e651d499f160213808e89492aeb2f861a93da9da Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 9 Aug 2023 18:05:26 -0400 Subject: [PATCH 07/13] Convert CDF example to ansible-navigator and collection playbooks Signed-off-by: Webster Mudge --- examples/cdf/definition.yml | 49 ----------- public-cloud/aws/cdf/.gitignore | 17 ++++ public-cloud/aws/cdf/README.md | 82 +++++++++++++++++++ public-cloud/aws/cdf/ansible-navigator.yml | 65 +++++++++++++++ .../aws}/cdf/application.yml | 64 +++++++-------- public-cloud/aws/cdf/definition.yml | 65 +++++++++++++++ public-cloud/aws/cdf/inventory.ini | 15 ++++ public-cloud/aws/cdf/main.yml | 34 ++++++++ public-cloud/aws/cdf/teardown.yml | 34 ++++++++ 9 files changed, 344 insertions(+), 81 deletions(-) delete mode 100644 examples/cdf/definition.yml create mode 100644 public-cloud/aws/cdf/.gitignore create mode 100644 public-cloud/aws/cdf/README.md create mode 100644 public-cloud/aws/cdf/ansible-navigator.yml rename {examples => public-cloud/aws}/cdf/application.yml (53%) create mode 100644 public-cloud/aws/cdf/definition.yml create mode 100644 public-cloud/aws/cdf/inventory.ini create mode 100644 public-cloud/aws/cdf/main.yml create mode 100644 public-cloud/aws/cdf/teardown.yml diff --git a/examples/cdf/definition.yml b/examples/cdf/definition.yml deleted file mode 100644 index e0828c5..0000000 --- a/examples/cdf/definition.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Commented values are shown with the defaults, uncomment and change to override - -df: - readyflows: - - flow_name: "Kafka to Kafka" -# customflows: -# - flow_name: test -# file: '/runner/project/cloudera-deploy/000-abc1-version-1.json' -# description: test -# comments: test -# min_k8s_nodes: 3 -# max_k8s_nodes: 5 -# public_loadbalancer: True -# loadbalancer_ip_ranges: [] -# kube_ip_ranges: [] -# cluster_subnets: [] -# loadbalancer_subnets: [] -# terminate_deployments: True # Whether to terminate Flow deployments before service teardown -# teardown: -# persist: False # Whether to keep backend database records for the service after teardown - -env: - public_endpoint_access: yes - tunnel: yes - -datahub: - definitions: - - definition: "Flow Management Light Duty" - suffix: nifi-dhub - - definition: "Streams Messaging Light Duty" - suffix: kafka-dhub - - definition: "Streaming Analytics Light Duty" - suffix: flink-dhub \ No newline at end of file diff --git a/public-cloud/aws/cdf/.gitignore b/public-cloud/aws/cdf/.gitignore new file mode 100644 index 0000000..dbf25ad --- /dev/null +++ b/public-cloud/aws/cdf/.gitignore @@ -0,0 +1,17 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator.log +runs +context diff --git a/public-cloud/aws/cdf/README.md b/public-cloud/aws/cdf/README.md new file mode 100644 index 0000000..8a97081 --- /dev/null +++ b/public-cloud/aws/cdf/README.md @@ -0,0 +1,82 @@ +# CDP Public Cloud - Cloudera Data Flow (CDF) + +> Constructs a set of Cloudera Data Flow (CDF) workspaces and data hubs within their own CDP Public Cloud Environment and Datalake. Generates via Ansible the AWS infrastructure and CDP artifacts, including SSH key, cross-account credentials, S3 buckets, etc. + +## Requirements + +To run, you need: + +* Docker (or a Docker clone[^1]) +* AWS credentials (set via `AWS_PROFILE`) +* CDP credentials (set via `CDP_PROFILE`) + +[^1]: [OrbStack](https://orbstack.dev) works well on OSX. + +## Set Up + +First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). + +Then, clone this project and change your working directory. + +```bash +git clone https://github.com/cloudera-labs/cloudera-deploy.git; cd cloudera-deploy/public-cloud/aws/cdf +``` + +## Configure + +Set the required environment variables: + +```bash +export AWS_PROFILE=your-aws-profile +export CDP_PROFILE=your-cdp-profile +``` + +Tweak the `definition.yml` parameters to your liking. Notably, you should add and/or change: + +```yaml +name_prefix: ex01 # Keep this short (4-7 characters) +admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. +infra_region: us-east-2 +``` + +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. + +### SSH Keys + +This definition will create a new SSH keypair on the host in your `~/.ssh` directory if you do not specify a SSH public key. If you wish to use an existing SSH key already loaded into AWS, set `public_key_id` to the key's label. If you wish to use an existing SSH key, but need to have it loaded into AWS, then set `public_key_file` to the key's path. + +## Execute + +Then set up the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run main.yml +``` + +To deploy an example CDF ReadyFlow, run next the playbook: + +```bash +ansible-navigator run application.yml +``` + +## Tear Down + +Tear down the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run teardown.yml +``` + +## FAQ + +### Using tags + +If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. + +### Using _extra variables_ + +Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. + +### Using Ansible collection and role paths + +Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. diff --git a/public-cloud/aws/cdf/ansible-navigator.yml b/public-cloud/aws/cdf/ansible-navigator.yml new file mode 100644 index 0000000..a0dc3ef --- /dev/null +++ b/public-cloud/aws/cdf/ansible-navigator.yml @@ -0,0 +1,65 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + ansible: + inventory: + entries: + - inventory.ini + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - AWS_PROFILE + - CDP_PROFILE + set: + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + image: ghcr.io/cloudera-labs/cldr-runner:aws-devel + pull: + arguments: + - "--tls-verify=false" + volume-mounts: + - src: "${ANSIBLE_COLLECTIONS_PATH}" + dest: "${ANSIBLE_COLLECTIONS_PATH}" + options: "Z" + - src: "~/.aws" + dest: "/runner/.aws" + options: "Z" + - src: "~/.cdp" + dest: "/runner/.cdp" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/examples/cdf/application.yml b/public-cloud/aws/cdf/application.yml similarity index 53% rename from examples/cdf/application.yml rename to public-cloud/aws/cdf/application.yml index 0d2a342..ab7b626 100644 --- a/examples/cdf/application.yml +++ b/public-cloud/aws/cdf/application.yml @@ -1,6 +1,6 @@ --- -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,49 +14,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Deploy example DFX Flow +- name: Deploy example CDF ReadyFlow (Kafka-to-Kafka) hosts: localhost connection: local gather_facts: no - become: no - tags: always + vars: + # Either define here or override using _extra variables_ in the CLI or AWX. + # For example, '-e name_prefix=cdfex' + # name_prefix: # You must specify a name prefix + # admin_password: # You must specify an admin password + env_name: "{{ name_prefix }}-aw-env" + smm_datahub: "{{ name_prefix }}-aw-kafka-dhub" tasks: - - name: Construct the deployment details - when: - - deployment is undefined - - "'run' in ansible_run_tags" - - "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera.exe.info + - name: Verify input parameters for Kafka-to-Kafka ReadyFlow deployment + ansible.builtin.assert: + that: + - name_prefix is defined + - admin_password is defined + fail_msg: "You must provide both 'name_prefix' and 'admin_password'" - name: Get CDP workload username for current user - when: - - "'teardown' not in ansible_run_tags" - - "'run' in ansible_run_tags" cloudera.cloud.iam_user_info: current_user: yes register: __iam - - name: Extract Kafka Brokers from Info - when: - - "'teardown' not in ansible_run_tags" - - "'run' in ansible_run_tags" - - "{{ deployment.datahubs | selectattr('workloadType', 'search', 'Streams Messaging') | length > 0 }}" + - name: Retrieve the Kafka brokers from the Data Hub ansible.builtin.set_fact: - kafka_brokers_fqdns: "{{ deployment.datahubs | selectattr('workloadType', 'search', 'Streams Messaging') | first | json_query(_jq) | first }}" - vars: - _jq: "instanceGroups[?name=='core_broker'].instances[*].fqdn" + kafka_brokers_fqdns: "{{ lookup('cloudera.cloud.datahub_instance', 'core_broker', datahub=smm_datahub) }}" + failed_when: kafka_brokers_fqdns | length < 1 + + - name: Ensure Kafka-to-Kafka ReadyFlow is imported + cloudera.cloud.df_readyflow: + name: "Kafka to Kafka" + register: __k2k + failed_when: not __k2k.readyflow - - name: Deploy example DFX Flow - Kafka to Kafka + - name: Deploy Kafka-to-Kafka ReadyFlow when: - - "'teardown' not in ansible_run_tags" - - "'run' in ansible_run_tags" - - kafka_brokers_fqdns is defined - kafka_brokers_fqdns | length > 0 cloudera.cloud.df_deployment: - name: "{{ globals.name_prefix }}_ex_kafkatokafka" - df_name: "{{ deployment.environment.environmentName }}" + name: "{{ name_prefix }}_kafka-to-kafka" + df_name: "{{ env_name }}" flow_name: "Kafka to Kafka" + flow_ver_crn: "{{ __k2k.readyflow.versions | sort(attribute='version') | map(attribute='crn') | first }}" autoscale: yes autostart_flow: yes parameter_groups: "{{ parameter_groups }}" @@ -66,11 +66,11 @@ - name: "kafka-to-kafka" parameters: - name: "CDP Workload User" - value: "{{ __iam.users[0].workloadUsername }}" + value: "{{ __iam.users | map(attribute='workloadUsername') | first }}" - name: "CDP Workload User Password" - value: "{{ globals.admin_password }}" + value: "{{ admin_password }}" - name: "Kafka Broker Endpoint" - value: "{{ kafka_brokers_fqdns | product([':9093']) | map('join') | list | join(',') }}" + value: "{{ kafka_brokers_fqdns | product(['9093']) | map('join', ':') | list | join(',') }}" - name: "Kafka Consumer Group ID" value: "ex_cdf_consg0" - name: "Kafka Destination Topic" @@ -88,4 +88,4 @@ frequencyTolerance: unit: id: MINUTES - value: 5 \ No newline at end of file + value: 5 diff --git a/public-cloud/aws/cdf/definition.yml b/public-cloud/aws/cdf/definition.yml new file mode 100644 index 0000000..fc23889 --- /dev/null +++ b/public-cloud/aws/cdf/definition.yml @@ -0,0 +1,65 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################ +# Global variables +# +################################################################################ +# Either define here or override using _extra variables_ in the CLI or AWX. +# For example, '-e name_prefix=cdfex' +# name_prefix: # You must specify a name prefix +# admin_password: # You must specify an admin password +infra_region: us-east-2 # CSP region for infra +infra_type: aws # CSP + +# Limit to the caller/controller +allowed_cidrs: "{{ lookup('ansible.builtin.url', 'https://api.ipify.org', wantlist=True) | product(['32']) | map('join', '/') | list }}" + +################################################################################ +# CDP Environment and Datalake variables +################################################################################ +env: + public_endpoint_access: yes + tunnel: yes + +infra: + vpc: + extra_cidr: "{{ allowed_cidrs }}" + extra_ports: [22, 443] + +################################################################################ +# Data Flow Configurations +################################################################################ +df: + readyflows: + - flow_name: "Kafka to Kafka" + public_loadbalancer: yes + loadbalancer_ip_ranges: "{{ allowed_cidrs }}" + k8s_ip_ranges: "{{ allowed_cidrs }}" + cluster_subnets_filter: "[?contains(subnetName,`pvt`)]" + loadbalancer_subnets_filter: "[?contains(subnetName,`pub`)]" + +datahub: + definitions: + - definition: "- Flow Management Light Duty" + suffix: nifi-dhub + multi_az: no + - definition: "Streams Messaging Light Duty" + suffix: kafka-dhub + multi_az: no + - definition: "Streaming Analytics Light Duty" + suffix: flink-dhub + multi_az: no diff --git a/public-cloud/aws/cdf/inventory.ini b/public-cloud/aws/cdf/inventory.ini new file mode 100644 index 0000000..fef0f4a --- /dev/null +++ b/public-cloud/aws/cdf/inventory.ini @@ -0,0 +1,15 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" \ No newline at end of file diff --git a/public-cloud/aws/cdf/main.yml b/public-cloud/aws/cdf/main.yml new file mode 100644 index 0000000..5cc5298 --- /dev/null +++ b/public-cloud/aws/cdf/main.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Set up CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_setup.yml + +- name: Set up CDP Public Cloud (CDF examples) + ansible.builtin.import_playbook: cloudera.exe.pbc_setup.yml diff --git a/public-cloud/aws/cdf/teardown.yml b/public-cloud/aws/cdf/teardown.yml new file mode 100644 index 0000000..fb7df50 --- /dev/null +++ b/public-cloud/aws/cdf/teardown.yml @@ -0,0 +1,34 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Tear down CDP Public Cloud (CDF examples) + ansible.builtin.import_playbook: cloudera.exe.pbc_teardown.yml + +- name: Tear down CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_teardown.yml From ac9f35e7870ff6ce3b347488952730e0bf99bc4e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 9 Aug 2023 18:07:47 -0400 Subject: [PATCH 08/13] Update to use allowed_cidr Signed-off-by: Webster Mudge --- public-cloud/aws/cml/definition.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/public-cloud/aws/cml/definition.yml b/public-cloud/aws/cml/definition.yml index eb95ae4..7212127 100644 --- a/public-cloud/aws/cml/definition.yml +++ b/public-cloud/aws/cml/definition.yml @@ -17,12 +17,15 @@ ################################################################################ # Global variables ################################################################################ -name_prefix: cmlex # name prefix for everything -infra_region: us-east-2 # CSP region for infra -infra_type: aws # CSP +# Either define here or override using _extra variables_ in the CLI or AWX. +# For example, '-e name_prefix=cmlex' +# name_prefix: # You must specify a name prefix +# admin_password: # You must specify an admin password +infra_region: us-east-2 # CSP region for infra +infra_type: aws # CSP -# globals: -# force_teardown: yes +# Limit to the caller/controller +allowed_cidrs: "{{ lookup('ansible.builtin.url', 'https://api.ipify.org', wantlist=True) | product(['32']) | map('join', '/') | list }}" ################################################################################ # CDP Environment and Datalake variables @@ -31,6 +34,11 @@ env: public_endpoint_access: yes tunnel: yes +infra: + vpc: + extra_cidr: "{{ allowed_cidrs }}" + extra_ports: [22, 443] + ################################################################################ # Machine Learning Configurations ################################################################################ @@ -41,7 +49,7 @@ ml: governance: yes metrics: yes public_loadbalancer: yes - ip_addresses: [ "0.0.0.0/0" ] # Unrestricted + ip_addresses: "{{ allowed_cidrs }}" instance_groups: - name: cpu_settings autoscaling: From f5dd9137f8a6eaa6df6dfb210bdfc7028fbea9b9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 9 Aug 2023 18:08:16 -0400 Subject: [PATCH 09/13] Add Frequently Asked Questions doc Signed-off-by: Webster Mudge --- FAQ.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 FAQ.md diff --git a/FAQ.md b/FAQ.md new file mode 100644 index 0000000..802c268 --- /dev/null +++ b/FAQ.md @@ -0,0 +1,55 @@ +# Frequently Asked Questions + +## Where did everything go? + +We undertook some serious remodeling, but rest assured, your definitions will still work as they did in the previous version of `cloudera-deploy`. + +So where did everything go? Well... + +1. The `quickstart.sh` migrated to `ansible-navigator`. Both applications use a container based on `ansible-runner`, i.e. [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner), yet `ansible-navigator` is configuration-driven and better aligned with how AWX runs Ansible in containers. Plus, `ansible-navigator` brings a nifty UI and the ease of use to handle different execution modes. (And we also migrated `cldr-runner` to use `ansible-builder`, but you can read more about that at the [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner) project.) +1. The original `cloudera-deploy` playbooks moved into `cloudera.exe`. Starting with Ansible `2.11`, [collections can contain playbooks](https://docs.ansible.com/ansible/latest/collections_guide/collections_using_playbooks.html#using-a-playbook-from-a-collection). We call them using `import_playbook` like roles. Using them requires you to first set up your `cloudera-deploy` variables _before_ calling the playbooks. +1. The _run-levels_ still remain, however, the playbooks are more granular. This move promotes composibility and reusability, and we are going to continue to break down the functions and operations within `cloudera-deploy` and the notably the collections that drive this application. We want you to adapt and create your own "deploy" application, one that caters to _your_ needs and operating parameters. Going more granular, more modular is key to this end. + +## `ansible-navigator` hangs when I run my playbook. What is going on? + +`ansible-navigator` does not handle user prompts when running in the `curses` UI, so actions in your playbook like: + +* Vault passwords +* SSH passphrases +* Debugger statements + +will not work out-of-the-box. You can enable `ansible-navigator` to run with prompts, but doing so will also disable the UI and instead run its operations using `stdout`. Try adding: + +```bash +ansible-navigator run --enable-prompts ... +``` + +to your execution. + +## How can I view a previous `ansible-navigator` run to debug an issue? + +Each example is configured to save execution runs in the project's `runs` directory. You can reload a run by using the `replay` command: + +```bash +ansible-navigator replay runs/-.json +``` + +Then you can use the UI to review the plays, tasks, and inventory for the previous run! + +## How can I enable the playbook debugger? + +The [playbook debugger](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_debugger.html) is enabled in `ansible-navigator` by setting the debugger and then enabling prompts. For example, + +```bash +ANSIBLE_ENABLE_TASK_DEBUGGER=True ansible-navigator run --enable-prompts main.yml +``` + +## How can I select just a single subnet using `subnet_filter`, say for a CDE definition? + +The various `filters`, like `subnet_filter`, `loadbalancer_subnets_filter`, etc., use [JMESPath](https://jmespath.org/) expressions against a list of subnet objects. Using expression like: + +```jmespath +[?contains(subnetName,`pvt`)] | [:1] +``` + +will limit the list of subnet objects to those with the term `pvt` and then select the first element of that reduced list. \ No newline at end of file From 5384b27305a98b0674346ac31305bba0adfad00f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 10 Aug 2023 10:42:18 -0400 Subject: [PATCH 10/13] Add JMESPath Playground example for subnet filters Signed-off-by: Webster Mudge --- FAQ.md | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/FAQ.md b/FAQ.md index 802c268..c1af632 100644 --- a/FAQ.md +++ b/FAQ.md @@ -52,4 +52,47 @@ The various `filters`, like `subnet_filter`, `loadbalancer_subnets_filter`, etc. [?contains(subnetName,`pvt`)] | [:1] ``` -will limit the list of subnet objects to those with the term `pvt` and then select the first element of that reduced list. \ No newline at end of file +will limit the list of subnet objects to those with the term `pvt` and then select the first element of that reduced list. + +You can [test sample filters](https://play.jmespath.org/?u=45e4d839-15f9-4569-9490-20a2cbc0cc88) using this example on the JMESPath Playground (link goes to a preloaded playground): + +```json +[ + { + "availabilityZone": "us-east-2c", + "cidr": "10.10.64.0/19", + "subnetId": "subnet-0123", + "subnetName": "sbnt-pub-2" + }, + { + "availabilityZone": "us-east-2a", + "cidr": "10.10.0.0/19", + "subnetId": "subnet-1234", + "subnetName": "sbnt-pub-0" + }, + { + "availabilityZone": "us-east-2c", + "cidr": "10.10.160.0/19", + "subnetId": "subnet-2345", + "subnetName": "sbnt-pvt-2" + }, + { + "availabilityZone": "us-east-2b", + "cidr": "10.10.128.0/19", + "subnetId": "subnet-3456", + "subnetName": "sbnt-pvt-1" + }, + { + "availabilityZone": "us-east-2b", + "cidr": "10.10.32.0/19", + "subnetId": "subnet-4567", + "subnetName": "sbnt-pub-1" + }, + { + "availabilityZone": "us-east-2a", + "cidr": "10.10.96.0/19", + "subnetId": "subnet-5678", + "subnetName": "sbnt-pvt-0" + } +] +``` From 2455fe6479832fb3d8b61d411fab867c2b9253b7 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 10 Aug 2023 12:02:31 -0400 Subject: [PATCH 11/13] Update documentation to use the main FAQ Signed-off-by: Webster Mudge --- FAQ.md | 21 +++++++++++++++++---- README.md | 5 +++-- public-cloud/aws/base/README.md | 20 +++----------------- public-cloud/aws/cde/README.md | 20 +++----------------- public-cloud/aws/cdf/README.md | 20 +++----------------- public-cloud/aws/cml/README.md | 20 +++----------------- 6 files changed, 32 insertions(+), 74 deletions(-) diff --git a/FAQ.md b/FAQ.md index c1af632..71d041c 100644 --- a/FAQ.md +++ b/FAQ.md @@ -2,13 +2,26 @@ ## Where did everything go? -We undertook some serious remodeling, but rest assured, your definitions will still work as they did in the previous version of `cloudera-deploy`. +The project undertook some serious remodeling, but rest assured, your definitions will still work as they did in the previous version of `cloudera-deploy`. So where did everything go? Well... -1. The `quickstart.sh` migrated to `ansible-navigator`. Both applications use a container based on `ansible-runner`, i.e. [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner), yet `ansible-navigator` is configuration-driven and better aligned with how AWX runs Ansible in containers. Plus, `ansible-navigator` brings a nifty UI and the ease of use to handle different execution modes. (And we also migrated `cldr-runner` to use `ansible-builder`, but you can read more about that at the [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner) project.) -1. The original `cloudera-deploy` playbooks moved into `cloudera.exe`. Starting with Ansible `2.11`, [collections can contain playbooks](https://docs.ansible.com/ansible/latest/collections_guide/collections_using_playbooks.html#using-a-playbook-from-a-collection). We call them using `import_playbook` like roles. Using them requires you to first set up your `cloudera-deploy` variables _before_ calling the playbooks. -1. The _run-levels_ still remain, however, the playbooks are more granular. This move promotes composibility and reusability, and we are going to continue to break down the functions and operations within `cloudera-deploy` and the notably the collections that drive this application. We want you to adapt and create your own "deploy" application, one that caters to _your_ needs and operating parameters. Going more granular, more modular is key to this end. +1. The `quickstart.sh` migrated to `ansible-navigator`. Both of these applications use a container based on `ansible-runner`, i.e. [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner), to execute the playbooks, yet `ansible-navigator` is configuration-driven and better aligned with how AWX runs Ansible in containers. + `ansible-navigator` brings a nifty UI and the ease of use to handle different execution modes. + We also migrated `cldr-runner` to use `ansible-builder`, but you can read more about that effort at the [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner) project. +1. The original `cloudera-deploy` playbooks moved into `cloudera.exe`. Starting with Ansible `2.11`, [collections can contain playbooks](https://docs.ansible.com/ansible/latest/collections_guide/collections_using_playbooks.html#using-a-playbook-from-a-collection). We call the playbooks using `import_playbook` like roles. PLEASE NOTE, if you are developing your own project playbooks, you must first set up your `cloudera-deploy` variables _before_ calling the playbooks by running the `cloudera.exe.init_deployment` role on `localhost`. +1. The _run-levels_ still remain; you can still use `-t infra` for example. However, the playbooks themselves are more granular and overall set up and tear down processes are now separate playbooks. + This change promotes composibility and reusability, and we are going to continue to break apart the functions and operations within `cloudera-deploy` and -- most importantly -- the collections that drive this application. We fully expect that you will want to adapt and create your own "deploy" application, one that caters to _your_ needs and operating parameters. Switching to a more granular, more modular approach is key to this objective. + +## How to I add _extra variables_ and tags to `ansible-navigator`? + +If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. + +Like tags, so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. + +## How do I tell `ansible-navigator` where to find collections and roles? + +By default, `cloudera-deploy` expects to use the collections, roles, and libraries within the _execution environment_ container, that is `cldr-runner`. Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and pass them to the running container. The underlying `ansible` application, like `ansible-playbook` will then pick up these environment variables and attempt to use them if set! This behavior is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. ## `ansible-navigator` hangs when I run my playbook. What is going on? diff --git a/README.md b/README.md index 7b80e8d..5d6f60c 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # Cloudera Deploy -## cdp-navigator +## Setting up `ansible-navigator` `cloudera-deploy` uses `ansible-navigator` to manage and execute the deployment definitions. Setting up `ansible-navigator` is straightforward; create and activate a new `virtualenv` and install the latest `ansible-core` and `ansible-navigator`. +You can name your virtual environment anything you want; by convention, we call it `cdp-navigator`. + ```bash python -m venv ~/cdp-navigator; source ~/cdp-navigator/bin/activate; pip install ansible-core ansible-navigator ``` - diff --git a/public-cloud/aws/base/README.md b/public-cloud/aws/base/README.md index faaf2e8..308acdd 100644 --- a/public-cloud/aws/base/README.md +++ b/public-cloud/aws/base/README.md @@ -10,11 +10,11 @@ To run, you need: * AWS credentials (set via `AWS_PROFILE`) * CDP credentials (set via `CDP_PROFILE`) -[^1]: [OrbStack](https://orbstack.dev) works well on OSX. +[^1]: For example, [OrbStack](https://orbstack.dev) works well on OSX. ## Set Up -First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). +First, set up your `ansible-navigator` aka `cdp-navigator` environment -- follow the instructions in the top-level [README](../../../README.md#setting-up-ansible-navigator). Then, clone this project and change your working directory. @@ -39,7 +39,7 @@ admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. infra_region: us-east-2 ``` -NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](../../../FAQ.md#how-to-i-add-extra-variables-and-tags-to-ansible-navigator) for details. ### SSH Keys @@ -60,17 +60,3 @@ Tear down the CDP Public Cloud by running the playbook: ```bash ansible-navigator run teardown.yml ``` - -## FAQ - -### Using tags - -If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. - -### Using _extra variables_ - -Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. - -### Using Ansible collection and role paths - -Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. diff --git a/public-cloud/aws/cde/README.md b/public-cloud/aws/cde/README.md index 958e8cd..af11202 100644 --- a/public-cloud/aws/cde/README.md +++ b/public-cloud/aws/cde/README.md @@ -10,11 +10,11 @@ To run, you need: * AWS credentials (set via `AWS_PROFILE`) * CDP credentials (set via `CDP_PROFILE`) -[^1]: [OrbStack](https://orbstack.dev) works well on OSX. +[^1]: For example, [OrbStack](https://orbstack.dev) works well on OSX. ## Set Up -First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). +First, set up your `ansible-navigator` aka `cdp-navigator` environment -- follow the instructions in the top-level [README](../../../README.md#setting-up-ansible-navigator). Then, clone this project and change your working directory. @@ -39,7 +39,7 @@ admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. infra_region: us-east-2 ``` -NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](../../../FAQ.md#how-to-i-add-extra-variables-and-tags-to-ansible-navigator) for details. ### SSH Keys @@ -60,17 +60,3 @@ Tear down the CDP Public Cloud by running the playbook: ```bash ansible-navigator run teardown.yml ``` - -## FAQ - -### Using tags - -If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. - -### Using _extra variables_ - -Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. - -### Using Ansible collection and role paths - -Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. diff --git a/public-cloud/aws/cdf/README.md b/public-cloud/aws/cdf/README.md index 8a97081..69b39bf 100644 --- a/public-cloud/aws/cdf/README.md +++ b/public-cloud/aws/cdf/README.md @@ -10,11 +10,11 @@ To run, you need: * AWS credentials (set via `AWS_PROFILE`) * CDP credentials (set via `CDP_PROFILE`) -[^1]: [OrbStack](https://orbstack.dev) works well on OSX. +[^1]: For example, [OrbStack](https://orbstack.dev) works well on OSX. ## Set Up -First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). +First, set up your `ansible-navigator` aka `cdp-navigator` environment -- follow the instructions in the top-level [README](../../../README.md#setting-up-ansible-navigator). Then, clone this project and change your working directory. @@ -39,7 +39,7 @@ admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. infra_region: us-east-2 ``` -NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](../../../FAQ.md#how-to-i-add-extra-variables-and-tags-to-ansible-navigator) for details. ### SSH Keys @@ -66,17 +66,3 @@ Tear down the CDP Public Cloud by running the playbook: ```bash ansible-navigator run teardown.yml ``` - -## FAQ - -### Using tags - -If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. - -### Using _extra variables_ - -Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. - -### Using Ansible collection and role paths - -Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. diff --git a/public-cloud/aws/cml/README.md b/public-cloud/aws/cml/README.md index f5bcb6f..8e8b86e 100644 --- a/public-cloud/aws/cml/README.md +++ b/public-cloud/aws/cml/README.md @@ -10,11 +10,11 @@ To run, you need: * AWS credentials (set via `AWS_PROFILE`) * CDP credentials (set via `CDP_PROFILE`) -[^1]: [OrbStack](https://orbstack.dev) works well on OSX. +[^1]: For example, [OrbStack](https://orbstack.dev) works well on OSX. ## Set Up -First, set up your `cdp-navigator` environment -- follow the instructions in the [README](../../README.md). +First, set up your `ansible-navigator` aka `cdp-navigator` environment -- follow the instructions in the top-level [README](../../../README.md#setting-up-ansible-navigator). Then, clone this project and change your working directory. @@ -39,7 +39,7 @@ admin_password: "Secret" # 1 upper, 1 special, 1 number, 8-64 chars. infra_region: us-east-2 ``` -NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](#faq) section. +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [FAQ](../../../FAQ.md#how-to-i-add-extra-variables-and-tags-to-ansible-navigator) for details. ### SSH Keys @@ -60,17 +60,3 @@ Tear down the CDP Public Cloud by running the playbook: ```bash ansible-navigator run teardown.yml ``` - -## FAQ - -### Using tags - -If you want to run a playbook with a given tag, e.g. `-t infra`, then simply add it as a parameter to the `ansible-navigator` commandline. For example, `ansible-navigator run playbook.yml -t infra`. - -### Using _extra variables_ - -Like [tags](#using-tags), so you can pass _extra variables_ to `ansible-navigator` and the underlying Ansible command. For example, `ansible-navigator run playbook.yml -e @some_config.yml -e some_var=yes`. - -### Using Ansible collection and role paths - -Make sure you do _not_ have `ANSIBLE_COLLECTIONS_PATH` or `ANSIBLE_ROLES_PATH` set or `ansible-navigator` will pick up these environment variables and attempt to use them if set! This is great if you want to use host-based collections, e.g. local development, but you need to ensure that you update the `ansible-navigator.yml` configuration file to mount the host collection and/or role directories into the execution environment container. From 96e62d68a32fb8dbeb3721fe9a9b7b7e86c78b74 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 10 Aug 2023 12:07:08 -0400 Subject: [PATCH 12/13] Formatting of migration entry Signed-off-by: Webster Mudge --- FAQ.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/FAQ.md b/FAQ.md index 71d041c..17d8ff7 100644 --- a/FAQ.md +++ b/FAQ.md @@ -4,14 +4,19 @@ The project undertook some serious remodeling, but rest assured, your definitions will still work as they did in the previous version of `cloudera-deploy`. -So where did everything go? Well... +Okay, but where did everything go? Well... + +1. The `quickstart.sh` migrated to `ansible-navigator`. Both of these applications use a container based on `ansible-runner`, i.e. [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner), to execute the playbooks, yet `ansible-navigator` is configuration-driven and better aligned with how AWX runs Ansible in containers. Also, `ansible-navigator` brings a nifty UI and the ease of use to handle different execution modes. + + We also migrated `cldr-runner` to use `ansible-builder`, but you can read more about that effort at the [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner) project. + +1. The original `cloudera-deploy` playbooks moved into `cloudera.exe`. Starting with Ansible `2.11`, [collections can contain playbooks](https://docs.ansible.com/ansible/latest/collections_guide/collections_using_playbooks.html#using-a-playbook-from-a-collection). We call the playbooks using `import_playbook` like roles. + + PLEASE NOTE, if you are developing your own project playbooks, you must first set up your `cloudera-deploy` variables _before_ calling the playbooks by running the `cloudera.exe.init_deployment` role on `localhost`. -1. The `quickstart.sh` migrated to `ansible-navigator`. Both of these applications use a container based on `ansible-runner`, i.e. [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner), to execute the playbooks, yet `ansible-navigator` is configuration-driven and better aligned with how AWX runs Ansible in containers. - `ansible-navigator` brings a nifty UI and the ease of use to handle different execution modes. - We also migrated `cldr-runner` to use `ansible-builder`, but you can read more about that effort at the [`cldr-runner`](https://github.com/cloudera-labs/cldr-runner) project. -1. The original `cloudera-deploy` playbooks moved into `cloudera.exe`. Starting with Ansible `2.11`, [collections can contain playbooks](https://docs.ansible.com/ansible/latest/collections_guide/collections_using_playbooks.html#using-a-playbook-from-a-collection). We call the playbooks using `import_playbook` like roles. PLEASE NOTE, if you are developing your own project playbooks, you must first set up your `cloudera-deploy` variables _before_ calling the playbooks by running the `cloudera.exe.init_deployment` role on `localhost`. 1. The _run-levels_ still remain; you can still use `-t infra` for example. However, the playbooks themselves are more granular and overall set up and tear down processes are now separate playbooks. - This change promotes composibility and reusability, and we are going to continue to break apart the functions and operations within `cloudera-deploy` and -- most importantly -- the collections that drive this application. We fully expect that you will want to adapt and create your own "deploy" application, one that caters to _your_ needs and operating parameters. Switching to a more granular, more modular approach is key to this objective. + + This change promotes composibility and reusability, and we are going to continue to break apart the functions and operations within `cloudera-deploy` and -- most importantly -- the collections that drive this application. We fully expect that you will want to adapt and create your own "deploy" application, one that caters to _your_ needs and operating parameters. Switching to a more granular, more modular approach is key to this objective. ## How to I add _extra variables_ and tags to `ansible-navigator`? From eaf0a9d78bdc6c4f74d1a0f6c4aea44c66b76b12 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Mon, 28 Aug 2023 20:30:40 +0100 Subject: [PATCH 13/13] Add TF example for CDP Public Cloud on AWS (#1) * Add TF example for CDP on AWS * Create SSH keypair Signed-off-by: Jim Enright --- public-cloud/aws/tf/.gitignore | 35 ++++++ public-cloud/aws/tf/README.md | 71 +++++++++++ public-cloud/aws/tf/ansible-navigator.yml | 65 ++++++++++ public-cloud/aws/tf/config.yml | 13 ++ public-cloud/aws/tf/inventory.ini | 15 +++ public-cloud/aws/tf/main.yml | 70 +++++++++++ public-cloud/aws/tf/pbc_deploy_tf/main.tf | 28 +++++ public-cloud/aws/tf/pbc_deploy_tf/outputs.tf | 7 ++ .../aws/tf/pbc_deploy_tf/variables.tf | 114 ++++++++++++++++++ public-cloud/aws/tf/pbc_infra_tf/main.tf | 66 ++++++++++ public-cloud/aws/tf/pbc_infra_tf/outputs.tf | 55 +++++++++ public-cloud/aws/tf/pbc_infra_tf/variables.tf | 25 ++++ public-cloud/aws/tf/teardown.yml | 48 ++++++++ .../aws/tf/templates/cdp_aws_deploy.tfvars.j2 | 27 +++++ .../tf/templates/cdp_aws_prereqs.tfvars.j2 | 11 ++ 15 files changed, 650 insertions(+) create mode 100644 public-cloud/aws/tf/.gitignore create mode 100644 public-cloud/aws/tf/README.md create mode 100644 public-cloud/aws/tf/ansible-navigator.yml create mode 100644 public-cloud/aws/tf/config.yml create mode 100644 public-cloud/aws/tf/inventory.ini create mode 100644 public-cloud/aws/tf/main.yml create mode 100644 public-cloud/aws/tf/pbc_deploy_tf/main.tf create mode 100644 public-cloud/aws/tf/pbc_deploy_tf/outputs.tf create mode 100644 public-cloud/aws/tf/pbc_deploy_tf/variables.tf create mode 100644 public-cloud/aws/tf/pbc_infra_tf/main.tf create mode 100644 public-cloud/aws/tf/pbc_infra_tf/outputs.tf create mode 100644 public-cloud/aws/tf/pbc_infra_tf/variables.tf create mode 100644 public-cloud/aws/tf/teardown.yml create mode 100644 public-cloud/aws/tf/templates/cdp_aws_deploy.tfvars.j2 create mode 100644 public-cloud/aws/tf/templates/cdp_aws_prereqs.tfvars.j2 diff --git a/public-cloud/aws/tf/.gitignore b/public-cloud/aws/tf/.gitignore new file mode 100644 index 0000000..19b7a8d --- /dev/null +++ b/public-cloud/aws/tf/.gitignore @@ -0,0 +1,35 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator.log +runs +context + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +**/*.tfstate +**/*.tfstate.* + +# Crash log files +**/crash.log +**/crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +**/*.tfvars +**/*.tfvars.json \ No newline at end of file diff --git a/public-cloud/aws/tf/README.md b/public-cloud/aws/tf/README.md new file mode 100644 index 0000000..2fc41c1 --- /dev/null +++ b/public-cloud/aws/tf/README.md @@ -0,0 +1,71 @@ +# CDP Public Cloud - Environment and Datalake Base Example + +> Constructs a CDP Public Cloud Environment and Datalake. Uses the [terraform-cdp-modules](https://github.com/cloudera-labs/terraform-cdp-modules), called via Ansible, to generate the AWS infrastructure pre-requisite resources and the CDP artifacts. + +> **NOTE:** This deployment example does not use a `definition.yml` based configuration file. Instead a standard Ansible extra vars configuration file is used. + +## Requirements + +To run, you need: + +* Docker (or a Docker clone[^1]) +* AWS credentials (set via `AWS_PROFILE`) +* CDP credentials (set via `CDP_PROFILE`) + +[^1]: For example, [OrbStack](https://orbstack.dev) works well on OSX. + +## Set Up + +First, set up your `ansible-navigator` aka `cdp-navigator` environment -- follow the instructions in the top-level [README](../../../README.md#setting-up-ansible-navigator). + +Then, clone this project and change your working directory. + +```bash +git clone https://github.com/cloudera-labs/cloudera-deploy.git; cd cloudera-deploy/public-cloud/aws/base +``` + +## Configure + +Set the required environment variables: + +```bash +export AWS_PROFILE=your-aws-profile +export CDP_PROFILE=your-cdp-profile +``` + +Tweak the `config.yml` parameters to your liking. Notably, you should add and/or change: + +```yaml +name_prefix: ex01 # Keep this short (4-7 characters) +infra_region: us-east-2 # CSP region for infra + +deployment_template: public # Specify the deployment pattern below. Options are public, semi-private or private +``` + +NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e name_prefix=ex01`. See the [FAQ](../../../FAQ.md#how-to-i-add-extra-variables-and-tags-to-ansible-navigator) for details. + +### SSH Keys + +This definition will create a new SSH keypair on the host of the name `-ssh-key.{pem,pub}`. This is stored in the `./pbc_infra_tf` directory. A AWS Keypair will be created using the generated public key. + +## Execute + +Then set up the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run main.yml +``` + +### Terraform resource files + +The Terraform root module resource files run by the playbook are in the `./pbc_infra_tf/` (for cloud infrastructure deployment) `./pbc_deploy_tf/` (CDP deployment) sub-directories. + +Standard Terraform commands - e.g. `terraform output`, `terraform console`, can be run from within these directories. + +## Tear Down + +Tear down the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run teardown.yml +``` diff --git a/public-cloud/aws/tf/ansible-navigator.yml b/public-cloud/aws/tf/ansible-navigator.yml new file mode 100644 index 0000000..a0dc3ef --- /dev/null +++ b/public-cloud/aws/tf/ansible-navigator.yml @@ -0,0 +1,65 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + ansible: + inventory: + entries: + - inventory.ini + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - AWS_PROFILE + - CDP_PROFILE + set: + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + image: ghcr.io/cloudera-labs/cldr-runner:aws-devel + pull: + arguments: + - "--tls-verify=false" + volume-mounts: + - src: "${ANSIBLE_COLLECTIONS_PATH}" + dest: "${ANSIBLE_COLLECTIONS_PATH}" + options: "Z" + - src: "~/.aws" + dest: "/runner/.aws" + options: "Z" + - src: "~/.cdp" + dest: "/runner/.cdp" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/public-cloud/aws/tf/config.yml b/public-cloud/aws/tf/config.yml new file mode 100644 index 0000000..2fb74b8 --- /dev/null +++ b/public-cloud/aws/tf/config.yml @@ -0,0 +1,13 @@ +--- +name_prefix: "" # You must specify a name prefix + +infra_region: eu-west-1 # CSP region for infra + +deployment_template: semi-private # Specify the deployment pattern below. Options are public, semi-private or private + +# Limit to the caller/controller +allowed_cidrs: "{{ lookup('ansible.builtin.url', 'https://api.ipify.org', wantlist=True) | product(['32']) | map('join', '/') | list }}" + +ingress_extra_cidrs_and_ports: + cidrs: "{{ allowed_cidrs }}" + ports: [22, 443] diff --git a/public-cloud/aws/tf/inventory.ini b/public-cloud/aws/tf/inventory.ini new file mode 100644 index 0000000..fef0f4a --- /dev/null +++ b/public-cloud/aws/tf/inventory.ini @@ -0,0 +1,15 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" \ No newline at end of file diff --git a/public-cloud/aws/tf/main.yml b/public-cloud/aws/tf/main.yml new file mode 100644 index 0000000..5b44f17 --- /dev/null +++ b/public-cloud/aws/tf/main.yml @@ -0,0 +1,70 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: no + tasks: + - name: Set up CDP Public Cloud infrastructure + block: + - name: Create TF input variables file for infra pre-reqs module + ansible.builtin.template: + src: cdp_aws_prereqs.tfvars.j2 + dest: "pbc_infra_tf/terraform.tfvars" + + - name: Terraform apply to set up the Cloud infrastructure + # cloud.terraform.terraform: # NOTE: When use this get error similar to https://github.com/ansible-collections/cloud.terraform/issues/86 + community.general.terraform: + project_path: "pbc_infra_tf/" + state: "present" + force_init: true + # provider_upgrade: true + + - name: Set up CDP Public Cloud (Env and DL example) + block: + - name: Create TF input variables file for CDP deploy module + ansible.builtin.template: + src: cdp_aws_deploy.tfvars.j2 + dest: "pbc_deploy_tf/terraform.tfvars" + vars: + infra_type: "aws" + aws_key_pair: "{{ lookup('cloud.terraform.tf_output', 'aws_key_name', project_path='pbc_infra_tf/') }}" + aws_vpc_id: "{{ lookup('cloud.terraform.tf_output', 'aws_vpc_id', project_path='pbc_infra_tf/') }}" + aws_public_subnet_ids: "{{ lookup('cloud.terraform.tf_output', 'aws_public_subnet_ids', project_path='pbc_infra_tf/') }}" + aws_private_subnet_ids: "{{ lookup('cloud.terraform.tf_output', 'aws_private_subnet_ids', project_path='pbc_infra_tf/') }}" + + aws_security_group_default_id: "{{ lookup('cloud.terraform.tf_output', 'aws_security_group_default_id', project_path='pbc_infra_tf/') }}" + aws_security_group_knox_id: "{{ lookup('cloud.terraform.tf_output', 'aws_security_group_knox_id', project_path='pbc_infra_tf/') }}" + + data_storage_location: "{{ lookup('cloud.terraform.tf_output', 'aws_data_storage_location', project_path='pbc_infra_tf/') }}" + log_storage_location: "{{ lookup('cloud.terraform.tf_output', 'aws_log_storage_location', project_path='pbc_infra_tf/') }}" + backup_storage_location: "{{ lookup('cloud.terraform.tf_output', 'aws_backup_storage_location', project_path='pbc_infra_tf/') }}" + + aws_xaccount_role_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_xaccount_role_arn', project_path='pbc_infra_tf/') }}" + aws_datalake_admin_role_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_datalake_admin_role_arn', project_path='pbc_infra_tf/') }}" + aws_ranger_audit_role_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_ranger_audit_role_arn', project_path='pbc_infra_tf/') }}" + + aws_log_instance_profile_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_log_instance_profile_arn', project_path='pbc_infra_tf/') }}" + aws_idbroker_instance_profile_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_idbroker_instance_profile_arn', project_path='pbc_infra_tf/') }}" + + - name: Terraform apply to set up CDP (may take up to 1 hour to complete) + # cloud.terraform.terraform: # NOTE: When use this get error similar to https://github.com/ansible-collections/cloud.terraform/issues/86 + community.general.terraform: + project_path: "pbc_deploy_tf/" + state: "present" + force_init: true + # provider_upgrade: true diff --git a/public-cloud/aws/tf/pbc_deploy_tf/main.tf b/public-cloud/aws/tf/pbc_deploy_tf/main.tf new file mode 100644 index 0000000..5bdfa07 --- /dev/null +++ b/public-cloud/aws/tf/pbc_deploy_tf/main.tf @@ -0,0 +1,28 @@ +module "cdp_deploy" { + source = "git::https://github.com/cloudera-labs/terraform-cdp-modules.git//modules/terraform-cdp-deploy?ref=v0.2.0" + + env_prefix = var.env_prefix + infra_type = "aws" + region = var.aws_region + keypair_name = var.aws_key_pair + deployment_template = var.deployment_template + + aws_vpc_id = var.aws_vpc_id + aws_public_subnet_ids = var.aws_public_subnet_ids + aws_private_subnet_ids = var.aws_private_subnet_ids + + aws_security_group_default_id = var.aws_security_group_default_id + aws_security_group_knox_id = var.aws_security_group_knox_id + + data_storage_location = var.data_storage_location + log_storage_location = var.log_storage_location + backup_storage_location = var.backup_storage_location + + aws_xaccount_role_arn = var.aws_xaccount_role_arn + aws_datalake_admin_role_arn = var.aws_datalake_admin_role_arn + aws_ranger_audit_role_arn = var.aws_ranger_audit_role_arn + + aws_log_instance_profile_arn = var.aws_log_instance_profile_arn + aws_idbroker_instance_profile_arn = var.aws_idbroker_instance_profile_arn + +} \ No newline at end of file diff --git a/public-cloud/aws/tf/pbc_deploy_tf/outputs.tf b/public-cloud/aws/tf/pbc_deploy_tf/outputs.tf new file mode 100644 index 0000000..2f432c7 --- /dev/null +++ b/public-cloud/aws/tf/pbc_deploy_tf/outputs.tf @@ -0,0 +1,7 @@ +output "cdp_environment_name" { + value = module.cdp_deploy.cdp_environment_name +} + +output "cdp_environment_crn" { + value = module.cdp_deploy.cdp_environment_crn +} diff --git a/public-cloud/aws/tf/pbc_deploy_tf/variables.tf b/public-cloud/aws/tf/pbc_deploy_tf/variables.tf new file mode 100644 index 0000000..3806c97 --- /dev/null +++ b/public-cloud/aws/tf/pbc_deploy_tf/variables.tf @@ -0,0 +1,114 @@ +# ------- Global settings ------- +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "infra_type" { + type = string + description = "Cloud Provider to deploy CDP." + +} + +variable "aws_key_pair" { + type = string + + description = "Name of the Public SSH key for the CDP environment" + +} + +# ------- CDP Environment Deployment ------- +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" +} + +# ------- Cloud Service Provider Settings - AWS specific ------- + +variable "aws_vpc_id" { + type = string + description = "AWS Virtual Private Network ID. Required for CDP deployment on AWS." + +} + +variable "aws_public_subnet_ids" { + type = list(string) + description = "List of public subnet ids. Required for CDP deployment on AWS." + +} + +variable "aws_private_subnet_ids" { + type = list(string) + description = "List of private subnet ids. Required for CDP deployment on AWS." + +} + +variable "aws_security_group_default_id" { + type = string + + description = "ID of the Default Security Group for CDP environment. Required for CDP deployment on AWS." + +} + +variable "aws_security_group_knox_id" { + type = string + + description = "ID of the Knox Security Group for CDP environment. Required for CDP deployment on AWS." + +} + +variable "data_storage_location" { + type = string + description = "Data storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs://" +} + +variable "log_storage_location" { + type = string + description = "Log storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs://" +} + +variable "backup_storage_location" { + type = string + description = "Backup storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs://" +} + +variable "aws_datalake_admin_role_arn" { + type = string + + description = "Datalake Admin Role ARN. Required for CDP deployment on AWS." + +} + +variable "aws_ranger_audit_role_arn" { + type = string + + description = "Ranger Audit Role ARN. Required for CDP deployment on AWS." + +} + +variable "aws_xaccount_role_arn" { + type = string + + description = "Cross Account Role ARN. Required for CDP deployment on AWS." + +} + +variable "aws_log_instance_profile_arn" { + type = string + + description = "Log Instance Profile ARN. Required for CDP deployment on AWS." + +} + +variable "aws_idbroker_instance_profile_arn" { + type = string + + description = "IDBroker Instance Profile ARN. Required for CDP deployment on AWS." + +} diff --git a/public-cloud/aws/tf/pbc_infra_tf/main.tf b/public-cloud/aws/tf/pbc_infra_tf/main.tf new file mode 100644 index 0000000..07982b1 --- /dev/null +++ b/public-cloud/aws/tf/pbc_infra_tf/main.tf @@ -0,0 +1,66 @@ +##### Terraform and Provider Configuration ##### +terraform { + required_providers { + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" + } + } + + required_version = ">= 0.13" +} + +provider "aws" { + region = var.aws_region +} + +##### Create SSH keys and AWS keypair ##### + +# Create and save a RSA key +resource "tls_private_key" "cdp_private_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Save the private key to -ssh-key.pem +resource "local_sensitive_file" "pem_file" { + filename = "${var.env_prefix}-ssh-key.pem" + file_permission = "600" + directory_permission = "700" + content = tls_private_key.cdp_private_key.private_key_pem +} + +# Save the public key to -ssh-key.pub +resource "local_file" "pub_file" { + filename = "${var.env_prefix}-ssh-key.pub" + content = tls_private_key.cdp_private_key.public_key_openssh +} + +# Create an AWS EC2 keypair from the generated public key +resource "aws_key_pair" "cdp_keypair" { + key_name = "${var.env_prefix}-keypair" + public_key = tls_private_key.cdp_private_key.public_key_openssh +} + +##### Find xaccount ids using CDP provider data source ##### + +# Use the CDP Terraform Provider to find the xaccount account and external ids +data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} + +##### Create the AWS pre-requisite resources for CDP ##### +# Using the terraform-cdp-aws-pre-reqs module +module "cdp_aws_prereqs" { + source = "git::https://github.com/cloudera-labs/terraform-cdp-modules.git//modules/terraform-cdp-aws-pre-reqs?ref=v0.2.0" + + env_prefix = var.env_prefix + aws_region = var.aws_region + + deployment_template = var.deployment_template + ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + + # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + + +} \ No newline at end of file diff --git a/public-cloud/aws/tf/pbc_infra_tf/outputs.tf b/public-cloud/aws/tf/pbc_infra_tf/outputs.tf new file mode 100644 index 0000000..f33760f --- /dev/null +++ b/public-cloud/aws/tf/pbc_infra_tf/outputs.tf @@ -0,0 +1,55 @@ +output "aws_key_name" { + value = aws_key_pair.cdp_keypair.key_name +} + +output "aws_vpc_id" { + value = module.cdp_aws_prereqs.aws_vpc_id +} + +output "aws_public_subnet_ids" { + value = module.cdp_aws_prereqs.aws_public_subnet_ids +} + +output "aws_private_subnet_ids" { + value = module.cdp_aws_prereqs.aws_private_subnet_ids +} + +output "aws_security_group_default_id" { + value = module.cdp_aws_prereqs.aws_security_group_default_id +} + +output "aws_security_group_knox_id" { + value = module.cdp_aws_prereqs.aws_security_group_knox_id +} + +output "aws_data_storage_location" { + value = module.cdp_aws_prereqs.aws_data_storage_location +} + +output "aws_log_storage_location" { + value = module.cdp_aws_prereqs.aws_log_storage_location +} + +output "aws_backup_storage_location" { + value = module.cdp_aws_prereqs.aws_backup_storage_location +} + +output "aws_xaccount_role_arn" { + value = module.cdp_aws_prereqs.aws_xaccount_role_arn +} + +output "aws_datalake_admin_role_arn" { + value = module.cdp_aws_prereqs.aws_datalake_admin_role_arn +} + +output "aws_ranger_audit_role_arn" { + value = module.cdp_aws_prereqs.aws_ranger_audit_role_arn +} + +output "aws_log_instance_profile_arn" { + value = module.cdp_aws_prereqs.aws_log_instance_profile_arn +} + +output "aws_idbroker_instance_profile_arn" { + value = module.cdp_aws_prereqs.aws_idbroker_instance_profile_arn +} diff --git a/public-cloud/aws/tf/pbc_infra_tf/variables.tf b/public-cloud/aws/tf/pbc_infra_tf/variables.tf new file mode 100644 index 0000000..1385f98 --- /dev/null +++ b/public-cloud/aws/tf/pbc_infra_tf/variables.tf @@ -0,0 +1,25 @@ +# ------- Global settings ------- +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} +# ------- CDP Environment Deployment ------- +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" +} + +# ------- Network Resources ------- +variable "ingress_extra_cidrs_and_ports" { + type = object({ + cidrs = list(string) + ports = list(number) + }) + description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" +} \ No newline at end of file diff --git a/public-cloud/aws/tf/teardown.yml b/public-cloud/aws/tf/teardown.yml new file mode 100644 index 0000000..ba0df35 --- /dev/null +++ b/public-cloud/aws/tf/teardown.yml @@ -0,0 +1,48 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + + - name: Tear down CDP Public Cloud (Env and DL example) + block: + + - name: Terraform destroy for CDP + # cloud.terraform.terraform: + community.general.terraform: + project_path: "pbc_deploy_tf/" + state: "absent" + + - name: Remove the TF input variables file for CDP deploy module + ansible.builtin.file: + path: "pbc_deploy_tf/terraform.tfvars" + state: "absent" + + - name: Tear down CDP Public Cloud infrastructure + block: + - name: Terraform destroy for Cloud infrastructure + # cloud.terraform.terraform: + community.general.terraform: + project_path: "pbc_infra_tf/" + state: "absent" + + - name: Remote TF input variables file for infra pre-reqs module + ansible.builtin.file: + dest: "pbc_infra_tf/terraform.tfvars" + state: "absent" diff --git a/public-cloud/aws/tf/templates/cdp_aws_deploy.tfvars.j2 b/public-cloud/aws/tf/templates/cdp_aws_deploy.tfvars.j2 new file mode 100644 index 0000000..870f812 --- /dev/null +++ b/public-cloud/aws/tf/templates/cdp_aws_deploy.tfvars.j2 @@ -0,0 +1,27 @@ +# ------- Global settings ------- +env_prefix = "{{ name_prefix }}" +infra_type = "{{ infra_type }}" +aws_region = "{{ infra_region }}" + + +# ------- CDP Deployment Settings ------- +deployment_template = "{{ deployment_template }}" +aws_key_pair = "{{ aws_key_pair }}" + +aws_vpc_id = "{{ aws_vpc_id }}" +aws_public_subnet_ids = {{ aws_public_subnet_ids | to_json }} +aws_private_subnet_ids = {{ aws_private_subnet_ids | to_json }} + +aws_security_group_default_id = "{{ aws_security_group_default_id }}" +aws_security_group_knox_id = "{{ aws_security_group_knox_id }}" + +data_storage_location = "{{ data_storage_location }}" +log_storage_location = "{{ log_storage_location }}" +backup_storage_location = "{{ backup_storage_location }}" + +aws_xaccount_role_arn = "{{ aws_xaccount_role_arn }}" +aws_datalake_admin_role_arn = "{{ aws_datalake_admin_role_arn }}" +aws_ranger_audit_role_arn = "{{ aws_ranger_audit_role_arn }}" + +aws_log_instance_profile_arn = "{{ aws_log_instance_profile_arn }}" +aws_idbroker_instance_profile_arn = "{{ aws_idbroker_instance_profile_arn }}" \ No newline at end of file diff --git a/public-cloud/aws/tf/templates/cdp_aws_prereqs.tfvars.j2 b/public-cloud/aws/tf/templates/cdp_aws_prereqs.tfvars.j2 new file mode 100644 index 0000000..8e07ed3 --- /dev/null +++ b/public-cloud/aws/tf/templates/cdp_aws_prereqs.tfvars.j2 @@ -0,0 +1,11 @@ +# ------- Global settings ------- +env_prefix = "{{ name_prefix }}" +aws_region = "{{ infra_region }}" +# ------- CDP Environment Deployment ------- +deployment_template = "{{ deployment_template }}" + +# ------- Network Settings ------- +ingress_extra_cidrs_and_ports = { + cidrs = {{ ingress_extra_cidrs_and_ports['cidrs'] | to_json }}, + ports = {{ ingress_extra_cidrs_and_ports['ports'] }} +}