From 34c15b2bab6429e0faab235ed5447450dc02c592 Mon Sep 17 00:00:00 2001 From: Stephan Baum Date: Mon, 17 Nov 2025 10:23:15 +0100 Subject: [PATCH 1/2] refactor(tdp_lib_dag): make yarn_nodemanager_install dependant of spark3_yarnshuffle_install --- tdp_lib_dag/spark3.yml | 7 ++----- tdp_lib_dag/yarn.yml | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tdp_lib_dag/spark3.yml b/tdp_lib_dag/spark3.yml index 1d75866a..78122d83 100644 --- a/tdp_lib_dag/spark3.yml +++ b/tdp_lib_dag/spark3.yml @@ -19,15 +19,13 @@ - spark3_historyserver_install - name: spark3_yarnshuffle_install - depends_on: - - yarn_nodemanager_install + depends_on: [] - name: spark3_client_config depends_on: - tdp-cluster_config - hadoop_client_config - spark3_kerberos_install - - hbase_client_config - name: spark3_historyserver_config depends_on: @@ -48,14 +46,13 @@ - name: spark3_historyserver_start depends_on: - spark3_jmx-exporter_config - - yarn_client_config - spark3_historyserver_config - spark3_hdfs_init - name: spark3_historyserver_init noop: yes depends_on: - - yarn_client_init + - spark3_historyserver_start - name: spark3_install noop: yes diff --git a/tdp_lib_dag/yarn.yml b/tdp_lib_dag/yarn.yml index 7667aba7..9c0c98f3 100644 --- a/tdp_lib_dag/yarn.yml +++ b/tdp_lib_dag/yarn.yml @@ -9,6 +9,7 @@ - name: yarn_nodemanager_install depends_on: - hadoop_install + - spark3_yarnshuffle_install - name: yarn_apptimelineserver_install depends_on: From 8e2f3e8627ae877a7bdeca348a01dd94eec552bc Mon Sep 17 00:00:00 2001 From: Stephan Baum Date: Fri, 21 Nov 2025 15:22:18 +0100 Subject: [PATCH 2/2] refactor(meta): regenerated the meta playbook --- playbooks/meta/all.yml | 6 +++--- playbooks/meta/all_per_service.yml | 2 +- playbooks/meta/restart.yml | 6 +++--- playbooks/meta/spark3.yml | 2 +- playbooks/meta/start.yml | 6 +++--- playbooks/meta/yarn.yml | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/playbooks/meta/all.yml b/playbooks/meta/all.yml index 62425247..764fe3ed 100644 --- a/playbooks/meta/all.yml +++ b/playbooks/meta/all.yml @@ -28,10 +28,8 @@ - ansible.builtin.import_playbook: ../yarn_apptimelineserver_install.yml - ansible.builtin.import_playbook: ../yarn_client_install.yml - ansible.builtin.import_playbook: ../yarn_mapred_jobhistoryserver_install.yml -- ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml - ansible.builtin.import_playbook: ../yarn_ranger_install.yml - ansible.builtin.import_playbook: ../yarn_resourcemanager_install.yml -- ansible.builtin.import_playbook: ../yarn_ssl-tls_install.yml - ansible.builtin.import_playbook: ../hive_client_install.yml - ansible.builtin.import_playbook: ../hive_hiveserver2_install.yml - ansible.builtin.import_playbook: ../hive_metastore_install.yml @@ -52,6 +50,8 @@ - ansible.builtin.import_playbook: ../spark3_historyserver_install.yml - ansible.builtin.import_playbook: ../spark3_ssl-tls_install.yml - ansible.builtin.import_playbook: ../spark3_yarnshuffle_install.yml +- ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml +- ansible.builtin.import_playbook: ../yarn_ssl-tls_install.yml - ansible.builtin.import_playbook: ../knox_gateway_install.yml - ansible.builtin.import_playbook: ../knox_kerberos_install.yml - ansible.builtin.import_playbook: ../knox_ranger_install.yml @@ -182,9 +182,9 @@ - ansible.builtin.import_playbook: ../spark3_hdfs_init.yml - ansible.builtin.import_playbook: ../spark3_historyserver_config.yml # spark3_config -# spark3_historyserver_init - ansible.builtin.import_playbook: ../spark3_jmx-exporter_config.yml - ansible.builtin.import_playbook: ../spark3_historyserver_start.yml +# spark3_historyserver_init # spark3_start # spark3_init - ansible.builtin.import_playbook: ../knox_gateway_config.yml diff --git a/playbooks/meta/all_per_service.yml b/playbooks/meta/all_per_service.yml index fc9acee3..c28228b8 100644 --- a/playbooks/meta/all_per_service.yml +++ b/playbooks/meta/all_per_service.yml @@ -9,8 +9,8 @@ - ansible.builtin.import_playbook: kerberos.yml - ansible.builtin.import_playbook: ranger.yml - ansible.builtin.import_playbook: hdfs.yml +- ansible.builtin.import_playbook: spark3.yml - ansible.builtin.import_playbook: yarn.yml - ansible.builtin.import_playbook: hbase.yml - ansible.builtin.import_playbook: hive.yml -- ansible.builtin.import_playbook: spark3.yml - ansible.builtin.import_playbook: knox.yml diff --git a/playbooks/meta/restart.yml b/playbooks/meta/restart.yml index de6e2484..ab1d1a12 100644 --- a/playbooks/meta/restart.yml +++ b/playbooks/meta/restart.yml @@ -28,10 +28,8 @@ # - ansible.builtin.import_playbook: ../yarn_apptimelineserver_install.yml # - ansible.builtin.import_playbook: ../yarn_client_install.yml # - ansible.builtin.import_playbook: ../yarn_mapred_jobhistoryserver_install.yml -# - ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml # - ansible.builtin.import_playbook: ../yarn_ranger_install.yml # - ansible.builtin.import_playbook: ../yarn_resourcemanager_install.yml -# - ansible.builtin.import_playbook: ../yarn_ssl-tls_install.yml # - ansible.builtin.import_playbook: ../hive_client_install.yml # - ansible.builtin.import_playbook: ../hive_hiveserver2_install.yml # - ansible.builtin.import_playbook: ../hive_metastore_install.yml @@ -52,6 +50,8 @@ # - ansible.builtin.import_playbook: ../spark3_historyserver_install.yml # - ansible.builtin.import_playbook: ../spark3_ssl-tls_install.yml # - ansible.builtin.import_playbook: ../spark3_yarnshuffle_install.yml +# - ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml +# - ansible.builtin.import_playbook: ../yarn_ssl-tls_install.yml # - ansible.builtin.import_playbook: ../knox_gateway_install.yml # - ansible.builtin.import_playbook: ../knox_kerberos_install.yml # - ansible.builtin.import_playbook: ../knox_ranger_install.yml @@ -182,9 +182,9 @@ # - ansible.builtin.import_playbook: ../spark3_hdfs_init.yml # - ansible.builtin.import_playbook: ../spark3_historyserver_config.yml ## spark3_config -## spark3_historyserver_init # - ansible.builtin.import_playbook: ../spark3_jmx-exporter_config.yml - ansible.builtin.import_playbook: ../spark3_historyserver_restart.yml +## spark3_historyserver_init ## spark3_restart ## spark3_init # - ansible.builtin.import_playbook: ../knox_gateway_config.yml diff --git a/playbooks/meta/spark3.yml b/playbooks/meta/spark3.yml index beda762d..aaab674f 100644 --- a/playbooks/meta/spark3.yml +++ b/playbooks/meta/spark3.yml @@ -12,8 +12,8 @@ - ansible.builtin.import_playbook: ../spark3_hdfs_init.yml - ansible.builtin.import_playbook: ../spark3_historyserver_config.yml # spark3_config -# spark3_historyserver_init - ansible.builtin.import_playbook: ../spark3_jmx-exporter_config.yml - ansible.builtin.import_playbook: ../spark3_historyserver_start.yml +# spark3_historyserver_init # spark3_start # spark3_init diff --git a/playbooks/meta/start.yml b/playbooks/meta/start.yml index d51aec60..b67ddd00 100644 --- a/playbooks/meta/start.yml +++ b/playbooks/meta/start.yml @@ -28,10 +28,8 @@ # - ansible.builtin.import_playbook: ../yarn_apptimelineserver_install.yml # - ansible.builtin.import_playbook: ../yarn_client_install.yml # - ansible.builtin.import_playbook: ../yarn_mapred_jobhistoryserver_install.yml -# - ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml # - ansible.builtin.import_playbook: ../yarn_ranger_install.yml # - ansible.builtin.import_playbook: ../yarn_resourcemanager_install.yml -# - ansible.builtin.import_playbook: ../yarn_ssl-tls_install.yml # - ansible.builtin.import_playbook: ../hive_client_install.yml # - ansible.builtin.import_playbook: ../hive_hiveserver2_install.yml # - ansible.builtin.import_playbook: ../hive_metastore_install.yml @@ -52,6 +50,8 @@ # - ansible.builtin.import_playbook: ../spark3_historyserver_install.yml # - ansible.builtin.import_playbook: ../spark3_ssl-tls_install.yml # - ansible.builtin.import_playbook: ../spark3_yarnshuffle_install.yml +# - ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml +# - ansible.builtin.import_playbook: ../yarn_ssl-tls_install.yml # - ansible.builtin.import_playbook: ../knox_gateway_install.yml # - ansible.builtin.import_playbook: ../knox_kerberos_install.yml # - ansible.builtin.import_playbook: ../knox_ranger_install.yml @@ -182,9 +182,9 @@ # - ansible.builtin.import_playbook: ../spark3_hdfs_init.yml # - ansible.builtin.import_playbook: ../spark3_historyserver_config.yml ## spark3_config -## spark3_historyserver_init # - ansible.builtin.import_playbook: ../spark3_jmx-exporter_config.yml - ansible.builtin.import_playbook: ../spark3_historyserver_start.yml +## spark3_historyserver_init ## spark3_start ## spark3_init # - ansible.builtin.import_playbook: ../knox_gateway_config.yml diff --git a/playbooks/meta/yarn.yml b/playbooks/meta/yarn.yml index d30213d9..ddceda5e 100644 --- a/playbooks/meta/yarn.yml +++ b/playbooks/meta/yarn.yml @@ -5,9 +5,9 @@ - ansible.builtin.import_playbook: ../yarn_apptimelineserver_install.yml - ansible.builtin.import_playbook: ../yarn_client_install.yml - ansible.builtin.import_playbook: ../yarn_mapred_jobhistoryserver_install.yml -- ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml - ansible.builtin.import_playbook: ../yarn_ranger_install.yml - ansible.builtin.import_playbook: ../yarn_resourcemanager_install.yml +- ansible.builtin.import_playbook: ../yarn_nodemanager_install.yml - ansible.builtin.import_playbook: ../yarn_ssl-tls_install.yml - ansible.builtin.import_playbook: ../yarn_kerberos_install.yml # yarn_install