Refactor hue

playbooks/hue_kerberos_install.yml

@@ -0,0 +1,15 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Kerberos Hue Server install
+  hosts: hue_server
+  strategy: linear
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_kerberos
+    - name: Install Hue Server Kerberos
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: kerberos
+    - meta: clear_facts # noqa unnamed-task

playbooks/hue_server_config.yml

@@ -0,0 +1,29 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Configure Hue Server
+  hosts: hue_server
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: livy-spark3_server
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_server
+    - name: Configure Hue Server
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: config
+    - meta: clear_facts # noqa unnamed-task
+
+- name: Configure Hive for Hue Server
+  hosts: hue_server
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hive_hue
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_server
+    - name: Configure Hive for Hue Server
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: config_hive
+    - meta: clear_facts # noqa unnamed-task

playbooks/hue_server_install.yml

@@ -0,0 +1,15 @@
+
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Install Hue Server
+  hosts: hue_server
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_server
+    - name: Install Hue Server
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: install
+    - meta: clear_facts # noqa unnamed-task

playbooks/hue_server_restart.yml

@@ -0,0 +1,14 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Restart Hue Server
+  hosts: hue_server
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_server
+    - name: Restart Hue Server
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: restart
+    - meta: clear_facts # noqa unnamed-task

playbooks/hue_server_start.yml

@@ -0,0 +1,14 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Start Hue Server
+  hosts: hue_server
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_server
+    - name: Start Hue Server
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: start
+    - meta: clear_facts # noqa unnamed-task

playbooks/hue_server_stop.yml

@@ -0,0 +1,14 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Stop Hue Server
+  hosts: hue_server
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_server
+    - name: Stop Hue Server
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: stop
+    - meta: clear_facts # noqa unnamed-task

playbooks/hue_ssl-tls_install.yml

@@ -0,0 +1,14 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: SSL-TLS Hue Install
+  hosts: hue_server
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: hue_ssl-tls
+    - name: Install Hue Server SSL-TLS
+      import_role:
+        name: tosit.tdp_extra.hue.server
+        tasks_from: ssl-tls
+    - meta: clear_facts # noqa unnamed-task

playbooks/meta/hue.yml

@@ -0,0 +1,14 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- import_playbook: ../hue_server_install.yml
+- import_playbook: ../hue_kerberos_install.yml
+- import_playbook: ../hue_ssl-tls_install.yml
+# hue_install
+- import_playbook: ../hue_server_config.yml
+# hue_config
+- import_playbook: ../hue_server_start.yml
+# hue_server_init
+# hue_start
+# hue_init

roles/hue/README.md

@@ -0,0 +1,39 @@
+# Ansible Hue TDP Extra
+
+Currently the roles only supports the deployment of HA, SSL-enabled, Kerberos authenticated Hadoop clusters.
+
+## Prerequisites
+
+- `java-1.8.0-openjdk` and `krb5-workstation` installed on all nodes
+- Hue release (`hue_dist_file` role variable) file available in `files`
+- Group `hue_server` defined in the Ansible inventory
+- Certificate files `{{ fqdn }}.key` and `{{ fqdn }}.pem` for every node available in `files`
+- Admin access to a KDC with the `realm`, `kadmin_principal` and `kadmin_password` role vars provided
+- The hue tarball must be present in the files directory. The tested version in [hue-release-4.10.0-TDP-0.1.0.tar.gz] for python2.7 and [hue-release-4.10.0-TDP-0.1.0-python36.tar.gz] for python3.6 (https://github.com/TOSIT-IO/hue/releases/tag/hue-release-4.10.0-TDP-0.1.0-python36).
+- The hue_user role must already exist on the target database where the hue desktop database will be created
+- The LDAP confifugration (`ldapauth` path of variable file) needs to be adapted to your environment.
+- All hue dependencies must exist on the target hue_server [check here for Hue dependencies](https://docs.gethue.com/administrator/installation/dependencies/)
+
+# Notes:
+- The first user logged to the hue web-ui will be the hue admin user.
+- Upon a successful deployment of hue, the default web-ui url is `https://<_hue_server_host_>:<hue_port>`
+- `yarn_rm` and `hdfs_nn` *must* be defined in the ansible hosts file
+- Errors resembling `The error was: ansible.errors.AnsibleUndefinedVariable: 'dict object' has no attribute 'ats'` are often related to a missing entry in the ansible hosts file.
+
+## Example
+
+The following hosts file and playbook are given as examples.
+
+### Host file
+
+```
+[hue_server]
+edge
+```
+
+### Available playbooks
+
+## TODO
+
+- Implement automatic failover for Hive and Hue
+

roles/hue/server/tasks/config.yml

@@ -0,0 +1,43 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Backup configuration
+  copy:
+    src: "{{ hue_conf_dir }}/"
+    dest: "{{ hue_conf_dir }}.{{ ansible_date_time.epoch }}"
+    owner: root
+    group: root
+    mode: "755"
+    remote_src: true
+  tags:
+    - backup
+
+- name: Get {{ hue_user }} uid
+  shell: id -u hue
+  register: hue_user_id_output
+
+- name: Template hue.ini
+  template:
+    src: hue.ini.j2
+    dest: "{{ hue_conf_dir }}/hue.ini"
+    owner: root
+    group: root
+    mode: "0644"
+  vars:
+    hue_user_id: "{{ hue_user_id_output.stdout | trim }}"
+    livy_port: "{{ livy_server_port }}"
+
+- name: Template Hue log configuration
+  template:
+    src: log.conf.j2
+    dest: "{{ hue_conf_dir }}/log.conf"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Initiate Hue database
+  command: |
+    {{ hue_install_dir }}/build/env/bin/hue migrate
+  register: hue_database_migrate
+  failed_when: hue_database_migrate.rc != 0

roles/hue/server/tasks/config_hive.yml

@@ -0,0 +1,11 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Template hive-site.xml
+  template:
+    src: hive-site.xml.j2
+    dest: "{{ hue_conf_dir }}/hive-site.xml"
+    owner: root
+    group: root
+    mode: "644"

roles/hue/server/tasks/install.yml

@@ -0,0 +1,128 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Ensure hadoop group exists
+  include_role:
+    name: tosit.tdp.utils.group
+  vars:
+    group: "{{ hadoop_group }}"
+
+- name: Ensure hue user exists
+  include_role:
+    name: tosit.tdp.utils.user
+  vars:
+    user: "{{ hue_user }}"
+    group: "{{ hadoop_group }}"
+
+- name: Upload {{ hue_dist_file }}
+  copy:
+    src: files/{{ hue_dist_file }}
+    dest: /tmp
+    owner: root
+    group: root
+    mode: "644"
+  diff: false
+
+- name: Ensure {{ hue_root_dir }} exists
+  file:
+    path: "{{ hue_root_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "755"
+
+- name: Extract {{ hue_dist_file }}
+  unarchive:
+    src: "/tmp/{{ hue_dist_file }}"
+    dest: "{{ hue_root_dir }}"
+    owner: root
+    group: root
+    mode: "755"
+    remote_src: true
+    creates: "{{ hue_root_dir }}/{{ hue_release }}"
+
+- name: Ensure hue installation permissions
+  file:
+    path: "{{ hue_root_dir }}/{{ hue_release }}"
+    owner: root
+    group: root
+    mode: "755"
+    state: directory
+    recurse: true
+
+- name: Create symbolic link to Hue installation
+  file:
+    src: "{{ hue_root_dir }}/{{ hue_release }}"
+    dest: "{{ hue_install_dir }}"
+    state: link
+
+- name: Create directory for pid
+  file:
+    path: "{{ hue_pid_dir }}"
+    state: directory
+    group: root
+    owner: root
+    mode: "755"
+
+- name: Create configuration directory
+  file:
+    path: "{{ hue_root_conf_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "755"
+
+- name: Create symbolic link to Hue installation
+  file:
+    src: "{{ hue_root_dir }}/{{ hue_release }}/desktop/conf"
+    dest: "{{ hue_conf_dir }}"
+    state: link
+
+- name: Create certificates directory
+  file:
+    path: "{{ hue_certs_folder }}"
+    state: directory
+    group: root
+    owner: root
+    mode: "0755"
+
+- name: Create directory for pid
+  file:
+    path: "{{ hue_pid_dir }}"
+    state: directory
+    group: root
+    owner: root
+    mode: "755"
+
+- name: Template Hue tmpfiles.d
+  template:
+    src: tmpfiles-hue.conf.j2
+    dest: /etc/tmpfiles.d/hue.conf
+    owner: root
+    group: root
+    mode: "644"
+
+- name: Create log directory
+  file:
+    path: "{{ hue_log_dir }}"
+    state: directory
+    owner: "{{ hue_user }}"
+    group: "{{ hadoop_group }}"
+    mode: "750"
+
+- name: Template Hue stop script
+  template:
+    src: stop-hue-server.sh.j2
+    dest: "{{ hue_install_dir }}/build/env/bin/stop-hue-server.sh"
+    owner: root
+    group: root
+    mode: "644"
+
+- name: Template Hue service file
+  template:
+    src: hue.service.j2
+    dest: /usr/lib/systemd/system/hue.service
+    owner: root
+    group: root
+    mode: "644" 

roles/hue/server/tasks/kerberos.yml

@@ -0,0 +1,32 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Ensure kerberos common installation steps are performed
+  import_role:
+    name: tosit.tdp.utils.kerberos
+    tasks_from: install
+
+- name: Ensure hue user's principal and keytab exist
+  import_role:
+    name: tosit.tdp.utils.kerberos
+    tasks_from: create_principal_keytab
+  vars:
+    principal: hue/{{ ansible_fqdn }}
+    keytab: hue.service.keytab
+    user: "{{ hue_user }}"
+    group: "{{ hadoop_group }}"
+    mode: "0600"
+  when: krb_create_principals_keytabs
+
+- name: Ensure hue's keytab is working
+  import_role:
+    name: tosit.tdp.utils.kerberos
+    tasks_from: check_secure_keytab
+  vars:
+    principal: hue/{{ ansible_fqdn }}
+    keytab: hue.service.keytab
+    user: "{{ hue_user }}"
+    group: "{{ hadoop_group }}"
+    mode: "0600"
+  when: not krb_create_principals_keytabs