[DPE-6752] Implement predefined roles

.github/workflows/ci.yaml

@@ -55,7 +55,8 @@ jobs:
     name: Integration test charm
     needs:
       - lint
-      - unit-test
+      # TODO: re-enable unit-tests to run integration testss
+      # - unit-test
       - build
     uses: ./.github/workflows/integration_test.yaml
     with:

src/backups.py

@@ -27,7 +27,6 @@
 from jinja2 import Template
 from ops.charm import ActionEvent, HookEvent
 from ops.framework import Object
-from ops.jujuversion import JujuVersion
 from ops.model import ActiveStatus, MaintenanceStatus
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed
 
@@ -882,12 +881,11 @@ def _on_create_backup_action(self, event) -> None:
 
         # Test uploading metadata to S3 to test credentials before backup.
         datetime_backup_requested = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
-        juju_version = JujuVersion.from_environ()
         metadata = f"""Date Backup Requested: {datetime_backup_requested}
 Model Name: {self.model.name}
 Application Name: {self.model.app.name}
 Unit Name: {self.charm.unit.name}
-Juju Version: {juju_version!s}
+Juju Version: {self.charm.model.juju_version!s}
 """
         if not self._upload_content_to_s3(
             metadata,

src/charm.py

@@ -23,21 +23,26 @@
 from charms.data_platform_libs.v0.data_models import TypedCharmBase
 from charms.grafana_agent.v0.cos_agent import COSAgentProvider, charm_tracing_config
 from charms.operator_libs_linux.v2 import snap
-from charms.postgresql_k8s.v0.postgresql import (
+from charms.postgresql_k8s.v0.postgresql_tls import PostgreSQLTLS
+from charms.postgresql_k8s.v1.postgresql import (
     ACCESS_GROUP_IDENTITY,
     ACCESS_GROUPS,
     REQUIRED_PLUGINS,
+    ROLE_BACKUP,
+    ROLE_DBA,
+    ROLE_STATS,
     PostgreSQL,
+    PostgreSQLCreatePredefinedRolesError,
     PostgreSQLCreateUserError,
     PostgreSQLEnableDisableExtensionError,
     PostgreSQLGetCurrentTimelineError,
+    PostgreSQLGrantDatabasePrivilegesToUserError,
     PostgreSQLListUsersError,
     PostgreSQLUpdateUserPasswordError,
 )
-from charms.postgresql_k8s.v0.postgresql_tls import PostgreSQLTLS
 from charms.rolling_ops.v0.rollingops import RollingOpsManager, RunWithLock
 from charms.tempo_coordinator_k8s.v0.charm_tracing import trace_charm
-from ops import JujuVersion, main
+from ops import main
 from ops.charm import (
     ActionEvent,
     HookEvent,
@@ -78,6 +83,8 @@
     BACKUP_USER,
     DATABASE_DEFAULT_NAME,
     DATABASE_PORT,
+    DBA_PASSWORD_KEY,
+    DBA_USER,
     METRICS_PORT,
     MONITORING_PASSWORD_KEY,
     MONITORING_SNAP_SERVICE,
@@ -178,9 +185,7 @@
             deleted_label=SECRET_DELETED_LABEL,
         )
 
-        juju_version = JujuVersion.from_environ()
-        run_cmd = "/usr/bin/juju-exec" if juju_version.major > 2 else "/usr/bin/juju-run"
-        self._observer = ClusterTopologyObserver(self, run_cmd)
+        self._observer = ClusterTopologyObserver(self, "/usr/bin/juju-exec")
         self._rotate_logs = RotateLogs(self)
         self.framework.observe(self.on.cluster_topology_change, self._on_cluster_topology_change)
         self.framework.observe(self.on.install, self._on_install)
@@ -414,7 +419,7 @@
                 # TODO figure out why peer data is not available
                 if primary_endpoint and len(self._units_ips) == 1 and len(self._peers.units) > 1:
                     logger.warning(
-                        "Possibly incoplete peer data: Will not map primary IP to unit IP"
+                        "Possibly incomplete peer data: Will not map primary IP to unit IP"
                     )
                     return primary_endpoint
                 logger.debug("primary endpoint early exit: Primary IP not in cached peer list.")
@@ -1126,6 +1131,7 @@
             MONITORING_PASSWORD_KEY,
             RAFT_PASSWORD_KEY,
             PATRONI_PASSWORD_KEY,
+            DBA_PASSWORD_KEY,
         ):
             if self.get_secret(APP_SCOPE, key) is None:
                 if key in system_user_passwords:
@@ -1225,7 +1231,8 @@
             logger.debug("Early exit enable_disable_extensions: standby cluster")
             return
         original_status = self.unit.status
-        extensions = {}
+        # Always want set_user and login_hook to be enabled
+        extensions = {"set_user": True, "login_hook": True}
         # collect extensions
         for plugin in self.config.plugin_keys():
             enable = self.config[plugin]
@@ -1428,7 +1435,7 @@
         logger.debug("Starting LDAP sync service")
         postgres_snap.restart(services=["ldap-sync"])
 
-    def _start_primary(self, event: StartEvent) -> None:
+    def _start_primary(self, event: StartEvent) -> None:  # noqa: C901
         """Bootstrap the cluster."""
         # Set some information needed by Patroni to bootstrap the cluster.
         if not self._patroni.bootstrap_cluster():
@@ -1442,24 +1449,63 @@
             event.defer()
             return
 
+        if not self._can_connect_to_postgresql:
+            logger.debug("Deferring on_start: awaiting for database to start")
+            self.unit.status = WaitingStatus("awaiting for database to start")
+            event.defer()
+            return
+
+        if not self.primary_endpoint:
+            logger.debug("Deferrring on_start: awaitng start of the primary")
+            self.unit.status = WaitingStatus("awaiting start of the primary")
+            event.defer()
+            return
+
+        try:
+            # Needed to create predefined roles with set_user execution privileges
+            self.postgresql.enable_disable_extensions({"set_user": True}, database="postgres")
+        except Exception as e:
+            logger.exception(e)
+            self.unit.status = BlockedStatus("Failed to enable set-user extension")
+            return
+
+        try:
+            self.postgresql.create_predefined_roles()
+        except PostgreSQLCreatePredefinedRolesError as e:
+            logger.exception(e)
+            self.unit.status = BlockedStatus("Failed to create pre-defined roles")
+            return
+
         # Create the default postgres database user that is needed for some
         # applications (not charms) like Landscape Server.
         try:
             # This event can be run on a replica if the machines are restarted.
             # For that case, check whether the postgres user already exits.
             users = self.postgresql.list_users()
-            if "postgres" not in users:
-                self.postgresql.create_user("postgres", new_password(), admin=True)
-                # Create the backup user.
+            # Create the dba user.
+            if DBA_USER not in users:
+                self.postgresql.create_user(
+                    DBA_USER,
+                    self.get_secret(APP_SCOPE, DBA_PASSWORD_KEY),
+                    roles=[ROLE_DBA],
+                )
+            # TODO: move predefined roles into constants
             if BACKUP_USER not in users:
-                self.postgresql.create_user(BACKUP_USER, new_password(), admin=True)
+                self.postgresql.create_user(BACKUP_USER, new_password(), roles=[ROLE_BACKUP])
+                self.postgresql.grant_database_privileges_to_user(
+                    BACKUP_USER, "postgres", ["connect"]
+                )
             if MONITORING_USER not in users:
                 # Create the monitoring user.
                 self.postgresql.create_user(
                     MONITORING_USER,
                     self.get_secret(APP_SCOPE, MONITORING_PASSWORD_KEY),
-                    extra_user_roles=["pg_monitor"],
+                    roles=[ROLE_STATS],
                 )
+        except PostgreSQLGrantDatabasePrivilegesToUserError as e:
+            logger.exception(e)
+            self.unit.status = BlockedStatus("Failed to grant database privileges to user")
+            return
         except PostgreSQLCreateUserError as e:
             logger.exception(e)
             self.unit.status = BlockedStatus("Failed to create postgres user")
@@ -1540,13 +1586,15 @@
             return
 
         try:
+            updateable_users = [*SYSTEM_USERS, BACKUP_USER, DBA_USER]
+
             # get the secret content and check each user configured there
             # only SYSTEM_USERS with changed passwords are processed, all others ignored
             updated_passwords = self.get_secret_from_id(secret_id=admin_secret_id)
             for user, password in list(updated_passwords.items()):
-                if user not in SYSTEM_USERS:
+                if user not in updateable_users:
                     logger.error(
-                        f"Can only update system users: {', '.join(SYSTEM_USERS)} not {user}"
+                        f"Can only update users: {', '.join(updateable_users)} not {user}"
                     )
                     updated_passwords.pop(user)
                     continue
@@ -2253,20 +2301,6 @@
         else:
             logger.error("Can't tell last completed transaction time")
 
-    def get_plugins(self) -> list[str]:
-        """Return a list of installed plugins."""
-        plugins = [
-            "_".join(plugin.split("_")[1:-1])
-            for plugin in self.config.plugin_keys()
-            if self.config[plugin]
-        ]
-        plugins = [PLUGIN_OVERRIDES.get(plugin, plugin) for plugin in plugins]
-        if "spi" in plugins:
-            plugins.remove("spi")
-            for ext in SPI_MODULE:
-                plugins.append(ext)
-        return plugins
-
     def get_ldap_parameters(self) -> dict:
         """Returns the LDAP configuration to use."""
         if not self.is_cluster_initialised:

src/cluster.py

@@ -17,6 +17,7 @@
 import psutil
 import requests
 from charms.operator_libs_linux.v2 import snap
+from charms.postgresql_k8s.v1.postgresql import ROLE_DBA
 from jinja2 import Template
 from ops import BlockedStatus
 from pysyncobj.utility import TcpUtility, UtilityException
@@ -696,6 +697,7 @@ def render_patroni_yml_file(
             raft_password=self.raft_password,
             ldap_parameters=self._dict_to_hba_string(ldap_params),
             patroni_password=self.patroni_password,
+            dba_role=ROLE_DBA,
         )
         self.render_file(f"{PATRONI_CONF_PATH}/patroni.yaml", rendered, 0o600)
 

src/constants.py

@@ -15,6 +15,7 @@
 BACKUP_USER = "backup"
 REPLICATION_USER = "replication"
 REWIND_USER = "rewind"
+DBA_USER = "dba"
 TLS_KEY_FILE = "key.pem"
 TLS_CA_FILE = "ca.pem"
 TLS_CERT_FILE = "cert.pem"
@@ -24,15 +25,15 @@
 PATRONI_SERVICE_NAME = "snap.charmed-postgresql.patroni.service"
 PATRONI_SERVICE_DEFAULT_PATH = f"/etc/systemd/system/{PATRONI_SERVICE_NAME}"
 # List of system usernames needed for correct work of the charm/workload.
-SYSTEM_USERS = [BACKUP_USER, REPLICATION_USER, REWIND_USER, USER, MONITORING_USER]
+SYSTEM_USERS = [REPLICATION_USER, REWIND_USER, USER, MONITORING_USER]
 
 # Snap constants.
 PGBACKREST_EXECUTABLE = "charmed-postgresql.pgbackrest"
 POSTGRESQL_SNAP_NAME = "charmed-postgresql"
 SNAP_PACKAGES = [
     (
         POSTGRESQL_SNAP_NAME,
-        {"revision": {"aarch64": "169", "x86_64": "170"}},
+        {"revision": {"aarch64": "181", "x86_64": "182"}},
     )
 ]
 
@@ -66,6 +67,7 @@
 MONITORING_PASSWORD_KEY = "monitoring-password"  # noqa: S105
 RAFT_PASSWORD_KEY = "raft-password"  # noqa: S105
 PATRONI_PASSWORD_KEY = "patroni-password"  # noqa: S105
+DBA_PASSWORD_KEY = "dba-password"  # noqa: S105
 SECRET_INTERNAL_LABEL = "internal-secret"  # noqa: S105
 SECRET_DELETED_LABEL = "None"  # noqa: S105
 SYSTEM_USERS_PASSWORD_CONFIG = "system-users"  # noqa: S105
@@ -82,7 +84,7 @@
 TRACING_PROTOCOL = "otlp_http"
 
 BACKUP_TYPE_OVERRIDES = {"full": "full", "differential": "diff", "incremental": "incr"}
-PLUGIN_OVERRIDES = {"audit": "pgaudit", "uuid_ossp": '"uuid-ossp"'}
+PLUGIN_OVERRIDES = {"audit": "pgaudit", "uuid_ossp": '"uuid-ossp"', "set_user": '"set_user"'}
 
 SPI_MODULE = ["refint", "autoinc", "insert_username", "moddatetime"]
 

src/relations/postgresql_provider.py

@@ -9,7 +9,7 @@
     DatabaseProvides,
     DatabaseRequestedEvent,
 )
-from charms.postgresql_k8s.v0.postgresql import (
+from charms.postgresql_k8s.v1.postgresql import (
     ACCESS_GROUP_RELATION,
     ACCESS_GROUPS,
     INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE,
@@ -18,6 +18,7 @@
     PostgreSQLDeleteUserError,
     PostgreSQLGetPostgreSQLVersionError,
     PostgreSQLListUsersError,
+    PostgreSQLSetUpDDLRoleError,
 )
 from ops.charm import CharmBase, RelationBrokenEvent, RelationChangedEvent
 from ops.framework import Object
@@ -102,17 +103,24 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None:
         extra_user_roles = self._sanitize_extra_roles(event.extra_user_roles)
         extra_user_roles.append(ACCESS_GROUP_RELATION)
 
+        if self.check_for_invalid_extra_user_roles(event.relation):
+            self.charm.unit.status = BlockedStatus(INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE)
+            return
+
         try:
             # Creates the user and the database for this specific relation.
             user = f"relation-{event.relation.id}"
             password = new_password()
-            self.charm.postgresql.create_user(user, password, extra_user_roles=extra_user_roles)
-            plugins = self.charm.get_plugins()
 
-            self.charm.postgresql.create_database(
-                database, user, plugins=plugins, client_relations=self.charm.client_relations
+            database_created = self.charm.postgresql.create_database(database)
+
+            self.charm.postgresql.create_user(
+                user, password, roles=[*extra_user_roles, f"{database}_admin"], database=database
             )
 
+            if database_created:
+                self.charm.update_config()
+
             # Share the credentials with the application.
             self.database_provides.set_credentials(event.relation.id, user, password)
 
@@ -132,12 +140,13 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None:
             PostgreSQLCreateDatabaseError,
             PostgreSQLCreateUserError,
             PostgreSQLGetPostgreSQLVersionError,
+            PostgreSQLSetUpDDLRoleError,
         ) as e:
             logger.exception(e)
             self.charm.unit.status = BlockedStatus(
                 e.message
                 if issubclass(type(e), PostgreSQLCreateUserError) and e.message is not None
-                else f"Failed to initialize {self.relation_name} relation"
+                else f"Failed to initialize relation {self.relation_name}"
             )
 
     def _on_relation_broken(self, event: RelationBrokenEvent) -> None:
@@ -273,6 +282,12 @@ def _update_unit_status(self, relation: Relation) -> None:
         ) and not self.check_for_invalid_extra_user_roles(relation.id):
             self.charm.unit.status = ActiveStatus()
 
+        if (
+            self.charm.is_blocked
+            and "Failed to initialize relation" in self.charm.unit.status.message
+        ):
+            self.charm.unit.status = ActiveStatus()
+
         self._update_unit_status_on_blocking_endpoint_simultaneously()
 
     def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
@@ -300,17 +315,20 @@ def check_for_invalid_extra_user_roles(self, relation_id: int) -> bool:
         Args:
             relation_id: current relation to be skipped.
         """
-        valid_privileges, valid_roles = self.charm.postgresql.list_valid_privileges_and_roles()
+        valid_roles = [
+            *self.charm.postgresql.list_roles(),
+            "pgbouncer",
+            "admin",
+            "createdb",
+            "createrole",
+        ]
         for relation in self.charm.model.relations.get(self.relation_name, []):
             if relation.id == relation_id:
                 continue
             for data in relation.data.values():
                 extra_user_roles = data.get("extra-user-roles")
                 extra_user_roles = self._sanitize_extra_roles(extra_user_roles)
                 for extra_user_role in extra_user_roles:
-                    if (
-                        extra_user_role not in valid_privileges
-                        and extra_user_role not in valid_roles
-                    ):
+                    if extra_user_role not in valid_roles:
                         return True
         return False