Implemented bug fixes + added new tests (create 500 vdisks from a template)

Author: jeroenmaelbrancke

helpers/backend.py

@@ -183,7 +183,7 @@ def get_preset_by_albabackend(preset_name, albabackend_name):
         """
         Fetches preset by albabackend_guid
 
-        :param preset_name: name of a existing preset
+        :param preset_name: name of an existing preset
         :type preset_name: str
         :param albabackend_name: name of a existing alba backend
         :type albabackend_name: str

helpers/ci_constants.py

@@ -42,7 +42,8 @@ def __get__(self, cls, owner):
     def api(cls):
         return OVSClient(cls.SETUP_CFG['ci']['grid_ip'],
                          cls.SETUP_CFG['ci']['user']['api']['username'],
-                         cls.SETUP_CFG['ci']['user']['api']['password'])
+                         cls.SETUP_CFG['ci']['user']['api']['password'],
+                         )
 
     @classmethod
     def get_vpool_names(cls):

helpers/service.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2016 iNuron NV
+#
+# This file is part of Open vStorage Open Source Edition (OSE),
+# as available from
+#
+#      http://www.openvstorage.org and
+#      http://www.openvstorage.com.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3)
+# as published by the Free Software Foundation, in version 3 as it comes
+# in the LICENSE.txt file of the Open vStorage OSE distribution.
+#
+# Open vStorage is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY of any kind.
+
+from ovs.dal.hybrids.service import Service
+from ovs.extensions.generic.sshclient import SSHClient
+from ovs.extensions.services.servicefactory import ServiceFactory
+from ovs.log.log_handler import LogHandler
+from ..helpers.ci_constants import CIConstants
+
+
+class ServiceHelper(CIConstants):
+    """
+    ServiceHelper class
+    """
+
+    LOGGER = LogHandler.get(source="setup", name="ci_service_setup")
+    SERVICE_MANAGER = ServiceFactory.get_manager()
+
+    @staticmethod
+    def get_service(guid):
+        """
+        Fetch Service object by service guid
+        :param guid: guid of the service
+        :return: a Service object
+        """
+        return Service(guid)
+
+    @staticmethod
+    def restart_service(guid, storagerouter):
+        """
+        Restart a service based on the guid located on storagerouter
+        :param guid: guid of the service
+        :param storagerouter: storagerouter where the service is running
+        :type storagerouter: StorageRouter
+        :return:
+        """
+        client = SSHClient(storagerouter, username='root')
+        service = ServiceHelper.get_service(guid)
+        return ServiceHelper.SERVICE_MANAGER.restart_service(service.name, client=client)

helpers/vdisk.py

@@ -13,14 +13,16 @@
 #
 # Open vStorage is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY of any kind.
-from ovs.log.log_handler import LogHandler
+
 from ovs.dal.hybrids.vdisk import VDisk
 from ovs.dal.lists.vdisklist import VDiskList
 from ovs.dal.lists.vpoollist import VPoolList
+from ovs.log.log_handler import LogHandler
+from ..helpers.ci_constants import CIConstants
 from ..helpers.exceptions import VPoolNotFoundError, VDiskNotFoundError
 
 
-class VDiskHelper(object):
+class VDiskHelper(CIConstants):
     """
     vDiskHelper class
     """
@@ -105,8 +107,8 @@ def get_snapshot_by_guid(snapshot_guid, vdisk_name, vpool_name):
             raise RuntimeError("Did not find snapshot with guid `{0}` on vdisk `{1}` on vpool `{2}`"
                                .format(snapshot_guid, vdisk_name, vpool_name))
 
-    @staticmethod
-    def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIMEOUT):
+    @classmethod
+    def get_config_params(cls, vdisk_name, vpool_name, timeout=GET_CONFIG_PARAMS_TIMEOUT, *args, **kwargs):
         """
         Fetch the config parameters of a vDisk
 
@@ -115,8 +117,6 @@ def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIM
         :type vdisk_name: str
         :param vpool_name: name of a existing vpool
         :type vpool_name: str
-        :param api: specify a valid api connection to the setup
-        :type api: ci.helpers.api.OVSClient
         :param timeout: time to wait for the task to complete
         :type timeout: int
         :return: a dict with config parameters, e.g.
@@ -133,8 +133,8 @@ def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIM
         """
 
         vdisk = VDiskHelper.get_vdisk_by_name(vdisk_name=vdisk_name, vpool_name=vpool_name)
-        task_guid = api.get(api='/vdisks/{0}/get_config_params'.format(vdisk.guid))
-        task_result = api.wait_for_task(task_id=task_guid, timeout=timeout)
+        task_guid = cls.api.get(api='/vdisks/{0}/get_config_params'.format(vdisk.guid))
+        task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout)
 
         if not task_result[0]:
             error_msg = "Setting config vDisk `{0}` has failed with error {1}".format(vdisk_name, task_result[1])
@@ -144,23 +144,21 @@ def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIM
             VDiskHelper.LOGGER.info("Setting config vDisk `{0}` should have succeeded".format(vdisk_name))
             return task_result[1]
 
-    @staticmethod
-    def scrub_vdisk(vdisk_guid, api, timeout=15 * 60, wait=True):
+    @classmethod
+    def scrub_vdisk(cls, vdisk_guid, timeout=15 * 60, wait=True, *args, **kwargs):
         """
         Scrub a specific vdisk
         :param vdisk_guid: guid of the vdisk to scrub
         :type vdisk_guid: str
-        :param api: specify a valid api connection to the setup
-        :type api: ci.helpers.api.OVSClient
         :param timeout: time to wait for the task to complete
         :type timeout: int
         :param wait: wait for task to finish or not
         :type wait: bool
-        :return: 
+        :return:
         """
-        task_guid = api.post(api='/vdisks/{0}/scrub/'.format(vdisk_guid), data={})
+        task_guid = cls.api.post(api='/vdisks/{0}/scrub/'.format(vdisk_guid), data={})
         if wait is True:
-            task_result = api.wait_for_task(task_id=task_guid, timeout=timeout)
+            task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout)
             if not task_result[0]:
                 error_msg = "Scrubbing vDisk `{0}` has failed with error {1}".format(vdisk_guid, task_result[1])
                 VDiskHelper.LOGGER.error(error_msg)
@@ -170,3 +168,24 @@ def scrub_vdisk(vdisk_guid, api, timeout=15 * 60, wait=True):
                 return task_result[1]
         else:
             return task_guid
+
+    @classmethod
+    def delete_vdisk(cls, vdisk_guid, timeout=GET_CONFIG_PARAMS_TIMEOUT):
+        """
+        Delete a specific vdisk
+        :param vdisk_guid: guid of the vdisk to delete
+        :type vdisk_guid: str
+        :param timeout: time to wait for the task to complete
+        :type timeout: int
+        :return: bool
+        """
+        task_guid = cls.api.delete(api='/vdisks/{0}'.format(vdisk_guid))
+        task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout)
+
+        if not task_result[0]:
+            error_msg = "Deleting of vDisk `{0}` has failed with error {1}".format(vdisk_guid, task_result[1])
+            VDiskHelper.LOGGER.error(error_msg)
+            raise RuntimeError(error_msg)
+        else:
+            VDiskHelper.LOGGER.info("Deleting of vDisk `{0}` should have succeeded".format(vdisk_guid))
+            return True

remove/vdisk.py

@@ -75,7 +75,7 @@ def remove_snapshot(cls, snapshot_guid, vdisk_name, vpool_name, timeout=REMOVE_S
         task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout)
 
         if not task_result[0]:
-            error_msg = "Deleting snapshot `{0}` for vdisk `{1}` has failed".format(snapshot_guid, vdisk_name)
+            error_msg = "Task `{0}`: deleting snapshot `{1}` for vdisk `{2}` has failed".format(task_guid, snapshot_guid, vdisk_name)
             VDiskRemover.LOGGER.error(error_msg)
             raise RuntimeError(error_msg)
         else:
@@ -97,11 +97,11 @@ def remove_vdisk(cls, vdisk_guid, timeout=REMOVE_VDISK_TIMEOUT, *args, **kwargs)
         task_guid = cls.api.post(api='vdisks/{0}/delete'.format(vdisk_guid))
         task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout)
         if not task_result[0]:
-            error_msg = "Deleting vDisk `{0}` has failed".format(vdisk_guid)
+            error_msg = "Task `{0}`: deleting vDisk `{1}` has failed".format(task_guid, vdisk_guid)
             VDiskRemover.LOGGER.error(error_msg)
             raise RuntimeError(error_msg)
         else:
-            VDiskRemover.LOGGER.info("Deleting vDisk `{0}` should have succeeded".format(vdisk_guid))
+            VDiskRemover.LOGGER.info("Task `{0}`: deleting vDisk `{1}` should have succeeded".format(task_guid, vdisk_guid))
             return True
 
     @classmethod
@@ -139,7 +139,7 @@ def remove_vtemplate_by_name(cls, vdisk_name, vpool_name, timeout=REMOVE_VTEMPLA
         task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout)
 
         if not task_result[0]:
-            error_msg = "Deleting vTemplate `{0}` has failed".format(vdisk_name)
+            error_msg = "Task `{0}`: deleting vTemplate `{1}` has failed".format(task_guid, vdisk_name)
             VDiskRemover.LOGGER.error(error_msg)
             raise RuntimeError(error_msg)
         else:

setup/arakoon.py

@@ -77,7 +77,10 @@ def add_arakoon(cluster_name, storagerouter_ip, cluster_basedir,
                                          base_dir=cluster_basedir,
                                          plugins=plugins,
                                          locked=False,
-                                         internal=False)
+                                         internal=False,
+                                         log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server'),
+                                         crash_log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server_crash')
+                                         )
         if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM:
             client.run(['ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)])
         elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM:
@@ -128,7 +131,9 @@ def extend_arakoon(cluster_name, master_storagerouter_ip, storagerouter_ip, clus
         arakoon_installer.load()
         arakoon_installer.extend_cluster(new_ip=storagerouter_ip,
                                          base_dir=cluster_basedir,
-                                         locked=False)
+                                         locked=False,
+                                         log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server'),
+                                         crash_log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server_crash'))
         if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM:
             client.run(['ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)])
         elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM:
@@ -194,6 +199,5 @@ def setup_external_arakoons(backend):
                                                     clustered_nodes=external_arakoon_mapping[arakoon_name]['all'])
             return external_arakoon_mapping
         else:
-            ArakoonSetup.LOGGER.info("Skipping external arakoon creation because backend `{0}` already exists"
-                                 .format(backend['name']))
+            ArakoonSetup.LOGGER.info("Skipping external arakoon creation because backend `{0}` already exists".format(backend['name']))
             return

setup/backend.py

@@ -18,8 +18,8 @@
 from ..helpers.albanode import AlbaNodeHelper
 from ..helpers.backend import BackendHelper
 from ..helpers.ci_constants import CIConstants
-from ..validate.decorators import required_roles, required_backend, required_preset, check_backend, check_preset, \
-    check_linked_backend, filter_osds
+from ..helpers.exceptions import PresetNotFoundError
+from ..validate.decorators import required_roles, required_backend, required_preset, check_backend, check_linked_backend, filter_osds
 
 
 class BackendSetup(CIConstants):
@@ -95,7 +95,6 @@ def add_backend(cls, backend_name, scaling='LOCAL', timeout=BACKEND_TIMEOUT, max
         return False
 
     @classmethod
-    @check_preset
     @required_backend
     def add_preset(cls, albabackend_name, preset_details, timeout=ADD_PRESET_TIMEOUT, *args, **kwargs):
         """
@@ -120,6 +119,15 @@ def add_preset(cls, albabackend_name, preset_details, timeout=ADD_PRESET_TIMEOUT
         :return: success or not
         :rtype: bool
         """
+
+        # CHECK PRESET
+        try:
+            BackendHelper.get_preset_by_albabackend(preset_details['name'], albabackend_name)
+            BackendSetup.LOGGER.info("Preset `{0}` already exists.".format(preset_details['name']))
+            return True
+        except PresetNotFoundError:
+            pass
+
         # BUILD_PRESET
         preset = {'name': preset_details['name'],
                   'policies': preset_details['policies'],

setup/vdisk.py

@@ -290,11 +290,11 @@ def create_from_template(cls, vdisk_name, vpool_name, new_vdisk_name, storagerou
         task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout)
 
         if not task_result[0]:
-            error_msg = "Creating vTemplate `{0}` has failed with error {1}".format(vdisk_name, task_result[1])
+            error_msg = "Creating `{0}` from vTemplate `{1}` has failed with error {2}".format(official_new_vdisk_name, vdisk_name, task_result[1])
             VDiskSetup.LOGGER.error(error_msg)
             raise RuntimeError(error_msg)
         else:
-            VDiskSetup.LOGGER.info("Creating vTemplate `{0}` should have succeeded".format(vdisk_name))
+            VDiskSetup.LOGGER.info("Creating `{0}` from vTemplate `{1}` should have succeeded".format(official_new_vdisk_name, vdisk_name))
             return task_result[1]
 
     @classmethod