From 7d30c9433e90faecf3eb1a46e2fb59e538d51cb7 Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Tue, 8 May 2018 17:36:49 +0200 Subject: [PATCH 1/5] Added new structure for easy access to Testrail objects (Projects, Test cases, results, tests,...) --- testrail/__init__.py | 15 ++ testrail/client.py | 196 +++++++++++++++++++++ testrail/containers/__init__.py | 15 ++ testrail/containers/case.py | 71 ++++++++ testrail/containers/milestone.py | 65 +++++++ testrail/containers/project.py | 53 ++++++ testrail/containers/result.py | 70 ++++++++ testrail/containers/run.py | 73 ++++++++ testrail/containers/section.py | 61 +++++++ testrail/containers/suite.py | 54 ++++++ testrail/containers/test.py | 52 ++++++ testrail/lists/__init__.py | 15 ++ testrail/lists/caselist.py | 43 +++++ testrail/lists/milestonelist.py | 39 +++++ testrail/lists/projectlist.py | 33 ++++ testrail/lists/runlist.py | 39 +++++ testrail/lists/sectionlist.py | 41 +++++ testrail/lists/suitelist.py | 39 +++++ testrail/lists/testlist.py | 32 ++++ testrail/testraillist.py | 183 +++++++++++++++++++ testrail/testrailobject.py | 289 +++++++++++++++++++++++++++++++ 21 files changed, 1478 insertions(+) create mode 100644 testrail/__init__.py create mode 100644 testrail/client.py create mode 100644 testrail/containers/__init__.py create mode 100644 testrail/containers/case.py create mode 100644 testrail/containers/milestone.py create mode 100644 testrail/containers/project.py create mode 100644 testrail/containers/result.py create mode 100644 testrail/containers/run.py create mode 100644 testrail/containers/section.py create mode 100644 testrail/containers/suite.py create mode 100644 testrail/containers/test.py create mode 100644 testrail/lists/__init__.py create mode 100644 testrail/lists/caselist.py create mode 100644 testrail/lists/milestonelist.py create mode 100644 testrail/lists/projectlist.py create mode 100644 testrail/lists/runlist.py create mode 100644 testrail/lists/sectionlist.py create mode 100644 testrail/lists/suitelist.py create mode 100644 testrail/lists/testlist.py create mode 100644 testrail/testraillist.py create mode 100644 testrail/testrailobject.py diff --git a/testrail/__init__.py b/testrail/__init__.py new file mode 100644 index 0000000..210476c --- /dev/null +++ b/testrail/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 2018 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. diff --git a/testrail/client.py b/testrail/client.py new file mode 100644 index 0000000..44c058c --- /dev/null +++ b/testrail/client.py @@ -0,0 +1,196 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +API client module. +- APIClient: An API client. The client has been changed to work with testrail auth keys +- APIError: Exception thrown by the API client +""" +import re +import sys +import json +import base64 +import urllib2 + + +class APIError(Exception): + pass + + +class APIClient(object): + def __init__(self, base_url, username=None, password=None, key=None): + assert (username and password) or key, "Credentials are needed for testrail connection, specify either user/password or basic auth key." + self.user = username + self.password = password + self.key = key + if not base_url.endswith('/'): + base_url += '/' + if not base_url.startswith('http://'): + base_url = 'http://{0}'.format(base_url) + self._url = base_url + 'index.php?/api/v2/' + + def send_get(self, uri): + """ + Issues a GET request (read) against the API and returns the result + :param uri: The API method to call including parameters (e.g. get_case/1) + :return: result of the get + :rtype: dict + """ + return self._send_request('GET', uri, None) + + def send_post(self, uri, data): + """ + Issues a POST request (write) against the API and returns the result + :param uri: The API method to call including parameters (e.g. add_case/1) + :type uri: str + :param data: The data to submit as part of the request (as Python dict, strings must be UTF-8 encoded) + :type data: dict + :return: result of the post + :rtype: dict + """ + return self._send_request('POST', uri, data) + + def _send_request(self, method, uri, data): + """ + Sends a request to the uri + :param method: request method (eg GET) + :param uri: The API method to call including parameters (e.g. add_case/1) + :param data: + :return: Result of the call + :rtype: dict + """ + url = self._url + uri + request = urllib2.Request(url) + if method == 'POST': + request.add_data(json.dumps(data)) + if self.key is not None: + auth = self.key + else: + auth = base64.b64encode('{0}:{1}'.format(self.user, self.password)) + request.add_header('Authorization', 'Basic {0}'.format(auth)) + request.add_header('Content-Type', 'application/json') + error = None + try: + response = urllib2.urlopen(request).read() + except urllib2.HTTPError as error: + response = error.read() + if response: + result = json.loads(response) + else: + result = {} + if error is not None: + if result and 'error' in result: + error_message = '"' + result['error'] + '"' + else: + error_message = 'No additional error message received' + raise APIError('TestRail API returned HTTP {0} ({1})'.format(error.code, error_message)) + return result + + @staticmethod + def _check_type(value, required_type): + """ + Validates whether a certain value is of a given type. Some types are treated as special + case: + - A 'str' type accepts 'str', 'unicode' and 'basestring' + - A 'float' type accepts 'float', 'int' + - A list instance acts like an enum + """ + given_type = type(value) + if required_type is str: + correct = isinstance(value, basestring) or value is None + allowed_types = ['str', 'unicode', 'basestring'] + elif required_type is float: + correct = isinstance(value, float) or isinstance(value, int) or value is None + allowed_types = ['float', 'int'] + elif required_type is int or required_type is long: + correct = isinstance(value, int) or isinstance(value, long) or value is None + allowed_types = ['int', 'long'] + elif isinstance(required_type, list): + # We're in an enum scenario. Field_type isn't a real type, but a list containing + # all possible enum values. Here as well, we need to do some str/unicode/basestring + # checking. + if isinstance(required_type[0], basestring): + value = str(value) + correct = value in required_type + allowed_types = required_type + given_type = value + else: + correct = isinstance(value, required_type) or value is None + allowed_types = [required_type.__name__] + return correct, allowed_types, given_type + + @classmethod + def verify_params(cls, given_params, param_mapping): + """ + Verifies parameters based on a given mapping + :param given_params: given params + :type given_params: dict + :param param_mapping: param mapping + :type param_mapping: dict + :return: None + :rtype: NoneType + """ + # @TODO replace with toolbox verify params after testing + if not isinstance(given_params, dict) or not isinstance(param_mapping, dict): + raise ValueError('Arguments are incorrect. Both arguments need to be of type {0}'.format(dict)) + error_messages = [] + compiled_regex_type = type(re.compile('some_regex')) + for required_key, key_info in param_mapping.iteritems(): + expected_type = key_info[0] + expected_value = key_info[1] + optional = len(key_info) == 3 and key_info[2] is False + + if optional is True and (required_key not in given_params or given_params[required_key] in ('', None)): + continue + + if required_key not in given_params: + error_messages.append('Missing required param "{0}" in actual parameters'.format(required_key)) + continue + + mandatory_or_optional = 'Optional' if optional is True else 'Mandatory' + actual_value = given_params[required_key] + if cls._check_type(actual_value, expected_type)[0] is False: + error_messages.append( + '{0} param "{1}" is of type "{2}" but we expected type "{3}"'.format(mandatory_or_optional, required_key, type(actual_value), expected_type)) + continue + if expected_value is None: + continue + + if expected_type == list: + if type(expected_value) == compiled_regex_type: # List of strings which need to match regex + for item in actual_value: + if not re.match(expected_value, item): + error_messages.append( + '{0} param "{1}" has an item "{2}" which does not match regex "{3}"'.format( + mandatory_or_optional, required_key, item, expected_value.pattern)) + elif expected_type == dict: + cls.verify_params(expected_value, given_params[required_key]) + elif expected_type == int or expected_type == float: + if isinstance(expected_value, list) and actual_value not in expected_value: + error_messages.append('{0} param "{1}" with value "{2}" should be 1 of the following: {3}'.format(mandatory_or_optional, required_key, actual_value, expected_value)) + if isinstance(expected_value, dict): + minimum = expected_value.get('min', sys.maxint * -1) + maximum = expected_value.get('max', sys.maxint) + if not minimum <= actual_value <= maximum: + error_messages.append('{0} param "{1}" with value "{2}" should be in range: {3} - {4}'.format(mandatory_or_optional, required_key, actual_value, minimum, maximum)) + else: + if cls._check_type(expected_value, list)[0] is True and actual_value not in expected_value: + error_messages.append('{0} param "{1}" with value "{2}" should be 1 of the following: {3}' + .format(mandatory_or_optional, required_key, actual_value, expected_value)) + elif cls._check_type(expected_value, compiled_regex_type)[0] is True and not re.match(expected_value, actual_value): + error_messages.append('{0} param "{1}" with value "{2}" does not match regex "{3}"'.format(mandatory_or_optional, required_key, actual_value, expected_value.pattern)) + if error_messages: + raise RuntimeError('\n' + '\n'.join(error_messages)) diff --git a/testrail/containers/__init__.py b/testrail/containers/__init__.py new file mode 100644 index 0000000..210476c --- /dev/null +++ b/testrail/containers/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 2018 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. diff --git a/testrail/containers/case.py b/testrail/containers/case.py new file mode 100644 index 0000000..a9ecd0d --- /dev/null +++ b/testrail/containers/case.py @@ -0,0 +1,71 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Case Module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Case(TestRailObject): + + _VERIFY_PARAMS = {'section_id': (int, None), + 'title': (str, None), + 'template_id': (int, None, False), + 'type_id': (int, None, False), + 'priority_id': (int, None, False), + 'estimate': (str, None, False), + 'milestone_id': (int, None, False), + 'refs': (str, None, False)} + + def __init__(self, id=None, section_id=None, title=None, template_id=None, type_id=None, priority_id=None, estimate=None, + milestone_id=None, refs=None, client=None, *args, **kwargs): + """ + Testrail Case + :param id: ID of the Section + :type id: int + :param section_id: The ID of the section the test case should be added to (required) + :type section_id: int + :param title: The title of the test case (required) + :type title: str + :param template_id: The ID of the template (field layout) (requires TestRail 5.2 or later) + :type template_id: int + :param type_id: The ID of the case type + :type type_id: int + :param priority_id: The ID of the case priority + :type priority_id: int + :param estimate: The estimate, e.g. "30s" or "1m 45s" + :type estimate: str + :param milestone_id: The ID of the milestone to link to the test case + :type milestone_id: int + :param refs: A comma-separated list of references/requirements + :type refs: str + """ + self.title = title + self.estimate = estimate + self.refs = refs + + # Relation + self.section_id = section_id + self.milestone_id = milestone_id + self.priority_id = priority_id + self.type_id = type_id + self.template_id = template_id + + super(Case, self).__init__(id=id, client=client, *args, **kwargs) + + def _get_url_add(self): + return '{0}/{1}'.format(super(Case, self)._get_url_add(), self.section_id) diff --git a/testrail/containers/milestone.py b/testrail/containers/milestone.py new file mode 100644 index 0000000..4d0c3b8 --- /dev/null +++ b/testrail/containers/milestone.py @@ -0,0 +1,65 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Milestone(TestRailObject): + + _VERIFY_PARAMS = {'description': (str, None, False), + 'name': (str, None), + 'due_on': (int, None, False), + 'parent_id': (int, None, False), + 'start_on': (int, None, False)} + + def __init__(self, id=None, name=None, description=None, project_id=None, parent_id=None, due_on=None, start_on=None, client=None, *args, **kwargs): + """ + Testrail Milestone + :param id: id of the milestone + :type id: int + :param name: The name of the milestone + :type name: str + :param description: The description of the milestone + :type description: str + :param project_id: The ID of the project + :type project_id: int + :param parent_id: The ID of the parent milestone + :type parent_id: int + :param due_on: The due date of the milestone (as Unix timestamp) + :type due_on: int + :param start_on: The scheduled start date of the milestone (as Unix timestamp) + :type start_on: int + """ + self.name = name + self.description = description + self.due_on = due_on + self.start_on = start_on + + # Relation + self.project_id = project_id + self.parent_id = parent_id + + super(Milestone, self).__init__(id=id, client=client, *args, **kwargs) + + def _get_url_add(self): + return '{0}/{1}'.format(super(Milestone, self)._get_url_add(), self.project_id) + + def _validate_data_update(self, data): + verify_params = dict(item for item in self._VERIFY_PARAMS.iteritems() if item[0] != 'project_id') + return self._client.verify_params(data, verify_params) diff --git a/testrail/containers/project.py b/testrail/containers/project.py new file mode 100644 index 0000000..59d905e --- /dev/null +++ b/testrail/containers/project.py @@ -0,0 +1,53 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" + +from ci.testrail.testrailobject import TestRailObject + + +class Project(TestRailObject): + _VERIFY_PARAMS = {'name': (str, None), + 'announcement': (str, None, False), + 'show_announcement': (bool, None, False), + 'is_completed': (bool, None, False)} + + def __init__(self, id=None, name=None, announcement=None, show_announcement=None, suite_mode=None, is_completed=None, client=None, *args, **kwargs): + """ + Testrail project + :param id: id of the project + :type id: int + :param name: The name of the project + :type name: str + :param announcement: The description of the project + :type announcement: str + :param show_announcement: True if the announcement should be displayed on the project's overview page and false otherwise + :type show_announcement: bool + :param is_completed: Specifies whether a project is considered completed or not + :type is_completed: bool + :param suite_mode: The suite mode of the project (1 for single suite mode, 2 for single suite + baselines, 3 for multiple suites) (added with TestRail 4.0) + :type suite_mode: int + """ + self.name = name + self.announcement = announcement + self.show_announcement = show_announcement + self.suite_mode = suite_mode + self.is_completed = is_completed + + super(Project, self).__init__(id=id, client=client, *args, **kwargs) + diff --git a/testrail/containers/result.py b/testrail/containers/result.py new file mode 100644 index 0000000..f0bcf54 --- /dev/null +++ b/testrail/containers/result.py @@ -0,0 +1,70 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Result(TestRailObject): + + _VERIFY_PARAMS = {'status_id': (int, None), + 'comment': (str, None, False), + 'elapsed': (str, None, False), + 'version': (str, None), + 'defects': (str, None), + 'assignedto_id': (int, None, False)} + + def __init__(self, id=None, test_id=None, status_id=None, comment=None, version=None, elapsed=None, defects=None, assignedto_id=None, client=None, *args, **kwargs): + """ + Testrail Result + :param id: id of the result + :type id: int + :param test_id: The id of the test the result should be added to + :type test_id: int + :param status_id: The id of test status: 1 passed, 2 blocked, 3 untested, 4 retest, 5 failed + :type status_id: int + :param comment: The comment/description of the test result + :type comment: str + :param version: The version or build you tested against + :type version: str + :param elapsed: The time it took to execute the test e.g. 30s or 1m 45s + :type elapsed: str + :param defects: A comma-separated list of defects to link to the test result + :type defects: str + :param assignedto_id: The id of a user the test should be assigned to + :type assignedto_id: int + """ + + self.comment = comment + self.version = version + self.elapsed = elapsed + self.defects = defects + + # Relation + self.test_id = test_id + self.status_id = status_id + self.assignedto_id = assignedto_id + + super(Result, self).__init__(id=id, client=client, *args, **kwargs) + + def _get_url_add(self): + return '{0}/{1}'.format(super(Result, self)._get_url_add(), self.test_id) + + def _validate_data_update(self, data): + verify_params = dict(item for item in self._VERIFY_PARAMS.iteritems() if item[0] != 'test_id') + return self._client.verify_params(data, verify_params) diff --git a/testrail/containers/run.py b/testrail/containers/run.py new file mode 100644 index 0000000..8044c60 --- /dev/null +++ b/testrail/containers/run.py @@ -0,0 +1,73 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Run(TestRailObject): + + _VERIFY_PARAMS = {'description': (str, None, False), + 'name': (str, None), + 'suite_id': (int, None, False), + 'milestone_id': (int, None), + 'assignedto_on': (int, None, False), + 'include_all': (bool, None, False), + 'case_ids': (list, None, False)} + + def __init__(self, id=None, name=None, description=None, suite_id=None, project_id=None, milestone_id=None, assignedto_on=None, include_all=None, case_ids=None, client=None, *args, **kwargs): + """ + Testrail Run + :param id: id of the run + :type id: int + :param name: The name of the run + :type name: str + :param description: The description of the run + :type description: str + :param suite_id: The id of the test suite + :type suite_id: int + :param project_id: The id of the project + :type project_id: int + :param milestone_id: The id of the milestone + :type milestone_id: int + :param assignedto_on: The id of the user the test run should be assigned to + :type assignedto_on: int + :param include_all: True for including all test cases of the test suite and false for a custom case selection + :type include_all: bool + :param case_ids: An array of case ID of the custom case selection + :type case_ids: list + """ + self.name = name + self.description = description + self.assignedto_on = assignedto_on + self.include_all = include_all + + # Relation + self.project_id = project_id + self.suite_id = suite_id + self.milestone_id = milestone_id + self.case_ids = case_ids + + super(Run, self).__init__(id=id, client=client, *args, **kwargs) + + def _get_url_add(self): + return '{0}/{1}'.format(super(Run, self)._get_url_add(), self.project_id) + + def _validate_data_update(self, data): + verify_params = dict(item for item in self._VERIFY_PARAMS.iteritems() if item[0] != 'project_id') + return self._client.verify_params(data, verify_params) diff --git a/testrail/containers/section.py b/testrail/containers/section.py new file mode 100644 index 0000000..a22a19b --- /dev/null +++ b/testrail/containers/section.py @@ -0,0 +1,61 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Section Module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Section(TestRailObject): + + _VERIFY_PARAMS = {'description': (str, None, False), + 'suite_id': (int, None, False), + 'parent_id': (int, None, False), + 'project_id': (int, None), + 'name': (str, None)} + + def __init__(self, id=None, project_id=None, description=None, suite_id=None, parent_id=None, name=None, client=None, *args, **kwargs): + """ + Testrail Suite + :param id: ID of the Section + :type id: int + :param project_id: The ID of the project + :param description: The description of the section (added with TestRail 4.0) + :type description: str + :param suite_id: The ID of the test suite (ignored if the project is operating in single suite mode, required otherwise) + :type suite_id: int + :param parent_id: The ID of the parent section (to build section hierarchies) + :type parent_id: int + :param name: The name of the Section (required) + :type name: str + """ + self.name = name + self.description = description + + # Relation + self.project_id = project_id + self.suite_id = suite_id + self.parent_id = parent_id + + super(Section, self).__init__(id=id, client=client, *args, **kwargs) + + def _get_url_add(self): + return '{0}/{1}'.format(super(Section, self)._get_url_add(), self.project_id) + + def _validate_data_update(self, data): + verify_params = dict(item for item in self._VERIFY_PARAMS.iteritems() if item[0] in ['name', 'description']) + return self._client.verify_params(data, verify_params) diff --git a/testrail/containers/suite.py b/testrail/containers/suite.py new file mode 100644 index 0000000..a59eac2 --- /dev/null +++ b/testrail/containers/suite.py @@ -0,0 +1,54 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Suite(TestRailObject): + + _VERIFY_PARAMS = {'description': (str, None, False), + 'name': (str, None), + 'project_id': (int, None)} + + def __init__(self, id=None, project_id=None, name=None, description=None, client=None, *args, **kwargs): + """ + Testrail Suite + :param id: id of the suite + :type id: int + :param description: The description of the test suite + :type description: str + :param project_id: ID of the project to link the suite + :type project_id: int + :param name: The name of the test suite + :type name: str + """ + self.name = name + self.description = description + + # Relation + self.project_id = project_id + + super(Suite, self).__init__(id=id, client=client, *args, **kwargs) + + def _get_url_add(self): + return '{0}/{1}'.format(super(Suite, self)._get_url_add(), self.project_id) + + def _validate_data_update(self, data): + verify_params = dict(item for item in self._VERIFY_PARAMS.iteritems() if item[0] != 'project_id') + return self._client.verify_params(data, verify_params) diff --git a/testrail/containers/test.py b/testrail/containers/test.py new file mode 100644 index 0000000..9121d76 --- /dev/null +++ b/testrail/containers/test.py @@ -0,0 +1,52 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Test(TestRailObject): + + _VERIFY_PARAMS = {'run_id': (int, None), + 'title': (str, None)} + + def __init__(self, id=None, run_id=None, title=None, client=None, *args, **kwargs): + """ + Testrail Test + :param id: Id of the test + :type id: int + :param run_id: The id of the test run + :type run_id: int + :param title: The title of test + :type title: str + """ + + self.title = title + + # Relation + self.run_id = run_id + + super(Test, self).__init__(id=id, client=client, *args, **kwargs) + + def save(self): + """ + Saves the current object on the backend + :return: None + :rtype: NoneType + """ + raise NotImplemented('Tests cannot be created. Only listed') diff --git a/testrail/lists/__init__.py b/testrail/lists/__init__.py new file mode 100644 index 0000000..210476c --- /dev/null +++ b/testrail/lists/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 2018 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. diff --git a/testrail/lists/caselist.py b/testrail/lists/caselist.py new file mode 100644 index 0000000..0ffcb5f --- /dev/null +++ b/testrail/lists/caselist.py @@ -0,0 +1,43 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.case import Case + + +class CaseList(TestRailList): + def __init__(self, project_id, suite_id, section_id, client, *args, **kwargs): + """ + Initializes a CaseList + :param project_id: ID of the Project to list cases for + :type project_id: int + :param suite_id: ID of the Suite to list cases for + :type suite_id: int + :param section_id: ID of the Section to list cases for + :type section_id: int + """ + super(CaseList, self).__init__(client=client, object_type=Case, plural='cases', *args, **kwargs) + self.load_url = self._generate_url(project_id, 'suite_id={0}'.format(suite_id), 'section_id={0}'.format(section_id)) + + def get_case_by_name(self, case_name): + for case in self.load(): + if case.title == case_name: + return case + + raise LookupError('Case `{0}` not found.'.format(case_name)) diff --git a/testrail/lists/milestonelist.py b/testrail/lists/milestonelist.py new file mode 100644 index 0000000..597e7eb --- /dev/null +++ b/testrail/lists/milestonelist.py @@ -0,0 +1,39 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.milestone import Milestone + + +class MilestoneList(TestRailList): + def __init__(self, project_id, client, *args, **kwargs): + """ + Initializes a MilestoneLilst + :param project_id: ID of the Project to list milstones for + :type project_id: int + """ + super(MilestoneList, self).__init__(client=client, object_type=Milestone, plural='milestones', *args, **kwargs) + self.load_url = self._generate_url(project_id) + + def get_milestone_by_name(self, milestone_name): + for milestone in self.load(): + if milestone.name == milestone_name: + return milestone + + raise LookupError('Milestone `{0}` not found.'.format(milestone_name)) diff --git a/testrail/lists/projectlist.py b/testrail/lists/projectlist.py new file mode 100644 index 0000000..757ff71 --- /dev/null +++ b/testrail/lists/projectlist.py @@ -0,0 +1,33 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.project import Project + + +class ProjectList(TestRailList): + def __init__(self, client, *args, **kwargs): + super(ProjectList, self).__init__(client=client, object_type=Project, plural='projects', *args, **kwargs) + + def get_project_by_name(self, project_name): + for project in self.load(): + if project.name == project_name: + return project + + raise LookupError('Project `{0}` not found.'.format(project_name)) diff --git a/testrail/lists/runlist.py b/testrail/lists/runlist.py new file mode 100644 index 0000000..d18073c --- /dev/null +++ b/testrail/lists/runlist.py @@ -0,0 +1,39 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.run import Run + + +class RunList(TestRailList): + def __init__(self, project_id, client, *args, **kwargs): + """ + Initializes a RunList + :param project_id: ID of the Project to list runs for + :type project_id: int + """ + super(RunList, self).__init__(client=client, object_type=Run, plural='runs', *args, **kwargs) + self.load_url = self._generate_url(project_id) + + def get_run_by_name(self, run_name): + for run in self.load(): + if run.name == run_name: + return run + + raise LookupError('Run `{0}` not found.'.format(run_name)) diff --git a/testrail/lists/sectionlist.py b/testrail/lists/sectionlist.py new file mode 100644 index 0000000..01a4e50 --- /dev/null +++ b/testrail/lists/sectionlist.py @@ -0,0 +1,41 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.section import Section + + +class SectionList(TestRailList): + def __init__(self, project_id, suite_id, client, *args, **kwargs): + """ + Initializes a SuitesList + :param project_id: ID of the Project to list sections for + :type project_id: int + :param suite_id: ID of the Suite to list sections for + :type suite_id: int + """ + super(SectionList, self).__init__(client=client, object_type=Section, plural='sections', *args, **kwargs) + self.load_url = self._generate_url(project_id, 'suite_id={0}'.format(suite_id)) + + def get_section_by_name(self, section_name): + for section in self.load(): + if section.name == section_name: + return section + + raise LookupError('Section `{0}` not found.'.format(section_name)) diff --git a/testrail/lists/suitelist.py b/testrail/lists/suitelist.py new file mode 100644 index 0000000..1d76d01 --- /dev/null +++ b/testrail/lists/suitelist.py @@ -0,0 +1,39 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.suite import Suite + + +class SuiteList(TestRailList): + def __init__(self, project_id, client, *args, **kwargs): + """ + Initializes a SuitesList + :param project_id: ID of the Project to list suites for + :type project_id: int + """ + super(SuiteList, self).__init__(client=client, object_type=Suite, plural='suites', *args, **kwargs) + self.load_url = self._generate_url(project_id) + + def get_suite_by_name(self, suite_name): + for suite in self.load(): + if suite.name == suite_name: + return suite + + raise LookupError('Suite `{0}` not found.'.format(suite_name)) diff --git a/testrail/lists/testlist.py b/testrail/lists/testlist.py new file mode 100644 index 0000000..8d77686 --- /dev/null +++ b/testrail/lists/testlist.py @@ -0,0 +1,32 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.test import Test + + +class TestList(TestRailList): + def __init__(self, run_id, client, *args, **kwargs): + """ + Initializes a TestList + :param run_id: ID of the Run to list tests for + :type run_id: int + """ + super(TestList, self).__init__(client=client, object_type=Test, plural='tests', *args, **kwargs) + self.load_url = self._generate_url(run_id) diff --git a/testrail/testraillist.py b/testrail/testraillist.py new file mode 100644 index 0000000..2b3a221 --- /dev/null +++ b/testrail/testraillist.py @@ -0,0 +1,183 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +TestRailList Module +""" +import operator +from ci.testrail.testrailobject import TestRailBase + + +class OperatorsMeta(type): + """ + Metaclass to allow Operators to remain fully static + """ + def __getattr__(cls, key): + """ + Overrule getattr to return the string represented in the map + """ + if key not in cls._map: + raise AttributeError('{0} does not have attribute \'{1}\''.format(str(cls), key)) + return key + + +class TestRailList(TestRailBase): + """ + TestRailList object. Serves as inheritable list which can load TestRailObjects + """ + # Embedding these inside the list for easier access + class Operators(object): + __metaclass__ = OperatorsMeta + # Map operators to functions for easier evaluation + _map = {'LT': operator.lt, + 'LE': operator.le, + 'EQ': operator.eq, + 'NE': operator.ne, + 'GE': operator.ge, + 'GT': operator.gt, + 'IN': lambda a, b: a in b} + + @classmethod + def get_operator(cls, key): + if key not in cls._map: + raise RuntimeError('Requested operator could not be found') + return cls._map[key] + + class WhereOperators(object): + AND = 'AND' + OR = 'OR' + + def __init__(self, object_type, client, plural, load_url=None, *args, **kwargs): + """ + :param object_type: Object Type to load (Example: ci.testrail.containers.project) + :param client: API Client + :param plural: Plural form of the object to load (used in url generation) (Eg. project -> projects) + :param load_url: The url to load the object from (Could differ when other ids need to be specified) + """ + super(TestRailList, self).__init__() + self._client = client + self._objects_data = [] + self.object_type = object_type + self.plural = plural + + # Could be set by inheritors (when project_id needs to be linked) + self.load_url = load_url or self._generate_url('get_all') + + def load(self): + """ + Loads all given object type items + :return: A list of instantiated objects (type is the same as the given object type) + :rtype list + """ + self._objects_data = self._client.send_get(self.load_url) + return self._generate_objects(self._objects_data) + + def _generate_objects(self, data): + """ + Return a generator which yields object instances of the loaded object type + :param data: List of dicts which represent the to-load object instance + :return: Generator + """ + return (self.object_type(client=self._client, **d) for d in data) + + def query(self, query, data=None): + """ + Perform a query on the current data list + :param query: Items describing the query + Definitions: + * : Should be a dictionary: + {'type' : WhereOperator.XYZ, + 'items': } + * : A tuple defining a single expression: + (, Operator.XYZ, ) + The field is any property you would also find on the given object + * : A list of one or more or items. This means the query structure is recursive and + complex queries are possible + :param data: Data list to use (default to all fetched data) + :return: Result of the query + :rtype: list + """ + if data is None: + if len(self._objects_data) == 0: + self.load() + data = self._objects_data + + query_items = query['items'] + query_type = query['type'] + results = [] + for data_item in data: + result, data_item = self._filter(query_items, query_type, data_item) + if result is True: + results.append(data_item) + return self._generate_objects(results) + + def _filter(self, query_items, where_operator, data_item): + """ + Filter the list + :param query_items: The query items + :param where_operator: The WHERE operator + :param data_item: The data to apply the query on + This object is passed along for future use (might load in an object based on certain properties) + :return: The result of the query (success not), the queried items + :rtype: tuple(bool, any) + """ + if where_operator not in [self.WhereOperators.AND, self.WhereOperators.OR]: + raise NotImplementedError('Invalid where operator specified') + if len(query_items) == 0: + return True, data_item + return_value = where_operator == self.WhereOperators.OR + for query_item in query_items: + if isinstance(query_item, dict): + # Nested + result, data_item = self._filter(query_item['items'], query_item['type'], data_item) + if result == return_value: + return return_value, data_item + else: + key = query_item[0] + op = query_item[1] + value = query_item[2] + result, data_item = self._evaluate(key, value, op, data_item) + if result == return_value: + return return_value, data_item + return not return_value, data_item + + def _evaluate(self, key, value, op, data_item): + """ + Evaluate a given query on a set of data + :param key: Key to check on + :param value: Value to look for + :param op: Operator to use + :param data_item: List of data + :return: The result of the query (success not), the queried items + :rtype: tuple(bool, list) + """ + if isinstance(data_item, self.object_type): + key_value = getattr(data_item, key) + elif isinstance(data_item, dict): + key_value = data_item[key] + else: + raise RuntimeError('Unable to apply query. Item is not a dict or the given object type') + + return self.Operators.get_operator(op)(key_value, value), data_item + + def _generate_url(self, *args, **kwargs): + """ + Generates a URL to fetch the object + :param args: Extra arguments to get passed. Will be separated by / + :param kwargs: Extra arguments to get passed. Values will be separated by / + :return: The generated url + """ + return super(TestRailList, self)._generate_url('get_all', *args, **kwargs) diff --git a/testrail/testrailobject.py b/testrail/testrailobject.py new file mode 100644 index 0000000..d227b3f --- /dev/null +++ b/testrail/testrailobject.py @@ -0,0 +1,289 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Testrail object module +- TestRailObject: An inheritable object which uses to API +""" + +import copy +import json +from itertools import chain + + +class ChangeTracker(object): + """ + Tracks changes + """ + def __init__(self): + self.changed = False + + +class TestRailBase(object): + """ + Base class for TestRail related objects + """ + def __init__(self): + super(TestRailBase, self).__init__() + # self._change_tracker = ChangeTracker() + # Needs to be set by inheritors + self.plural = None + + def _assert_key(self, keyname): + error = RuntimeError('Unable to fetch the object without the identifier.') + if hasattr(self, keyname): + if getattr(self, keyname) is None: + raise error + return + raise error + + def _generate_url(self, method, *args, **kwargs): + """ + Generates a URL to fetch the object + :param method: Method to generate for + :param args: Extra arguments to get passed. Will be separated by / + :param kwargs: Extra arguments to get passed. Values will be separated by / + :return: The generated url + """ + supported_methods = ['update', 'add', 'delete', 'get', 'get_all'] + if method not in supported_methods: + raise RuntimeError('Unsupported method {0}. Supported methods are: {1}'.format(method, ', '.join(supported_methods))) + if method == 'get_all': # Special case + self._assert_key('plural') + extra_args = list(chain((str(a) for a in args), (str(k) for k in kwargs.itervalues()))) + url = 'get_{1}'.format(method, self.plural.lower()) + if len(extra_args) > 0: + extra = '/{0}'.format('&'.join(extra_args)) + url += extra + return url + return '{0}_{1}'.format(method, self.__class__.__name__.lower()) + + @classmethod + def _compare_dicts(cls, dict1, dict2, strict_key_checking=False, excluded_keys=None, should_raise=True): + """ + Compare two dicts and their values + :param dict1: Dicts to base keys off + :param dict2: Dicts to equals based on keys + :param strict_key_checking: Check all keys + :param excluded_keys: Exclude certain keys from checking + :param should_raise: Choice between raising errors or returning the errors + :return: List of errors when no errors is found or should_raise is True + :rtype: list + :raises: RuntimeError when not-matching keys are detected + """ + if excluded_keys is None: + excluded_keys = [] + d1_keys = set(dict1.iterkeys()) - set(excluded_keys) + d2_keys = set(dict2.iterkeys()) - set(excluded_keys) + errors = [] + if d1_keys != d2_keys: + if strict_key_checking is True: + errors.append('The dicts do not have the same keys!') + intersect_keys = set(d1_keys).intersection(set(d2_keys)) + for key in intersect_keys: + if dict1[key] != dict2[key]: + if isinstance(dict1[key], dict) and isinstance(dict1[key], dict): + errors.extend(cls._compare_dicts(dict1[key], dict2[key], + strict_key_checking=strict_key_checking, + should_raise=False)) + else: + errors.append('Keys \'{0}\' are different'.format(key)) + if should_raise is True and len(errors) > 0: + raise RuntimeError('Dicts are not equal: {0}'.format('\n -'.join(errors))) + return errors + + +class Serializable(object): + def to_json(self): + return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + + +class TestRailObject(TestRailBase, Serializable): + + # Keys on which tracking should not be done + _EXCLUDED_KEYS_TRACKING = ['_change_tracker', '_client', '_data_api', '_data_original', 'id', '_loaded', '_loading'] + # Mapping of parameters - type - required or not + # @Todo base full property/doc generation on this variable (might need a rename then though) as the current code makes a bit repetitive + _VERIFY_PARAMS = {} + + def __init__(self, id=None, client=None, load=True,*args, **kwargs): + """ + Initialize a testrail object + :param id: Identifier of the object + :param client: api client pointer + :param load: Auto load the object when an ID is specified + """ + super(TestRailObject, self).__init__() + self.id = id + self._client = client + + # Keep track of api data when loading the object + self._data_api = {} + + self._loaded = False + self._loading = False + + if self.id is not None and load is True: + self.load() + + def _assert_key(self, keyname): + error = RuntimeError('Unable to fetch the object without the identifier.') + if hasattr(self, keyname): + if getattr(self, keyname) is None: + raise error + return + raise error + + def has_changed(self): + """ + Has this object changed + :return: True when changed, False when not + :rtype bool + """ + # Experimental + # return self._change_tracker.changed + # Old + excluded_keys = self._EXCLUDED_KEYS_TRACKING[:] + excluded_keys.remove('id') + if self._loaded is True: + # Compare API data + data_to_compare = self._data_api + else: + # Compare to an empty object (the excluded keys will filter out the object-bound stuff) + data_to_compare = self.__class__().__dict__ + return len(self._compare_dicts(self.__dict__, data_to_compare, excluded_keys=excluded_keys, should_raise=False)) > 0 + + def delete(self): + """ + Deletes the given object + :return: None + :rtype: NoneType + """ + self._assert_key('id') + response = self._client.send_post('{0}/{1}'.format(self._generate_url('delete'), self.id), {}) + # Clear current instance + new_obj = self.__class__(client=self._client) + for key, value in new_obj.__dict__.items(): + if key in ['_change_tracker', '_client', '_data_original']: + continue + self.__dict__[key] = copy.deepcopy(value) + + def load(self): + """ + Loads the given object + :return: None + :rtype: NoneType + """ + self._assert_key('id') + self._loading = True + data = self._client.send_get('{0}/{1}'.format(self._generate_url('get'), self.id)) + self._data_api = data + self._set_attributes(data) + self._loaded = True + self._loading = False + + def _set_attributes(self, kwargs): + """ + Sets a number of attributes on the model + :param kwargs: Dict of attributes to add + :return: None + :rtype: NoneType + """ + for key, value in kwargs.iteritems(): + if hasattr(self, key): + setattr(self, key, value) + + # Hooks for update + def _get_data_update(self): + """ + Return data to update the update + Defaults to returning all data from the create + :return: Returns the data to update the object on the backend + :rtype: dict + """ + return self._get_data_create() + + def _validate_data_update(self, data): + """ + Validates the data to update + Defaults to the same validation as the add + :param data: Data to update + :return: None + """ + return self._validate_data_create(data) + + def _update(self): + """ + Update the given object on the API + :return: None + :rtype: NoneType + """ + self._assert_key('id') + data = self._get_data_update() + self._validate_data_update(data) + return self._client.send_post('{0}/{1}'.format(self._generate_url('update'), self.id), data) + + # Hooks for add + def _get_data_create(self): + """ + Returns all data required for adding + The data it returns is based on the _VERIFY_PARAMS + :return: Data required to add the object + :rtype: dict + """ + return dict((key, getattr(self, key)) for key in self._VERIFY_PARAMS.iterkeys()) + + def _validate_data_create(self, data): + """ + Validates the passed in data given. + Defaults to checking the keys given in _VERIFY_PARAMS + :return: + """ + self._client.verify_params(data, self._VERIFY_PARAMS) + + def _get_url_add(self): + """ + Returns the URL to post for registering the object to the backend + :return: + """ + return self._generate_url('add') + + def _add(self): + """ + Adds a new object on the backend + :return: None + :rtype: NoneType + """ + data = self._get_data_create() + self._validate_data_create(data) + response = self._client.send_post(self._get_url_add(), data) + self._data_api = response + self._set_attributes(response) + self._loaded = True + self._loading = False + + def save(self): + """ + Saves the current object on the backend + :return: None + :rtype: NoneType + """ + if self.has_changed() is False: + return + if self._loaded is True: + return self._update() + else: + return self._add() From cd882ea37f9e9316343fed9807bbfe1c36313e34 Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Wed, 9 May 2018 17:42:16 +0200 Subject: [PATCH 2/5] added extra testrail objects --- helpers/testrailapi.py | 4 ++- testrail/containers/plan.py | 61 +++++++++++++++++++++++++++++++++ testrail/containers/priority.py | 55 +++++++++++++++++++++++++++++ testrail/containers/status.py | 51 +++++++++++++++++++++++++++ testrail/lists/planlist.py | 39 +++++++++++++++++++++ testrail/lists/prioritylist.py | 37 ++++++++++++++++++++ testrail/lists/statuslist.py | 37 ++++++++++++++++++++ testrail/testrailobject.py | 2 +- 8 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 testrail/containers/plan.py create mode 100644 testrail/containers/priority.py create mode 100644 testrail/containers/status.py create mode 100644 testrail/lists/planlist.py create mode 100644 testrail/lists/prioritylist.py create mode 100644 testrail/lists/statuslist.py diff --git a/helpers/testrailapi.py b/helpers/testrailapi.py index 9fc1c9a..6eaefc8 100644 --- a/helpers/testrailapi.py +++ b/helpers/testrailapi.py @@ -25,9 +25,11 @@ class TestrailResult: PASSED = 1 BLOCKED = 2 + UNTESTED = 3 + RETEST = 4 FAILED = 5 + ONGOING = 10 SKIPPED = 11 - UNTESTED = 3 class TestrailPriority(object): diff --git a/testrail/containers/plan.py b/testrail/containers/plan.py new file mode 100644 index 0000000..3c9bd8a --- /dev/null +++ b/testrail/containers/plan.py @@ -0,0 +1,61 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Plan(TestRailObject): + + _VERIFY_PARAMS = {'description': (str, None, False), + 'name': (str, None), + 'milestone_id': (int, None), + 'entries': (list, None, False)} + + def __init__(self, id=None, name=None, description=None, project_id=None, milestone_id=None, entries=None, client=None, *args, **kwargs): + """ + Testrail Plan + :param id: id of the plan + :type id: int + :param name: The name of the plan + :type name: str + :param description: The description of the plan + :type description: str + :param project_id: The id of the project + :type project_id: int + :param milestone_id: The id of the milestone + :type milestone_id: int + :param entries: An array of objects describing the test runs of the plan + :type entries: list + """ + self.name = name + self.description = description + + # Relation + self.project_id = project_id + self.milestone_id = milestone_id + self.entries = entries + + super(Plan, self).__init__(id=id, client=client, *args, **kwargs) + + def _get_url_add(self): + return '{0}/{1}'.format(super(Plan, self)._get_url_add(), self.project_id) + + def _validate_data_update(self, data): + verify_params = dict(item for item in self._VERIFY_PARAMS.iteritems() if item[0] != 'project_id') + return self._client.verify_params(data, verify_params) diff --git a/testrail/containers/priority.py b/testrail/containers/priority.py new file mode 100644 index 0000000..c206d6f --- /dev/null +++ b/testrail/containers/priority.py @@ -0,0 +1,55 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Priority(TestRailObject): + + _VERIFY_PARAMS = {'name': (str, None), + 'shot_name': (str, None), + 'priority': (int, None)} + + def __init__(self, id=None, name=None, short_name=None, priority=None, client=None, *args, **kwargs): + """ + Testrail Priority + :param id: Id of the priority + :type id: int + :param name: The name of the priority + :type name: str + :param short_name: The short name of the priority + :type short_name: str + :param priority: Determines the priority + :type priority: int + """ + + self.name = name + self.short_name = short_name + self.priority = priority + + # Status is a bit special. It has no single-get API. Data can only be loaded through the list + super(Priority, self).__init__(id=id, client=client, load=False, *args, **kwargs) + + def save(self): + """ + Saves the current object on the backend + :return: None + :rtype: NoneType + """ + raise NotImplemented('Statuses cannot be created. Only listed') diff --git a/testrail/containers/status.py b/testrail/containers/status.py new file mode 100644 index 0000000..c7ad6e2 --- /dev/null +++ b/testrail/containers/status.py @@ -0,0 +1,51 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Project module +""" +from ci.testrail.testrailobject import TestRailObject + + +class Status(TestRailObject): + + _VERIFY_PARAMS = {'name': (str, None), + 'label': (str, None)} + + def __init__(self, id=None, name=None, label=None, client=None, *args, **kwargs): + """ + Testrail Status + :param id: Id of the status + :type id: int + :param name: The name of the status + :type name: str + :param label: The label of the status + :type label: str + """ + + self.name = name + self.label = label + + # Status is a bit special. It has no single-get API. Data can only be loaded through the list + super(Status, self).__init__(id=id, client=client, load=False, *args, **kwargs) + + def save(self): + """ + Saves the current object on the backend + :return: None + :rtype: NoneType + """ + raise NotImplemented('Statuses cannot be created. Only listed') diff --git a/testrail/lists/planlist.py b/testrail/lists/planlist.py new file mode 100644 index 0000000..9cb0854 --- /dev/null +++ b/testrail/lists/planlist.py @@ -0,0 +1,39 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.plan import Plan + + +class PlanList(TestRailList): + def __init__(self, project_id, client, *args, **kwargs): + """ + Initializes a RunList + :param project_id: ID of the Project to list runs for + :type project_id: int + """ + super(PlanList, self).__init__(client=client, object_type=Plan, plural='plans', *args, **kwargs) + self.load_url = self._generate_url(project_id) + + def get_plan_by_name(self, plan_name): + for plan in self.load(): + if plan.name == plan_name: + return plan + + raise LookupError('Plan `{0}` not found.'.format(plan_name)) diff --git a/testrail/lists/prioritylist.py b/testrail/lists/prioritylist.py new file mode 100644 index 0000000..350c21b --- /dev/null +++ b/testrail/lists/prioritylist.py @@ -0,0 +1,37 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.status import Status + + +class PriorityList(TestRailList): + def __init__(self, client, *args, **kwargs): + """ + Initializes a StatusList + """ + # Status is a bit special. It has no single-get API. Data can only be loaded through the list + super(PriorityList, self).__init__(client=client, object_type=Status, plural='priorities', *args, **kwargs) + + def get_priority_by_name(self, priority_name): + for priority in self.load(): + if priority_name in priority.name: + return priority + + raise LookupError('Priority `{0}` not found.'.format(priority_name)) diff --git a/testrail/lists/statuslist.py b/testrail/lists/statuslist.py new file mode 100644 index 0000000..4f36779 --- /dev/null +++ b/testrail/lists/statuslist.py @@ -0,0 +1,37 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ci.testrail.testraillist import TestRailList +from ci.testrail.containers.status import Status + + +class StatusList(TestRailList): + def __init__(self, client, *args, **kwargs): + """ + Initializes a StatusList + """ + # Status is a bit special. It has no single-get API. Data can only be loaded through the list + super(StatusList, self).__init__(client=client, object_type=Status, plural='statuses', *args, **kwargs) + + def get_status_by_name(self, status_name): + for status in self.load(): + if status.name == status_name: + return status + + raise LookupError('Status `{0}` not found.'.format(status_name)) diff --git a/testrail/testrailobject.py b/testrail/testrailobject.py index d227b3f..57943f9 100644 --- a/testrail/testrailobject.py +++ b/testrail/testrailobject.py @@ -119,7 +119,7 @@ class TestRailObject(TestRailBase, Serializable): # @Todo base full property/doc generation on this variable (might need a rename then though) as the current code makes a bit repetitive _VERIFY_PARAMS = {} - def __init__(self, id=None, client=None, load=True,*args, **kwargs): + def __init__(self, id=None, client=None, load=True, *args, **kwargs): """ Initialize a testrail object :param id: Identifier of the object From 6ee3863708371e9567f7214a0854d2d44868bcdd Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Thu, 17 May 2018 10:46:06 +0200 Subject: [PATCH 3/5] fixed imports and add some extra logging. --- testrail/containers/case.py | 2 +- testrail/containers/milestone.py | 2 +- testrail/containers/plan.py | 2 +- testrail/containers/priority.py | 2 +- testrail/containers/project.py | 2 +- testrail/containers/result.py | 2 +- testrail/containers/run.py | 2 +- testrail/containers/section.py | 2 +- testrail/containers/status.py | 2 +- testrail/containers/suite.py | 2 +- testrail/containers/test.py | 2 +- testrail/lists/caselist.py | 4 ++-- testrail/lists/milestonelist.py | 4 ++-- testrail/lists/planlist.py | 4 ++-- testrail/lists/prioritylist.py | 4 ++-- testrail/lists/projectlist.py | 4 ++-- testrail/lists/runlist.py | 4 ++-- testrail/lists/sectionlist.py | 4 ++-- testrail/lists/statuslist.py | 4 ++-- testrail/lists/suitelist.py | 4 ++-- testrail/lists/testlist.py | 4 ++-- testrail/testraillist.py | 4 ++-- testrail/testrailobject.py | 2 +- 23 files changed, 34 insertions(+), 34 deletions(-) diff --git a/testrail/containers/case.py b/testrail/containers/case.py index a9ecd0d..fe98b57 100644 --- a/testrail/containers/case.py +++ b/testrail/containers/case.py @@ -17,7 +17,7 @@ """ Case Module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Case(TestRailObject): diff --git a/testrail/containers/milestone.py b/testrail/containers/milestone.py index 4d0c3b8..3205fa9 100644 --- a/testrail/containers/milestone.py +++ b/testrail/containers/milestone.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Milestone(TestRailObject): diff --git a/testrail/containers/plan.py b/testrail/containers/plan.py index 3c9bd8a..7fef1b3 100644 --- a/testrail/containers/plan.py +++ b/testrail/containers/plan.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Plan(TestRailObject): diff --git a/testrail/containers/priority.py b/testrail/containers/priority.py index c206d6f..110a8f2 100644 --- a/testrail/containers/priority.py +++ b/testrail/containers/priority.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Priority(TestRailObject): diff --git a/testrail/containers/project.py b/testrail/containers/project.py index 59d905e..b3a9c31 100644 --- a/testrail/containers/project.py +++ b/testrail/containers/project.py @@ -18,7 +18,7 @@ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Project(TestRailObject): diff --git a/testrail/containers/result.py b/testrail/containers/result.py index f0bcf54..52644e6 100644 --- a/testrail/containers/result.py +++ b/testrail/containers/result.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Result(TestRailObject): diff --git a/testrail/containers/run.py b/testrail/containers/run.py index 8044c60..9027614 100644 --- a/testrail/containers/run.py +++ b/testrail/containers/run.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Run(TestRailObject): diff --git a/testrail/containers/section.py b/testrail/containers/section.py index a22a19b..a1d9b1e 100644 --- a/testrail/containers/section.py +++ b/testrail/containers/section.py @@ -17,7 +17,7 @@ """ Section Module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Section(TestRailObject): diff --git a/testrail/containers/status.py b/testrail/containers/status.py index c7ad6e2..d893734 100644 --- a/testrail/containers/status.py +++ b/testrail/containers/status.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Status(TestRailObject): diff --git a/testrail/containers/suite.py b/testrail/containers/suite.py index a59eac2..8fd325f 100644 --- a/testrail/containers/suite.py +++ b/testrail/containers/suite.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Suite(TestRailObject): diff --git a/testrail/containers/test.py b/testrail/containers/test.py index 9121d76..9e190e3 100644 --- a/testrail/containers/test.py +++ b/testrail/containers/test.py @@ -17,7 +17,7 @@ """ Project module """ -from ci.testrail.testrailobject import TestRailObject +from ..testrailobject import TestRailObject class Test(TestRailObject): diff --git a/testrail/lists/caselist.py b/testrail/lists/caselist.py index 0ffcb5f..8c69a8a 100644 --- a/testrail/lists/caselist.py +++ b/testrail/lists/caselist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.case import Case +from ..testraillist import TestRailList +from ..containers.case import Case class CaseList(TestRailList): diff --git a/testrail/lists/milestonelist.py b/testrail/lists/milestonelist.py index 597e7eb..aa96ff4 100644 --- a/testrail/lists/milestonelist.py +++ b/testrail/lists/milestonelist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.milestone import Milestone +from ..testraillist import TestRailList +from ..containers.milestone import Milestone class MilestoneList(TestRailList): diff --git a/testrail/lists/planlist.py b/testrail/lists/planlist.py index 9cb0854..efbb7b2 100644 --- a/testrail/lists/planlist.py +++ b/testrail/lists/planlist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.plan import Plan +from ..testraillist import TestRailList +from ..containers.plan import Plan class PlanList(TestRailList): diff --git a/testrail/lists/prioritylist.py b/testrail/lists/prioritylist.py index 350c21b..a668793 100644 --- a/testrail/lists/prioritylist.py +++ b/testrail/lists/prioritylist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.status import Status +from ..testraillist import TestRailList +from ..containers.status import Status class PriorityList(TestRailList): diff --git a/testrail/lists/projectlist.py b/testrail/lists/projectlist.py index 757ff71..d218e36 100644 --- a/testrail/lists/projectlist.py +++ b/testrail/lists/projectlist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.project import Project +from ..testraillist import TestRailList +from ..containers.project import Project class ProjectList(TestRailList): diff --git a/testrail/lists/runlist.py b/testrail/lists/runlist.py index d18073c..1c9e891 100644 --- a/testrail/lists/runlist.py +++ b/testrail/lists/runlist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.run import Run +from ..testraillist import TestRailList +from ..containers.run import Run class RunList(TestRailList): diff --git a/testrail/lists/sectionlist.py b/testrail/lists/sectionlist.py index 01a4e50..67b5f78 100644 --- a/testrail/lists/sectionlist.py +++ b/testrail/lists/sectionlist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.section import Section +from ..testraillist import TestRailList +from ..containers.section import Section class SectionList(TestRailList): diff --git a/testrail/lists/statuslist.py b/testrail/lists/statuslist.py index 4f36779..6c02ddb 100644 --- a/testrail/lists/statuslist.py +++ b/testrail/lists/statuslist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.status import Status +from ..testraillist import TestRailList +from ..containers.status import Status class StatusList(TestRailList): diff --git a/testrail/lists/suitelist.py b/testrail/lists/suitelist.py index 1d76d01..818de11 100644 --- a/testrail/lists/suitelist.py +++ b/testrail/lists/suitelist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.suite import Suite +from ..testraillist import TestRailList +from ..containers.suite import Suite class SuiteList(TestRailList): diff --git a/testrail/lists/testlist.py b/testrail/lists/testlist.py index 8d77686..4c0924b 100644 --- a/testrail/lists/testlist.py +++ b/testrail/lists/testlist.py @@ -17,8 +17,8 @@ """ ProjectList module """ -from ci.testrail.testraillist import TestRailList -from ci.testrail.containers.test import Test +from ..testraillist import TestRailList +from ..containers.test import Test class TestList(TestRailList): diff --git a/testrail/testraillist.py b/testrail/testraillist.py index 2b3a221..99aa10d 100644 --- a/testrail/testraillist.py +++ b/testrail/testraillist.py @@ -18,7 +18,7 @@ TestRailList Module """ import operator -from ci.testrail.testrailobject import TestRailBase +from .testrailobject import TestRailBase class OperatorsMeta(type): @@ -62,7 +62,7 @@ class WhereOperators(object): def __init__(self, object_type, client, plural, load_url=None, *args, **kwargs): """ - :param object_type: Object Type to load (Example: ci.testrail.containers.project) + :param object_type: Object Type to load (Example: ..containers.project) :param client: API Client :param plural: Plural form of the object to load (used in url generation) (Eg. project -> projects) :param load_url: The url to load the object from (Could differ when other ids need to be specified) diff --git a/testrail/testrailobject.py b/testrail/testrailobject.py index 57943f9..de4e531 100644 --- a/testrail/testrailobject.py +++ b/testrail/testrailobject.py @@ -95,7 +95,7 @@ def _compare_dicts(cls, dict1, dict2, strict_key_checking=False, excluded_keys=N intersect_keys = set(d1_keys).intersection(set(d2_keys)) for key in intersect_keys: if dict1[key] != dict2[key]: - if isinstance(dict1[key], dict) and isinstance(dict1[key], dict): + if isinstance(dict1[key], dict) and isinstance(dict2[key], dict): errors.extend(cls._compare_dicts(dict1[key], dict2[key], strict_key_checking=strict_key_checking, should_raise=False)) From 62c832f5fb3eabafa32f2f9387dd0cb42fb49226 Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Tue, 22 May 2018 14:12:14 +0200 Subject: [PATCH 4/5] Fixed stuck threads. Added CaseType from Testrail. --- helpers/ci_constants.py | 55 +++++ helpers/disk.py | 38 ++- helpers/domain.py | 4 +- helpers/exceptions.py | 2 +- helpers/hypervisor/hypervisor.py | 85 +++++-- helpers/storagedriver.py | 2 +- helpers/storagerouter.py | 164 +++++-------- helpers/system.py | 2 +- helpers/testrailapi.py | 393 ------------------------------- helpers/thread.py | 7 +- helpers/vpool.py | 3 +- remove/arakoon.py | 3 +- remove/backend.py | 109 +++++---- remove/roles.py | 66 +++--- remove/vdisk.py | 53 ++--- remove/vpool.py | 14 +- setup/arakoon.py | 52 +++- setup/backend.py | 265 +++++++++------------ setup/celery.py | 2 +- setup/domain.py | 41 ++-- setup/proxy.py | 6 +- setup/roles.py | 39 ++- setup/storagedriver.py | 4 +- setup/vdisk.py | 89 +++---- setup/vpool.py | 47 ++-- testrail/containers/casetype.py | 50 ++++ testrail/lists/casetypelist.py | 38 +++ validate/backend.py | 53 +++-- 28 files changed, 717 insertions(+), 969 deletions(-) create mode 100644 helpers/ci_constants.py delete mode 100644 helpers/testrailapi.py create mode 100644 testrail/containers/casetype.py create mode 100644 testrail/lists/casetypelist.py diff --git a/helpers/ci_constants.py b/helpers/ci_constants.py new file mode 100644 index 0000000..540498c --- /dev/null +++ b/helpers/ci_constants.py @@ -0,0 +1,55 @@ +# Copyright (C) 2016 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. +import json +from ci.api_lib.helpers.api import OVSClient + + +class CIConstants(object): + """ + Collection of multiple constants and constant related instances + """ + + CONFIG_LOC = "/opt/OpenvStorage/ci/config/setup.json" + TEST_SCENARIO_LOC = "/opt/OpenvStorage/ci/scenarios/" + TESTRAIL_LOC = "/opt/OpenvStorage/ci/config/testrail.json" + + with open(CONFIG_LOC, 'r') as JSON_CONFIG: + SETUP_CFG = json.load(JSON_CONFIG) + + HYPERVISOR_INFO = SETUP_CFG['ci'].get('hypervisor') + DOMAIN_INFO = SETUP_CFG['setup']['domains'] + BACKEND_INFO = SETUP_CFG['setup']['backends'] + STORAGEROUTER_INFO = SETUP_CFG['setup']['storagerouters'] + + class classproperty(property): + def __get__(self, cls, owner): + return classmethod(self.fget).__get__(None, owner)() + + @classproperty + def api(cls): + return OVSClient(cls.SETUP_CFG['ci']['grid_ip'], + cls.SETUP_CFG['ci']['user']['api']['username'], + cls.SETUP_CFG['ci']['user']['api']['password']) + + @classmethod + def get_vpool_names(cls): + names = [] + for sr_ip, items in cls.STORAGEROUTER_INFO.iteritems(): + vpools = items.get('vpools') + for vp_name, vp_info in vpools.iteritems(): + if vp_name not in names: + names.append(vp_name) + return names \ No newline at end of file diff --git a/helpers/disk.py b/helpers/disk.py index d6db106..d9d18e2 100644 --- a/helpers/disk.py +++ b/helpers/disk.py @@ -24,14 +24,10 @@ class DiskHelper(object): DiskHelper class """ - def __init__(self): - pass - @staticmethod def get_diskpartitions_by_guid(diskguid): """ Fetch disk partitions by disk guid - :param diskguid: ip address of a storagerouter :type diskguid: str :return: list of DiskPartition Objects @@ -41,54 +37,48 @@ def get_diskpartitions_by_guid(diskguid): return [dp for dp in DiskPartitionList.get_partitions() if dp.disk_guid == diskguid] @staticmethod - def get_roles_from_disks(storagerouter_ip=None): + def get_roles_from_disks(storagerouter_guid=None): """ Fetch disk roles from all disks with optional storagerouter_ip - - :param storagerouter_ip: ip address of a storage router - :type storagerouter_ip: str + :param storagerouter_guid: guid of a storage router + :type storagerouter_guid: str :return: list of lists with roles :rtype: list > list """ - if not storagerouter_ip: + if not storagerouter_guid: return [partition.roles for disk in DiskList.get_disks() for partition in disk.partitions] else: - storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip) return [partition.roles for disk in DiskList.get_disks() if disk.storagerouter_guid == storagerouter_guid for partition in disk.partitions] @staticmethod - def get_disk_by_diskname(storagerouter_ip, disk_name): + def get_disk_by_diskname(storagerouter_guid, disk_name): """ - Get a disk object by storagerouter ip and disk name - - :param storagerouter_ip: ip address of a storage router - :type storagerouter_ip: str + Get a disk object by storagerouter guid and disk name + :param storagerouter_guid: guid address of a storage router + :type storagerouter_guid: str :param disk_name: name of a disk (e.g. sda) :type disk_name: str :return: disk object :rtype: ovs.dal.hybrids.Disk """ - - storagerouter = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip=storagerouter_ip) + storagerouter = StoragerouterHelper.get_storagerouter_by_guid(storagerouter_guid=storagerouter_guid) for disk in storagerouter.disks: if disk.name == disk_name: return disk @staticmethod - def get_roles_from_disk(storagerouter_ip, disk_name): + def get_roles_from_disk(storagerouter_guid, disk_name): """ Get the roles from a certain disk - - :param storagerouter_ip: ip address of a storage router - :type storagerouter_ip: str + :param storagerouter_guid: guid address of a storage router + :type storagerouter_guid: str :param disk_name: name of a disk (e.g. sda) :type disk_name: str :return: list of roles of all partitions on a certain disk :rtype: list """ - - disk = DiskHelper.get_disk_by_diskname(storagerouter_ip, disk_name) + disk = DiskHelper.get_disk_by_diskname(storagerouter_guid, disk_name) roles_on_disk = [] if disk: for diskpartition in disk.partitions: @@ -96,4 +86,4 @@ def get_roles_from_disk(storagerouter_ip, disk_name): roles_on_disk.append(role) return roles_on_disk else: - raise RuntimeError("Disk with name `{0}` not found on storagerouter `{1}`".format(disk_name, storagerouter_ip)) + raise RuntimeError("Disk with name `{0}` not found on storagerouter `{1}`".format(disk_name, storagerouter_guid)) diff --git a/helpers/domain.py b/helpers/domain.py index a0760ef..7a5db5a 100644 --- a/helpers/domain.py +++ b/helpers/domain.py @@ -1,6 +1,9 @@ # Copyright (C) 2016 iNuron NV # # This file is part of Open vStorage Open Source Edition (OSE), +# Copyright (C) 2016 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), # as available from # # http://www.openvstorage.org and @@ -109,4 +112,3 @@ def get_storagedrivers_in_same_domain(domain_guid): :rtype: list """ return [storagedriver for storagedriver in StorageDriverList.get_storagedrivers() if domain_guid in storagedriver.storagerouter.regular_domains] - diff --git a/helpers/exceptions.py b/helpers/exceptions.py index 1c043c9..96032e0 100644 --- a/helpers/exceptions.py +++ b/helpers/exceptions.py @@ -86,4 +86,4 @@ class ImageConvertError(Exception): """ Raised when an object was queries that doesn't exist """ - pass + pass \ No newline at end of file diff --git a/helpers/hypervisor/hypervisor.py b/helpers/hypervisor/hypervisor.py index 4b88185..c0c4230 100644 --- a/helpers/hypervisor/hypervisor.py +++ b/helpers/hypervisor/hypervisor.py @@ -16,39 +16,74 @@ Hypervisor/ManagementCenter factory module Using the module requires libvirt api to be available on the MACHINE THAT EXECUTES THE CODE """ - from ovs_extensions.generic.filemutex import file_mutex +from ovs_extensions.generic.toolbox import ExtensionsToolbox +from ovs.lib.helpers.toolbox import Toolbox +from ...helpers.ci_constants import CIConstants -class HypervisorFactory(object): +class HypervisorFactory(CIConstants): """ HypervisorFactory class provides functionality to get abstracted hypervisor """ - hypervisors = {} - @staticmethod - def get(ip, username, password, hvtype): + @classmethod + def get(cls, hv_credentials=None): """ Returns the appropriate hypervisor client class for a given PMachine + :param hv_credentials: object that contains ip, user, password and hypervisor type + :type hv_credentials: HypervisorCredentials object """ - key = '{0}_{1}'.format(ip, username) - if key not in HypervisorFactory.hypervisors: - mutex = file_mutex('hypervisor_{0}'.format(key)) - try: - mutex.acquire(30) - if key not in HypervisorFactory.hypervisors: - if hvtype == 'VMWARE': - # Not yet tested. Needs to be rewritten - raise NotImplementedError("{0} has not yet been implemented".format(hvtype)) - from .hypervisors.vmware import VMware - hypervisor = VMware(ip, username, password) - elif hvtype == 'KVM': - from .hypervisors.kvm import KVM - hypervisor = KVM(ip, username, password) - else: - raise NotImplementedError('Hypervisor {0} is not yet supported'.format(hvtype)) - HypervisorFactory.hypervisors[key] = hypervisor - finally: - mutex.release() - return HypervisorFactory.hypervisors[key] + if hv_credentials is None: + return cls.get(HypervisorCredentials(ip=CIConstants.HYPERVISOR_INFO['ip'], + user=CIConstants.HYPERVISOR_INFO['user'], + password=CIConstants.HYPERVISOR_INFO['password'], + type=CIConstants.HYPERVISOR_INFO['type'])) + if not isinstance(hv_credentials, HypervisorCredentials): + raise TypeError('Credentials must be of type HypervisorCredentials') + return cls.hypervisors.get(hv_credentials, cls._add_hypervisor(hv_credentials)) + + @staticmethod + def _add_hypervisor(hypervisor_credentials): + ip = hypervisor_credentials.ip + username = hypervisor_credentials.user + password = hypervisor_credentials.password + hvtype = hypervisor_credentials.type + mutex = file_mutex('hypervisor_{0}'.format(hash(hypervisor_credentials))) + try: + mutex.acquire(30) + if hypervisor_credentials not in HypervisorFactory.hypervisors: + if hvtype == 'VMWARE': + # Not yet tested. Needs to be rewritten + raise NotImplementedError("{0} has not yet been implemented".format(hvtype)) + from .hypervisors.vmware import VMware + hypervisor = VMware(ip, username, password) + elif hvtype == 'KVM': + from .hypervisors.kvm import KVM + hypervisor = KVM(ip, username, password) + else: + raise NotImplementedError('Hypervisor {0} is not yet supported'.format(hvtype)) + HypervisorFactory.hypervisors[hypervisor_credentials] = hypervisor + return hypervisor + finally: + mutex.release() + + +class HypervisorCredentials(object): + def __init__(self, ip, user, password, type): + required_params = {'ip': (str, Toolbox.regex_ip), + 'user': (str, None), + 'password': (str, None), + 'type': (str, ['KVM', 'VMWARE'])} + ExtensionsToolbox.verify_required_params(required_params, {'ip': ip, + 'user': user, + 'password': password, + 'type': type}) + self.ip = ip + self.user = user + self.password = password + self.type = type + + def __str__(self): + return 'hypervisor at ip {0} of type {1}'.format(self.ip, self.type) diff --git a/helpers/storagedriver.py b/helpers/storagedriver.py index 5cf66af..3213622 100644 --- a/helpers/storagedriver.py +++ b/helpers/storagedriver.py @@ -17,9 +17,9 @@ from ovs.dal.hybrids.storagedriver import StorageDriver from ovs.dal.lists.storagedriverlist import StorageDriverList from ovs.extensions.generic.configuration import Configuration +from ovs.log.log_handler import LogHandler from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.services.servicefactory import ServiceFactory -from ovs.log.log_handler import LogHandler class StoragedriverHelper(object): diff --git a/helpers/storagerouter.py b/helpers/storagerouter.py index 7b72c78..e996846 100644 --- a/helpers/storagerouter.py +++ b/helpers/storagerouter.py @@ -13,13 +13,13 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. -from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.dal.hybrids.storagerouter import StorageRouter -from ovs.extensions.generic.system import System +from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants -class StoragerouterHelper(object): +class StoragerouterHelper(CIConstants): """ StoragerouterHelper class @@ -42,77 +42,51 @@ def get_storagerouter_by_guid(storagerouter_guid): """ return StorageRouter(storagerouter_guid) - @staticmethod - def get_storagerouter_guid_by_ip(storagerouter_ip): - """ - - :param storagerouter_ip: ip of a storagerouter - :type storagerouter_ip: str - :return: storagerouter guid - :rtype: str - """ - return StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip).guid - @staticmethod def get_storagerouter_by_ip(storagerouter_ip): """ - :param storagerouter_ip: ip of a storagerouter :type storagerouter_ip: str - :return: storagerouter object + :return: storagerouter :rtype: ovs.dal.hybrids.storagerouter.StorageRouter """ return StorageRouterList.get_by_ip(storagerouter_ip) @staticmethod - def get_disks_by_ip(storagerouter_ip): + def get_storagerouter_ip(storagerouter_guid): """ - - :param storagerouter_ip: - :type storagerouter_ip: str - :return: disks found for the storagerouter ip - :rtype: list of + :param storagerouter_guid: guid of a storagerouter + :type storagerouter_guid: str + :return: storagerouter ip + :rtype: str """ - storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip) - return StorageRouter(storagerouter_guid).disks + return StorageRouter(storagerouter_guid).ip @staticmethod - def get_disk_by_ip(ip, diskname): + def get_disk_by_name(guid, diskname): """ - Fetch a disk by its ip and name + Fetch a disk by its guid and name - :param ip: ip address of a storagerouter + :param guid: guid of a storagerouter + :type guid: str :param diskname: shortname of a disk (e.g. sdb) :return: Disk Object :rtype: ovs.dal.hybrids.disk.disk """ - - storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(ip) - disks = StorageRouter(storagerouter_guid).disks + disks = StoragerouterHelper.get_storagerouter_by_guid(guid).disks for d in disks: if d.name == diskname: return d - @staticmethod - def get_local_storagerouter(): - """ - Fetch the local storagerouter settings - - :return: StorageRouter Object - :rtype: ovs.dal.hybrids.storagerouter.StorageRouter - """ - - return System.get_my_storagerouter() - @staticmethod def get_storagerouter_ips(): - """ - Fetch all the ip addresses in this cluster + """ + Fetch all the ip addresses in this cluster - :return: list with storagerouter ips - :rtype: list - """ - return [storagerouter.ip for storagerouter in StorageRouterList.get_storagerouters()] + :return: list with storagerouter ips + :rtype: list + """ + return [storagerouter.ip for storagerouter in StorageRouterList.get_storagerouters()] @staticmethod def get_storagerouters(): @@ -122,72 +96,54 @@ def get_storagerouters(): :return: list with storagerouters :rtype: list """ - return StorageRouterList.get_storagerouters() - @staticmethod - def get_master_storagerouters(): - """ - Fetch the master storagerouters - - :return: list with master storagerouters - :rtype: list - """ - - return StorageRouterList.get_masters() - - @staticmethod - def get_master_storagerouter_ips(): - """ - Fetch the master storagerouters ips - - :return: list with master storagerouters ips - :rtype: list - """ - - return [storagerouter.ip for storagerouter in StorageRouterList.get_masters()] - - @staticmethod - def get_slave_storagerouters(): - """ - Fetch the slave storagerouters - - :return: list with slave storagerouters - :rtype: list - """ - - return StorageRouterList.get_slaves() - - @staticmethod - def get_slave_storagerouter_ips(): - """ - Fetch the slave storagerouters ips - - :return: list with slave storagerouters ips - :rtype: list - """ - - return [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()] - - @staticmethod - def sync_disk_with_reality(api, guid=None, ip=None, timeout=None): + @classmethod + def sync_disk_with_reality(cls, guid=None, ip=None, timeout=None, *args, **kwargs): """ - - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param guid: guid of the storagerouter :type guid: str :param ip: ip of the storagerouter :type ip: str :param timeout: timeout time in seconds :type timeout: int + """ + if guid is not None: + if ip is not None: + Logger.warning('Both storagerouter guid and ip passed, using guid for sync.') + storagerouter_guid = guid + elif ip is not None: + storagerouter_guid = StoragerouterHelper.get_storagerouter_by_ip(ip).guid + else: + raise ValueError('No guid or ip passed.') + task_id = cls.api.post(api='/storagerouters/{0}/rescan_disks/'.format(storagerouter_guid), data=None) + return cls.api.wait_for_task(task_id=task_id, timeout=timeout) + + @classmethod + def get_storagerouters_by_role(cls): + """ + Gets storagerouters based on roles :return: """ - storagerouter_guid = guid - if ip is not None: - storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(ip) - if storagerouter_guid is None: - raise ValueError('No guid or ip found.') + voldr_str_1 = None # Will act as volumedriver node + voldr_str_2 = None # Will act as volumedriver node + compute_str = None # Will act as compute node + if isinstance(cls.HYPERVISOR_INFO, dict): # Hypervisor section is filled in -> VM environment + nodes_info = {} + for hv_ip, hv_info in cls.HYPERVISOR_INFO['vms'].iteritems(): + nodes_info[hv_ip] = hv_info + elif cls.SETUP_CFG['ci'].get('nodes') is not None: # Physical node section -> Physical environment + nodes_info = cls.SETUP_CFG['ci']['nodes'] else: - task_id = api.post(api='/storagerouters/{0}/rescan_disks/'.format(storagerouter_guid), data=None) - return api.wait_for_task(task_id=task_id, timeout=timeout) + raise RuntimeError('Unable to fetch node information. Either hypervisor section or node section is missing!') + for node_ip, node_details in nodes_info.iteritems(): + if node_details['role'] == "VOLDRV": + if voldr_str_1 is None: + voldr_str_1 = StoragerouterHelper.get_storagerouter_by_ip(node_ip) + elif voldr_str_2 is None: + voldr_str_2 = StoragerouterHelper.get_storagerouter_by_ip(node_ip) + elif node_details['role'] == "COMPUTE" and compute_str is None: + compute_str = StoragerouterHelper.get_storagerouter_by_ip(node_ip) + assert voldr_str_1 is not None and voldr_str_2 is not None and compute_str is not None,\ + 'Could not fetch 2 storagedriver nodes and 1 compute node based on the setup.json config.' + return voldr_str_1, voldr_str_2, compute_str diff --git a/helpers/system.py b/helpers/system.py index 1a4a284..2b54609 100644 --- a/helpers/system.py +++ b/helpers/system.py @@ -15,9 +15,9 @@ # but WITHOUT ANY WARRANTY of any kind. import time from ovs.log.log_handler import LogHandler -from ovs.extensions.services.servicefactory import ServiceFactory from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.generic.system import System +from ovs.extensions.services.servicefactory import ServiceFactory class SystemHelper(object): diff --git a/helpers/testrailapi.py b/helpers/testrailapi.py deleted file mode 100644 index 6eaefc8..0000000 --- a/helpers/testrailapi.py +++ /dev/null @@ -1,393 +0,0 @@ -# Copyright (C) 2016 iNuron NV -# -# This file is part of Open vStorage Open Source Edition (OSE), -# as available from -# -# http://www.openvstorage.org and -# http://www.openvstorage.com. -# -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) -# as published by the Free Software Foundation, in version 3 as it comes -# in the LICENSE.txt file of the Open vStorage OSE distribution. -# -# Open vStorage is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY of any kind. -import base64 -import requests -import urllib2 - - -class TestrailResult: - """ - Testrail Result class - """ - - PASSED = 1 - BLOCKED = 2 - UNTESTED = 3 - RETEST = 4 - FAILED = 5 - ONGOING = 10 - SKIPPED = 11 - - -class TestrailPriority(object): - """ - Testrail Priority class - """ - - MUST_TEST_HIGH = 5 - MUST_TEST_LOW = 4 - TEST_IF_TIME_HIGH = 3 - TEST_IF_TIME_LOW = 2 - DONT_TEST = 1 - - -class TestrailCaseType(object): - """ - Testrail Priority class - """ - - ADMINISTRATION = 'Administration' - AT_EXT = 'AT_Extensive' - AT_QUICK = 'AT_Quick' - FUNCTIONAL = 'Functionality' - MANUAL = 'Manual' - PERFORMANCE = 'Performance' - REGRESSION = 'Regression' - STABILITY = 'Stability' - - -class TestrailApi(object): - """ - Testrail API class - - on init will load all existing projects - - projects / suites / sections are not mutable via this class so are assumed to be present - this allows more control on testrail - - testcases can be added dynamically to an already existing project/suite/(sub)section combo - """ - - def __init__(self, server, user=None, password=None, key=None): - self.server = server - assert (user and password) or key, \ - "Credentials are needed for testrail connection, specify either user/password or basic auth key" - self.base64_authentication = key or base64.encodestring('%s:%s' % (user, password)).replace('\n', '') - self.URL = "http://%s/index.php?/api/v2/%s" - self.projects = self.get_projects() - self.AT_QUICK_ID = self.get_case_type_by_name('AT_Quick')['id'] - - def _get_from_testrail(self, testrail_item, main_id=None, url_params=None): - if main_id: - url = self.URL % (self.server, '%s/%s' % (testrail_item, main_id)) - if url_params: - for key in url_params: - url += "&%s=%s" % (key, url_params[key]) - else: - url = self.URL % (self.server, testrail_item) - - headers = {'Content-Type': 'application/json', 'Authorization': "Basic %s" % self.base64_authentication} - - try: - content = requests.get(url, headers=headers) - except urllib2.HTTPError as e: - print e.reason - raise - except urllib2.URLError as e: - print e - print e.reason - raise - except: - raise - - return content.json() - - def _add_to_testrail(self, testrail_item, main_id=None, values=None, sub_id=None): - - if main_id: - main_id = str(main_id) - if sub_id: - url = self.URL % (self.server, '%s/%s/%s' % (testrail_item, main_id, sub_id)) - else: - url = self.URL % (self.server, '%s/%s' % (testrail_item, main_id)) - else: - url = self.URL % (self.server, '%s' % testrail_item) - - headers = {'Content-Type': 'application/json', 'Authorization': "Basic %s" % self.base64_authentication} - - try: - content = requests.post(url, json=values, headers=headers) - except urllib2.HTTPError as e: - print e.reason - raise - except urllib2.URLError as e: - print e.reason - raise - except: - raise - - return content.json() if content.json() else None - - def get_case(self, case_id): - return self._get_from_testrail("get_case", case_id) - - def get_cases(self, project_id, suite_id, section_id=None): - extra_params = {'suite_id': suite_id} - if section_id: - extra_params['section_id'] = section_id - return self._get_from_testrail("get_cases", project_id, extra_params) - - def add_case(self, section_id, title, type_id=None, priority_id=None, estimate=None, milestone_id=None, refs=None, - custom_fields=None): - extra_params = {'title': title} - if not type_id: - type_id = self.AT_QUICK_ID - extra_params['type_id'] = type_id - if priority_id: - extra_params['priority_id'] = priority_id - if estimate: - extra_params['estimate'] = estimate - if milestone_id: - extra_params['milestone_id'] = milestone_id - if refs: - extra_params['refs'] = refs - if custom_fields: - for key, value in custom_fields.iteritems(): - assert "custom_" in key, "Custom fields need to start with 'custom_'" - extra_params[key] = value - return self._add_to_testrail('add_case', section_id, extra_params) - - def update_case(self, case_id, title=None, type_id=None, priority_id=None, estimate=None, milestone_id=None, - refs=None, custom_fields=None): - extra_params = {} - if title: - extra_params['title'] = title - if type_id: - extra_params['type_id'] = type_id - if priority_id: - extra_params['priority_id'] = priority_id - if estimate: - extra_params['estimate'] = estimate - if milestone_id: - extra_params['milestone_id'] = milestone_id - if refs: - extra_params['refs'] = refs - if custom_fields: - for key, value in custom_fields.iteritems(): - assert "custom_" in key, "Custom fields need to start with 'custom_'" - extra_params[key] = value - return self._add_to_testrail('update_case', case_id, extra_params) - - def delete_case(self, case_id): - return self._add_to_testrail('delete_case', case_id) - - def get_case_fields(self): - return self._get_from_testrail("get_case_fields") - - def get_case_types(self): - return self._get_from_testrail("get_case_types") - - def get_case_type_by_name(self, name): - case_types = [case_type for case_type in self.get_case_types() if case_type['name'] == name] - if not case_types or len(case_types) > 1: - raise Exception("No or multiple case types found with name: {0} ".format(name)) - return case_types[0] - - def get_section(self, section_id): - return self._get_from_testrail("get_section", section_id) - - def get_sections(self, project_id, suite_id): - return self._get_from_testrail("get_sections", project_id, {'suite_id': suite_id}) - - def get_section_by_name(self, project_id, suite_id, name): - sections = [section for section in self.get_sections(project_id, suite_id) if section['name'] == name] - if not sections or len(sections) > 1: - raise Exception("No or multiple suites found with name: {0} ".format(name)) - return sections[0] - - def get_suite(self, suite_id): - return self._get_from_testrail('get_suite', suite_id) - - def get_suites(self, project_id): - return self._get_from_testrail('get_suites', project_id) - - def get_suite_by_name(self, project_id, name): - suites = [suite for suite in self.get_suites(project_id) if suite['name'] == name] - if not suites or len(suites) > 1: - raise Exception("No or multiple suites found with name: {0} ".format(name)) - return suites[0] - - def get_plan(self, plan_id): - return self._get_from_testrail('get_plan', plan_id) - - def get_plans(self, project_id): - return self._get_from_testrail('get_plans', project_id) - - def add_plan(self, project_id, name, description=None, milestone_id=None, entries=None): - extra_params = {'name': name} - if description: - extra_params['description'] = description - if milestone_id: - extra_params['milestone_id'] = milestone_id - if entries: - extra_params['entries'] = entries - return self._add_to_testrail('add_plan', project_id, extra_params) - - def add_plan_entry(self, plan_id, suite_id, name, assigned_to_id=None, include_all=True, case_ids=None): - extra_params = {'suite_id': suite_id, 'name': name} - if assigned_to_id: - extra_params['assignedto_id'] = assigned_to_id - if not include_all and case_ids: - extra_params['include_all'] = False - extra_params['case_ids'] = case_ids - - return self._add_to_testrail('add_plan_entry', plan_id, extra_params) - - def update_plan(self, plan_id, name, description=None, milestone_id=None): - extra_params = {'name': name} - if description: - extra_params['description'] = description - if milestone_id: - extra_params['milestone_id'] = milestone_id - return self._add_to_testrail('update_plan', plan_id, extra_params) - - def update_plan_entry(self, plan_id, entry_id, include_all=True, case_ids=None): - extra_params = dict() - if not include_all and case_ids: - extra_params['include_all'] = False - extra_params['case_ids'] = case_ids - - return self._add_to_testrail('update_plan_entry', plan_id, extra_params, entry_id) - - def close_plan(self, plan_id): - return self._add_to_testrail('close_plan', plan_id) - - def delete_plan(self, plan_id): - return self._add_to_testrail('delete_plan', plan_id) - - def get_priorities(self): - return self._get_from_testrail("get_priorities") - - def get_project(self, project_id): - return self._get_from_testrail('get_project', project_id) - - def get_projects(self): - return self._get_from_testrail("get_projects") - - def get_project_by_name(self, name): - projects = [project for project in self.projects if project['name'] == name] - if not projects or len(projects) > 1: - raise Exception("No or multiple projects found with name: {0} ".format(name)) - return projects[0] - - def get_result_fields(self): - return self._get_from_testrail("get_result_fields") - - def get_run(self, run_id): - return self._get_from_testrail('get_run', run_id) - - def get_runs(self, project_id): - return self._get_from_testrail('get_runs', project_id) - - def add_run(self, project_id, suite_id, name, description=None, assigned_to_id=None, include_all=True, - case_ids=None): - extra_params = {'suite_id': suite_id, 'name': name} - if description: - extra_params['description'] = description - if assigned_to_id: - extra_params['assignedto_id'] = assigned_to_id - if not include_all and case_ids: - extra_params['include_all'] = False - extra_params['case_ids'] = case_ids - return self._add_to_testrail('add_run', project_id, extra_params) - - def update_run(self, run_id, name, description=None, include_all=True, case_ids=None): - extra_params = {'name': name} - if description: - extra_params['description'] = description - if not include_all and case_ids: - extra_params['include_all'] = False - extra_params['case_ids'] = case_ids - return self._add_to_testrail('update_run', run_id, extra_params) - - def close_run(self, run_id): - return self._add_to_testrail('close_run', run_id) - - def delete_run(self, run_id): - return self._add_to_testrail('delete_run', run_id) - - def get_test(self, test_id): - return self._get_from_testrail('get_test', test_id) - - def get_tests(self, run_id): - return self._get_from_testrail('get_tests', run_id) - - def get_results(self, test_id, limit=None): - extra_params = {'limit': limit} if limit else None - return self._get_from_testrail('get_results', test_id, extra_params) - - def add_result(self, test_id, status_id, comment=None, version=None, elapsed=None, defects=None, - assigned_to_id=None, custom_fields=None): - extra_params = {'status_id': status_id} - if comment: - extra_params['comment'] = comment - if version: - extra_params['version'] = version - if elapsed: - extra_params['elapsed'] = elapsed - if defects: - extra_params['defects'] = defects - if assigned_to_id: - extra_params['assignedto_id'] = assigned_to_id - if custom_fields: - for key, value in custom_fields.iteritems(): - assert "custom_" in key, "Custom fields need to start with 'custom_'" - extra_params[key] = value - return self._add_to_testrail('add_result', test_id, extra_params) - - def add_result_for_case(self, run_id, case_id, status_id, comment=None, version=None, elapsed=None, defects=None, - assigned_to_id=None, custom_fields=None): - extra_params = {'status_id': status_id} - if comment: - extra_params['comment'] = comment - if version: - extra_params['version'] = version - if elapsed: - extra_params['elapsed'] = elapsed - if defects: - extra_params['defects'] = defects - if assigned_to_id: - extra_params['assignedto_id'] = assigned_to_id - if custom_fields: - for key, value in custom_fields.iteritems(): - assert "custom_" in key, "Custom fields need to start with 'custom_'" - extra_params[key] = value - return self._add_to_testrail('add_result_for_case', "%s/%s" % (run_id, case_id), extra_params) - - def get_statuses(self): - return self._get_from_testrail('get_statuses') - - def get_milestone(self, milestone_id): - return self._get_from_testrail('get_milestone', milestone_id) - - def get_milestones(self, project_id): - return self._get_from_testrail('get_milestones', project_id) - - def get_milestone_by_name(self, project_id, name): - milestones = [milestone for milestone in self.get_milestones(project_id) if milestone['name'] == name] - if not milestones or len(milestones) > 1: - raise Exception("No or multiple suites found with name: {0} ".format(name)) - return milestones[0] - - def get_case_by_name(self, project_id, suite_id, name, section_id=None): - cases = [case for case in self.get_cases(project_id, suite_id, section_id) if case['title'] == name] - if not cases or len(cases) > 1: - raise Exception("No or multiple cases found with name: {0} ".format(name)) - return cases[0] - - def get_test_by_name(self, run_id, name): - tests = [test for test in self.get_tests(run_id) if test['title'] == name] - if not tests or len(tests) > 1: - raise Exception("No or multiple tests found with name: {0} ".format(name)) - return tests[0] - diff --git a/helpers/thread.py b/helpers/thread.py index d752ee2..dcec0ab 100644 --- a/helpers/thread.py +++ b/helpers/thread.py @@ -25,7 +25,8 @@ class ThreadHelper(object): @staticmethod def start_thread_with_event(target, name, args=(), kwargs=None): """ - Starts a thread and an event to it + Starts a thread and an event to it. + The passed target function needs to accept an param 'event' which will contain the stopEvent object :param target: target - usually a method :type target: object :param name: name of the thread @@ -37,9 +38,11 @@ def start_thread_with_event(target, name, args=(), kwargs=None): """ if kwargs is None: kwargs = {} + if 'event' in kwargs: + raise ValueError('event is a reserved keyword of this function') ThreadHelper.LOGGER.info('Starting thread with target {0}'.format(target)) event = threading.Event() - args = args + (event,) + kwargs['event'] = event thread = threading.Thread(target=target, args=tuple(args), kwargs=kwargs) thread.setName(str(name)) thread.setDaemon(True) diff --git a/helpers/vpool.py b/helpers/vpool.py index 56ab63a..923bc4f 100644 --- a/helpers/vpool.py +++ b/helpers/vpool.py @@ -14,12 +14,11 @@ # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. -from ovs.log.log_handler import LogHandler from ovs.dal.lists.vpoollist import VPoolList +from ovs.log.log_handler import LogHandler from ..helpers.exceptions import VPoolNotFoundError - class VPoolHelper(object): """ BackendHelper class diff --git a/remove/arakoon.py b/remove/arakoon.py index 6ad8b91..95ac1b6 100644 --- a/remove/arakoon.py +++ b/remove/arakoon.py @@ -14,13 +14,12 @@ # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. -from ovs.log.log_handler import LogHandler from ovs.extensions.db.arakooninstaller import ArakoonInstaller +from ovs.log.log_handler import LogHandler from ..validate.decorators import required_arakoon_cluster class ArakoonRemover(object): - LOGGER = LogHandler.get(source="remove", name="ci_arakoon_remover") def __init__(self): diff --git a/remove/backend.py b/remove/backend.py index bb0deec..95e2950 100644 --- a/remove/backend.py +++ b/remove/backend.py @@ -16,26 +16,28 @@ from ovs.log.log_handler import LogHandler from ..helpers.albanode import AlbaNodeHelper from ..helpers.backend import BackendHelper +from ..helpers.ci_constants import CIConstants from ..validate.decorators import required_backend, required_preset -class BackendRemover(object): +class BackendRemover(CIConstants): LOGGER = LogHandler.get(source="remove", name="ci_backend_remover") REMOVE_ASD_TIMEOUT = 60 REMOVE_DISK_TIMEOUT = 60 REMOVE_BACKEND_TIMEOUT = 60 REMOVE_PRESET_TIMEOUT = 60 + UNLINK_BACKEND_TIMEOUT = 60 def __init__(self): pass - @staticmethod - def remove_claimed_disk(api): + @classmethod + def remove_claimed_disk(cls): pass - @staticmethod - def remove_asds(albabackend_name, target, disks, api): + @classmethod + def remove_asds(cls, albabackend_name, target, disks, *args, **kwargs): """ Remove all asds from a backend @@ -43,8 +45,6 @@ def remove_asds(albabackend_name, target, disks, api): :type target: str :param disks: dict with diskname as key and amount of osds as value :type disks: dict - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param albabackend_name: Name of the AlbaBackend to configure :type albabackend_name: str :return: preset_name @@ -53,9 +53,9 @@ def remove_asds(albabackend_name, target, disks, api): albabackend_guid = BackendHelper.get_alba_backend_guid_by_name(albabackend_name) # target is a node - node_mapping = AlbaNodeHelper._map_alba_nodes(api) + node_mapping = AlbaNodeHelper._map_alba_nodes() - local_stack = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name, api=api) + local_stack = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name) for disk, amount_of_osds in disks.iteritems(): disk_object = AlbaNodeHelper.get_disk_by_ip(ip=target, diskname=disk) # Get the name of the disk out of the path, only expecting one with ata- @@ -64,15 +64,13 @@ def remove_asds(albabackend_name, target, disks, api): # Check if the alba_node_id has the disk if disk_path in local_stack['local_stack'][alba_node_id]: # Remove asds - if disk_path in local_stack['local_stack'][alba_node_id]: - for asd_id, asd_info in local_stack['local_stack'][alba_node_id][disk_path]['asds'].iteritems(): - BackendRemover.LOGGER.info('Removing asd {0} for disk {1}'.format(asd_id, local_stack['local_stack'][alba_node_id][disk_path]['guid'])) - asd_safety = BackendHelper.get_asd_safety(albabackend_guid=albabackend_guid, asd_id=asd_id, api=api) - BackendRemover._remove_asd(alba_node_guid=alba_node_guid, asd_id=asd_id, asd_safety=asd_safety, api=api) + for asd_id, asd_info in local_stack['local_stack'][alba_node_id][disk_path]['osds'].iteritems(): + BackendRemover.LOGGER.info('Removing asd {0} for disk {1}'.format(asd_id, disk_path)) + asd_safety = BackendHelper.get_asd_safety(albabackend_guid=albabackend_guid, asd_id=asd_id) + BackendRemover._remove_asd(alba_node_guid=alba_node_guid, asd_id=asd_id, asd_safety=asd_safety) # Restarting iteration to avoid too many local stack calls: - local_stack = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name, - api=api) + local_stack = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name) for disk, amount_of_osds in disks.iteritems(): disk_object = AlbaNodeHelper.get_disk_by_ip(ip=target, diskname=disk) # Get the name of the disk out of the path, only expecting one with ata- @@ -82,10 +80,10 @@ def remove_asds(albabackend_name, target, disks, api): if disk_path in local_stack['local_stack'][alba_node_id]: # Initialize disk: BackendRemover.LOGGER.info('Removing {0}.'.format(disk_path)) - BackendRemover._remove_disk(alba_node_guid=alba_node_guid, diskname=disk_path, api=api) + BackendRemover._remove_disk(alba_node_guid=alba_node_guid, diskname=disk_path) - @staticmethod - def _remove_asd(alba_node_guid, asd_id, asd_safety, api, timeout=REMOVE_ASD_TIMEOUT): + @classmethod + def _remove_asd(cls, alba_node_guid, asd_id, asd_safety, timeout=REMOVE_ASD_TIMEOUT, *args, **kwargs): """ Remove a asd from a backend @@ -95,8 +93,6 @@ def _remove_asd(alba_node_guid, asd_id, asd_safety, api, timeout=REMOVE_ASD_TIME :type asd_id: str :param asd_safety: :type asd_safety: dict - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: max. time to wait for a task to complete :type timeout: int :return: @@ -106,19 +102,19 @@ def _remove_asd(alba_node_guid, asd_id, asd_safety, api, timeout=REMOVE_ASD_TIME 'asd_id': asd_id, 'safety': asd_safety } - task_guid = api.post( + task_guid = cls.api.post( api='/alba/nodes/{0}/reset_asd/'.format(alba_node_guid), data=data ) - result = api.wait_for_task(task_id=task_guid, timeout=timeout) + result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if result[0] is False: error_msg = "Removal of ASD '{0}; failed with {1}".format(asd_id, result[1]) BackendRemover.LOGGER.error(error_msg) raise RuntimeError(error_msg) return result[0] - @staticmethod - def _remove_disk(alba_node_guid, diskname, api, timeout=REMOVE_DISK_TIMEOUT): + @classmethod + def _remove_disk(cls, alba_node_guid, diskname, timeout=REMOVE_DISK_TIMEOUT, *args, **kwargs): """ Removes a an initiliazed disk from the model @@ -126,8 +122,6 @@ def _remove_disk(alba_node_guid, diskname, api, timeout=REMOVE_DISK_TIMEOUT): :type alba_node_guid: str :param diskname: name of the disk :type diskname: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: max. time to wait for the task to complete :type timeout: int :return: @@ -135,27 +129,25 @@ def _remove_disk(alba_node_guid, diskname, api, timeout=REMOVE_DISK_TIMEOUT): data = { 'disk': diskname, } - task_guid = api.post( + task_guid = cls.api.post( api='/alba/nodes/{0}/remove_disk/'.format(alba_node_guid), data=data ) - result = api.wait_for_task(task_id=task_guid, timeout=timeout) + result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if result[0] is False: errormsg = "Removal of ASD '{0}' failed with '{1}'".format(diskname, result[1]) BackendRemover.LOGGER.error(errormsg) raise RuntimeError(errormsg) return result[0] - @staticmethod + @classmethod @required_backend - def remove_backend(albabackend_name, api, timeout=REMOVE_BACKEND_TIMEOUT): + def remove_backend(cls, albabackend_name, timeout=REMOVE_BACKEND_TIMEOUT, *args, **kwargs): """ Removes a alba backend from the ovs cluster :param albabackend_name: the name of a existing alba backend :type albabackend_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: max. time to wait for a task to complete :type timeout: int :return: task was succesfull or not @@ -163,20 +155,19 @@ def remove_backend(albabackend_name, api, timeout=REMOVE_BACKEND_TIMEOUT): """ alba_backend_guid = BackendHelper.get_alba_backend_guid_by_name(albabackend_name) - task_guid = api.delete(api='/alba/backends/{0}'.format(alba_backend_guid)) - - result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_guid = cls.api.delete(api='/alba/backends/{0}'.format(alba_backend_guid)) + result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if result[0] is False: errormsg = "Removal of backend '{0}' failed with '{1}'".format(albabackend_name, result[1]) BackendRemover.LOGGER.error(errormsg) raise RuntimeError(errormsg) return result[0] - @staticmethod + @classmethod @required_preset @required_backend - def remove_preset(preset_name, albabackend_name, api, timeout=REMOVE_PRESET_TIMEOUT): + def remove_preset(cls, preset_name, albabackend_name, timeout=REMOVE_PRESET_TIMEOUT, *args, **kwargs): """ Removes a alba backend from the ovs cluster @@ -184,8 +175,6 @@ def remove_preset(preset_name, albabackend_name, api, timeout=REMOVE_PRESET_TIME :type preset_name: str :param albabackend_name: name of the albabackend :type albabackend_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: max. time to wait for a task to complete :type timeout: int :return: task was succesfull or not @@ -194,12 +183,48 @@ def remove_preset(preset_name, albabackend_name, api, timeout=REMOVE_PRESET_TIME alba_backend_guid = BackendHelper.get_alba_backend_guid_by_name(albabackend_name) data = {"name": preset_name} - task_guid = api.post(api='/alba/backends/{0}/delete_preset'.format(alba_backend_guid), data=data) + task_guid = cls.api.post(api='/alba/backends/{0}/delete_preset'.format(alba_backend_guid), data=data) - result = api.wait_for_task(task_id=task_guid, timeout=timeout) + result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if result[0] is False: errormsg = "Removal of preset '{0}' for backend '{1}' failed with '{2}'".format(preset_name, albabackend_name, result[1]) BackendRemover.LOGGER.error(errormsg) raise RuntimeError(errormsg) return result[0] + + + @classmethod + #@required_backend + def unlink_backend(cls, globalbackend_name, albabackend_name, timeout=UNLINK_BACKEND_TIMEOUT, *args, **kwargs): + """ + Link a LOCAL backend to a GLOBAL backend + + :param globalbackend_name: name of a GLOBAL alba backend + :type globalbackend_name: str + :param albabackend_name: name of a backend to unlink + :type albabackend_name: str + :param timeout: timeout counter in seconds + :type timeout: int + :return: + """ + data = { + "linked_guid": BackendHelper.get_alba_backend_guid_by_name(albabackend_name) + } + + task_guid = cls.api.post( + api='/alba/backends/{0}/unlink_alba_backends' + .format(BackendHelper.get_alba_backend_guid_by_name(globalbackend_name)), + data=data + ) + + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) + if not task_result[0]: + error_msg = "Unlinking backend `{0}` from global backend `{1}` has failed with error '{2}'".format( + albabackend_name, globalbackend_name, task_result[1]) + BackendRemover.LOGGER.error(error_msg) + raise RuntimeError(error_msg) + else: + BackendRemover.LOGGER.info("Unlinking backend `{0}` from global backend `{1}` should have succeeded" + .format(albabackend_name, globalbackend_name)) + return task_result[0] diff --git a/remove/roles.py b/remove/roles.py index a82c55b..f4faf50 100644 --- a/remove/roles.py +++ b/remove/roles.py @@ -13,85 +13,97 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. -from subprocess import check_output + +from ovs.extensions.generic.system import System +from ovs.extensions.generic.sshclient import SSHClient from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants from ..helpers.fstab import FstabHelper from ..helpers.storagerouter import StoragerouterHelper from ..setup.roles import RoleSetup -class RoleRemover(object): +class RoleRemover(CIConstants): LOGGER = LogHandler.get(source="remove", name="ci_role_remover") CONFIGURE_DISK_TIMEOUT = 300 @staticmethod - def _umount(mountpoint): + def _umount(mountpoint, client=None): """ Unmount the given partition :param mountpoint: Location where the mountpoint is mounted :type mountpoint: str :return: """ + if client is None: + client = SSHClient(System.get_my_storagerouter(), username='root') try: - check_output('umount {0}'.format(mountpoint), shell=True) + client.run(['umount', mountpoint]) except Exception: RoleRemover.LOGGER.exception('Unable to umount mountpoint {0}'.format(mountpoint)) raise RuntimeError('Could not unmount {0}'.format(mountpoint)) @staticmethod - def _remove_filesystem(device, alias_part_label): + def _remove_filesystem(device, alias_part_label, client=None): """ :param alias_part_label: eg /dev/disk/by-partlabel/ata-QEMU_HARDDISK_QM00011 :type alias_part_label: str :return: """ + if client is None: + client = SSHClient(System.get_my_storagerouter(), username='root') try: partition_cmd = "udevadm info --name={0} | awk -F '=' '/ID_PART_ENTRY_NUMBER/{{print $NF}}'".format(alias_part_label) - partition_number = check_output(partition_cmd, shell=True) - format_cmd = 'parted {0} rm {1}'.format(device, partition_number) - check_output(format_cmd, shell=True) + partition_number = client.run(partition_cmd, allow_insecure=True) + if partition_number: + format_cmd = 'parted {0} rm {1}'.format(device, partition_number) + client.run(format_cmd.split()) except Exception: RoleRemover.LOGGER.exception('Unable to remove filesystem of {0}'.format(alias_part_label)) raise RuntimeError('Could not remove filesystem of {0}'.format(alias_part_label)) - @staticmethod - def remove_role(ip, diskname, api): - allowed_roles = ['WRITE', 'READ', 'SCRUB', 'DB'] - RoleRemover.LOGGER.info("Starting removal of disk roles.") - + @classmethod + def remove_role(cls, storagerouter_ip, diskname, *args, **kwargs): + allowed_roles = ['WRITE', 'DTL', 'SCRUB', 'DB'] + cls.LOGGER.info("Starting removal of disk roles.") # Fetch information - storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(ip) - disk = StoragerouterHelper.get_disk_by_ip(ip, diskname) + + storagerouter = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip=storagerouter_ip) + disk = StoragerouterHelper.get_disk_by_name(guid=storagerouter.guid, diskname=diskname) # Check if there are any partitions on the disk, if so check if there is enough space + client = SSHClient(storagerouter, username='root') + if len(disk.partitions) > 0: for partition in disk.partitions: # Remove all partitions that have roles if set(partition.roles).issubset(allowed_roles) and len(partition.roles) > 0: - RoleRemover.LOGGER.info("Removing {0} from partition {1} on disk {2}".format(partition.roles, partition.guid, diskname)) - RoleSetup.configure_disk(storagerouter_guid=storagerouter_guid, + cls.LOGGER.info("Removing {0} from partition {1} on disk {2}".format(partition.roles, partition.guid, diskname)) + RoleSetup.configure_disk(storagerouter_guid=storagerouter.guid, disk_guid=disk.guid, offset=partition.offset, size=disk.size, roles=[], - api=api, partition_guid=partition.guid) - # Unmount partition - RoleRemover.LOGGER.info("Umounting disk {2}".format(partition.roles, partition.guid, diskname)) - RoleRemover._umount(partition.mountpoint) + + + cls._umount(partition.mountpoint, client=client) # Remove from fstab - RoleRemover.LOGGER.info("Removing {0} from fstab".format(partition.mountpoint, partition.guid, diskname)) - FstabHelper().remove_by_mountpoint(partition.mountpoint) + + cls.LOGGER.info("Removing {0} from fstab".format(partition.mountpoint, partition.guid, diskname)) + FstabHelper(client=client).remove_by_mountpoint(partition.mountpoint,client) # Remove filesystem - RoleRemover.LOGGER.info("Removing filesystem on partition {0} on disk {1}".format(partition.guid, diskname)) + cls.LOGGER.info("Removing filesystem on partition {0} on disk {1}".format(partition.guid, diskname)) alias = partition.aliases[0] device = '/dev/{0}'.format(diskname) - RoleRemover._remove_filesystem(device, alias) + cls._remove_filesystem(device, alias,client=client) # Remove partition from model - RoleRemover.LOGGER.info("Removing partition {0} on disk {1} from model".format(partition.guid, diskname)) + cls.LOGGER.info("Removing partition {0} on disk {1} from model".format(partition.guid, diskname)) partition.delete() else: - RoleRemover.LOGGER.info("Found no roles on partition {1} on disk {2}".format(partition.roles, partition.guid, diskname)) + print 'Found no roles on partition' + RoleRemover.LOGGER.info("{1} on disk {2}".format(partition.roles, partition.guid, diskname)) else: + print 'found no partition' RoleRemover.LOGGER.info("Found no partition on the disk.") diff --git a/remove/vdisk.py b/remove/vdisk.py index fb19245..7c8664a 100644 --- a/remove/vdisk.py +++ b/remove/vdisk.py @@ -15,11 +15,12 @@ # but WITHOUT ANY WARRANTY of any kind. from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants from ..helpers.vdisk import VDiskHelper from ..validate.decorators import required_vtemplate -class VDiskRemover(object): +class VDiskRemover(CIConstants): LOGGER = LogHandler.get(source="remove", name="ci_vdisk_remover") REMOVE_SNAPSHOT_TIMEOUT = 60 @@ -29,15 +30,13 @@ class VDiskRemover(object): def __init__(self): pass - @staticmethod - def remove_vdisks_with_structure(vdisks, api, timeout=REMOVE_VDISK_TIMEOUT): + @classmethod + def remove_vdisks_with_structure(cls, vdisks, timeout=REMOVE_VDISK_TIMEOUT, *args, **kwargs): """ Remove many vdisks at once. Will keep the parent structure in mind :param vdisks: list of vdisks - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: seconds to elapse before raising a timeout error (for each volume) - :return: + :return: """ removed_guids = [] for vdisk in vdisks: @@ -45,13 +44,13 @@ def remove_vdisks_with_structure(vdisks, api, timeout=REMOVE_VDISK_TIMEOUT): continue if len(vdisk.child_vdisks_guids) > 0: for vdisk_child_guid in vdisk.child_vdisks_guids: - VDiskRemover.remove_vdisk(vdisk_child_guid, api) + VDiskRemover.remove_vdisk(vdisk_child_guid) removed_guids.append(vdisk_child_guid) - VDiskRemover.remove_vdisk(vdisk.guid, api, timeout) + VDiskRemover.remove_vdisk(vdisk.guid, timeout) removed_guids.append(vdisk.guid) - @staticmethod - def remove_snapshot(snapshot_guid, vdisk_name, vpool_name, api, timeout=REMOVE_SNAPSHOT_TIMEOUT): + @classmethod + def remove_snapshot(cls, snapshot_guid, vdisk_name, vpool_name, timeout=REMOVE_SNAPSHOT_TIMEOUT, *args, **kwargs): """ Remove a existing snapshot from a existing vdisk :param vdisk_name: location of a vdisk on a vpool @@ -59,8 +58,6 @@ def remove_snapshot(snapshot_guid, vdisk_name, vpool_name, api, timeout=REMOVE_S :type vdisk_name: str :param snapshot_guid: unique guid of a snapshot :type snapshot_guid: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :param vpool_name: name of a existing vpool @@ -71,11 +68,11 @@ def remove_snapshot(snapshot_guid, vdisk_name, vpool_name, api, timeout=REMOVE_S vdisk_guid = VDiskHelper.get_vdisk_by_name(vdisk_name, vpool_name).guid data = {"snapshot_id": snapshot_guid} - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/remove_snapshot/'.format(vdisk_guid), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Deleting snapshot `{0}` for vdisk `{1}` has failed".format(snapshot_guid, vdisk_name) @@ -86,21 +83,19 @@ def remove_snapshot(snapshot_guid, vdisk_name, vpool_name, api, timeout=REMOVE_S .format(snapshot_guid, vdisk_name)) return True - @staticmethod - def remove_vdisk(vdisk_guid, api, timeout=REMOVE_VDISK_TIMEOUT): + @classmethod + def remove_vdisk(cls, vdisk_guid, timeout=REMOVE_VDISK_TIMEOUT, *args, **kwargs): """ Remove a vdisk from a vPool :param vdisk_guid: guid of a existing vdisk :type vdisk_guid: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :return: if success :rtype: bool """ - task_guid = api.post(api='vdisks/{0}/delete'.format(vdisk_guid)) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_guid = cls.api.post(api='vdisks/{0}/delete'.format(vdisk_guid)) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Deleting vDisk `{0}` has failed".format(vdisk_guid) VDiskRemover.LOGGER.error(error_msg) @@ -109,43 +104,39 @@ def remove_vdisk(vdisk_guid, api, timeout=REMOVE_VDISK_TIMEOUT): VDiskRemover.LOGGER.info("Deleting vDisk `{0}` should have succeeded".format(vdisk_guid)) return True - @staticmethod - def remove_vdisk_by_name(vdisk_name, vpool_name, api, timeout=REMOVE_VDISK_TIMEOUT): + @classmethod + def remove_vdisk_by_name(cls, vdisk_name, vpool_name, timeout=REMOVE_VDISK_TIMEOUT, *args, **kwargs): """ Remove a vdisk from a vPool :param vdisk_name: name of a existing vdisk (e.g. test.raw) :type vdisk_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param vpool_name: name of a existing vpool :type vpool_name: str :return: if success :rtype: bool """ vdisk_guid = VDiskHelper.get_vdisk_by_name(vdisk_name, vpool_name).guid - return VDiskRemover.remove_vdisk(vdisk_guid, api, timeout) + return VDiskRemover.remove_vdisk(vdisk_guid, timeout) - @staticmethod + @classmethod @required_vtemplate - def remove_vtemplate_by_name(vdisk_name, vpool_name, api, timeout=REMOVE_VTEMPLATE_TIMEOUT): + def remove_vtemplate_by_name(cls, vdisk_name, vpool_name, timeout=REMOVE_VTEMPLATE_TIMEOUT, *args, **kwargs): """ Remove a vTemplate from a cluster :param vdisk_name: name of a existing vdisk (e.g. test.raw) :type vdisk_name: str :param vpool_name: name of a existing vpool :type vpool_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :return: if success :rtype: bool """ vdisk_guid = VDiskHelper.get_vdisk_by_name(vdisk_name, vpool_name).guid - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/delete_vtemplate/'.format(vdisk_guid) ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Deleting vTemplate `{0}` has failed".format(vdisk_name) diff --git a/remove/vpool.py b/remove/vpool.py index 26ba4c3..e8bf1b6 100644 --- a/remove/vpool.py +++ b/remove/vpool.py @@ -15,25 +15,24 @@ # but WITHOUT ANY WARRANTY of any kind. from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants from ..helpers.storagerouter import StoragerouterHelper from ..helpers.vpool import VPoolHelper -class VPoolRemover(object): +class VPoolRemover(CIConstants): LOGGER = LogHandler.get(source="remove", name="ci_vpool_remover") REMOVE_VPOOL_TIMEOUT = 500 - @staticmethod - def remove_vpool(vpool_name, storagerouter_ip, api, timeout=REMOVE_VPOOL_TIMEOUT): + @classmethod + def remove_vpool(cls, vpool_name, storagerouter_ip, timeout=REMOVE_VPOOL_TIMEOUT, *args, **kwargs): """ Removes a existing vpool from a storagerouter :param vpool_name: the name of a existing vpool :type vpool_name: str :param storagerouter_ip: the ip address of a existing storagerouter :type storagerouter_ip: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: max. time to wait for a task to complete :type timeout: int :return: None @@ -42,9 +41,8 @@ def remove_vpool(vpool_name, storagerouter_ip, api, timeout=REMOVE_VPOOL_TIMEOUT vpool_guid = VPoolHelper.get_vpool_by_name(vpool_name).guid storagerouter_guid = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip).guid data = {"storagerouter_guid": storagerouter_guid} - task_guid = api.post(api='/vpools/{0}/shrink_vpool/'.format(vpool_guid), data=data) - - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_guid = cls.api.post(api='/vpools/{0}/shrink_vpool/'.format(vpool_guid), data=data) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Deleting vPool `{0}` on storagerouter `{1}` has failed with error {2}".format(vpool_name, storagerouter_ip, task_result[1]) diff --git a/setup/arakoon.py b/setup/arakoon.py index 2245df6..a0381f1 100644 --- a/setup/arakoon.py +++ b/setup/arakoon.py @@ -16,11 +16,12 @@ from ovs.dal.hybrids.servicetype import ServiceType from ovs.extensions.db.arakooninstaller import ArakoonInstaller +from ovs.log.log_handler import LogHandler from ovs.extensions.generic.sshclient import SSHClient from ovs.lib.alba import AlbaController -from ovs.log.log_handler import LogHandler from ..helpers.backend import BackendHelper from ..validate.decorators import required_backend, required_arakoon_cluster +from ..validate.backend import BackendValidation class ArakoonSetup(object): @@ -76,9 +77,7 @@ def add_arakoon(cluster_name, storagerouter_ip, cluster_basedir, base_dir=cluster_basedir, plugins=plugins, locked=False, - internal=False, - log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server'), - crash_log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server_crash')) + internal=False) if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: client.run(['ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)]) elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: @@ -129,9 +128,7 @@ def extend_arakoon(cluster_name, master_storagerouter_ip, storagerouter_ip, clus arakoon_installer.load() arakoon_installer.extend_cluster(new_ip=storagerouter_ip, base_dir=cluster_basedir, - locked=False, - log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server'), - crash_log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server_crash')) + locked=False) if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: client.run(['ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)]) elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: @@ -159,3 +156,44 @@ def checkup_nsm_hosts(albabackend_name, amount): """ alba_backend_guid = BackendHelper.get_alba_backend_guid_by_name(albabackend_name) return AlbaController.nsm_checkup(backend_guid=alba_backend_guid, min_nsms=int(amount)) + + + @staticmethod + def setup_external_arakoons(backend): + """ + Setup external arakoons for a backend + + :param backend: all backend details + :type backend: dict + :return: mapped external arakoons + :rtype: dict + """ + + # if backend does not exists, deploy the external arakoons + if not BackendValidation.check_backend(backend_name=backend['name']): + external_arakoon_mapping = {} + for ip, arakoons in backend['external_arakoon'].iteritems(): + for arakoon_name, arakoon_settings in arakoons.iteritems(): + # check if we already created one or not + if arakoon_name not in external_arakoon_mapping: + # if not created yet, create one and map it + external_arakoon_mapping[arakoon_name] = {} + external_arakoon_mapping[arakoon_name]['master'] = ip + external_arakoon_mapping[arakoon_name]['all'] = [ip] + ArakoonSetup.add_arakoon(cluster_name=arakoon_name, storagerouter_ip=ip, + cluster_basedir=arakoon_settings['base_dir'], + service_type=arakoon_settings['type']) + else: + # if created, extend it and map it + external_arakoon_mapping[arakoon_name]['all'].append(ip) + ArakoonSetup.extend_arakoon(cluster_name=arakoon_name, + master_storagerouter_ip=external_arakoon_mapping[arakoon_name]['master'], + storagerouter_ip=ip, + cluster_basedir=arakoon_settings['base_dir'], + service_type=arakoon_settings['type'], + clustered_nodes=external_arakoon_mapping[arakoon_name]['all']) + return external_arakoon_mapping + else: + ArakoonSetup.LOGGER.info("Skipping external arakoon creation because backend `{0}` already exists" + .format(backend['name'])) + return diff --git a/setup/backend.py b/setup/backend.py index 964540d..64179b7 100644 --- a/setup/backend.py +++ b/setup/backend.py @@ -17,11 +17,12 @@ from ovs.log.log_handler import LogHandler from ..helpers.albanode import AlbaNodeHelper from ..helpers.backend import BackendHelper +from ..helpers.ci_constants import CIConstants from ..validate.decorators import required_roles, required_backend, required_preset, check_backend, check_preset, \ check_linked_backend, filter_osds -class BackendSetup(object): +class BackendSetup(CIConstants): LOGGER = LogHandler.get(source="setup", name="ci_backend_setup") LOCAL_STACK_SYNC = 30 @@ -32,27 +33,23 @@ class BackendSetup(object): CLAIM_ASD_TIMEOUT = 60 LINK_BACKEND_TIMEOUT = 60 MAX_BACKEND_TRIES = 20 - MAX_CLAIM_RETRIES = 5 def __init__(self): pass - @staticmethod + @classmethod @check_backend @required_roles(['DB']) - def add_backend(backend_name, api, scaling='LOCAL', timeout=BACKEND_TIMEOUT, max_tries=MAX_BACKEND_TRIES): + def add_backend(cls, backend_name, scaling='LOCAL', timeout=BACKEND_TIMEOUT, max_tries=MAX_BACKEND_TRIES, *args, **kwargs): """ Add a new backend - :param backend_name: Name of the Backend to add :type backend_name: str :param scaling: LOCAL or GLOBAL :type scaling: str :return: backend_name :rtype: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: timeout between tries :type timeout: int :param max_tries: amount of max. tries to check if a backend has been successfully created @@ -60,9 +57,8 @@ def add_backend(backend_name, api, scaling='LOCAL', timeout=BACKEND_TIMEOUT, max :returns: creation is successfully succeeded? :rtype: bool """ - # ADD_BACKEND - backend = api.post( + backend = cls.api.post( api='backends', data={ 'name': backend_name, @@ -72,7 +68,7 @@ def add_backend(backend_name, api, scaling='LOCAL', timeout=BACKEND_TIMEOUT, max ) # ADD_ALBABACKEND - api.post(api='alba/backends', data={'backend_guid': backend['guid'], 'scaling': scaling}) + cls.api.post(api='alba/backends', data={'backend_guid': backend['guid'], 'scaling': scaling}) # CHECK_STATUS until done backend_running_status = "RUNNING" @@ -95,17 +91,15 @@ def add_backend(backend_name, api, scaling='LOCAL', timeout=BACKEND_TIMEOUT, max time.sleep(timeout) BackendSetup.LOGGER.error("Creation of Backend `{0}` and scaling `{1}` failed with status: {2}!" - .format(backend_name, scaling, - BackendHelper.get_backend_status_by_name(backend_name))) + .format(backend_name, scaling, BackendHelper.get_backend_status_by_name(backend_name))) return False - @staticmethod + @classmethod @check_preset @required_backend - def add_preset(albabackend_name, preset_details, api, timeout=ADD_PRESET_TIMEOUT): + def add_preset(cls, albabackend_name, preset_details, timeout=ADD_PRESET_TIMEOUT, *args, **kwargs): """ Add a new preset - :param albabackend_name: albabackend name (e.g. 'mybackend') :type albabackend_name: str :param preset_details: dictionary with details of a preset e.g. @@ -121,14 +115,11 @@ def add_preset(albabackend_name, preset_details, api, timeout=ADD_PRESET_TIMEOUT "fragment_size": 2097152 } :type preset_details: dict - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: amount of max time that preset may take to be added :type timeout: int :return: success or not :rtype: bool """ - # BUILD_PRESET preset = {'name': preset_details['name'], 'policies': preset_details['policies'], @@ -137,51 +128,45 @@ def add_preset(albabackend_name, preset_details, api, timeout=ADD_PRESET_TIMEOUT 'fragment_size': preset_details['fragment_size']} # ADD_PRESET - task_guid = api.post( + task_guid = cls.api.post( api='/alba/backends/{0}/add_preset'.format(BackendHelper.get_alba_backend_guid_by_name(albabackend_name)), data=preset ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = "Preset `{0}` has failed to create on backend `{1}`"\ - .format(preset_details['name'], albabackend_name) + error_msg = "Preset `{0}` has failed to create on backend `{1}`".format(preset_details['name'], albabackend_name) BackendSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: - BackendSetup.LOGGER.info("Creation of preset `{0}` should have succeeded on backend `{1}`" - .format(preset_details['name'], albabackend_name)) + BackendSetup.LOGGER.info("Creation of preset `{0}` should have succeeded on backend `{1}`".format(preset_details['name'], albabackend_name)) return True - @staticmethod + @classmethod @required_preset @required_backend - def update_preset(albabackend_name, preset_name, policies, api, timeout=UPDATE_PRESET_TIMEOUT): + def update_preset(cls, albabackend_name, preset_name, policies, timeout=UPDATE_PRESET_TIMEOUT, *args, **kwargs): """ Update a existing preset - :param albabackend_name: albabackend name :type albabackend_name: str :param preset_name: name of a existing preset :type preset_name: str :param policies: policies to be updated (e.g. [[1,1,2,2], [1,1,1,2]]) :type policies: list > list - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: amount of max time that preset may take to be added :type timeout: int :return: success or not :rtype: bool """ - - task_guid = api.post( + task_guid = cls.api.post( api='/alba/backends/{0}/update_preset' .format(BackendHelper.get_alba_backend_guid_by_name(albabackend_name)), data={"name": preset_name, "policies": policies} ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Preset `{0}` has failed to update with policies `{1}` on backend `{2}`"\ @@ -193,105 +178,93 @@ def update_preset(albabackend_name, preset_name, policies, api, timeout=UPDATE_P .format(preset_name, albabackend_name)) return True - @staticmethod + @classmethod @required_backend @filter_osds - def add_asds(target, disks, albabackend_name, api, claim_retries=MAX_CLAIM_RETRIES): + def add_asds(cls, target, disks, albabackend_name, claim_retries=MAX_CLAIM_RETRIES, *args, **kwargs): """ Initialize and claim a new asds on given disks - :param target: target to add asds too :type target: str :param disks: dict with diskname as key and amount of osds as value :type disks: dict - :param claim_retries: Maximum amount of claim retries - :type claim_retries: int - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param albabackend_name: Name of the AlbaBackend to configure :type albabackend_name: str + :param claim_retries: Maximum amount of claim retries + :type claim_retries: int :return: preset_name :rtype: str """ - # Make sure all backends are registered - BackendSetup._discover_and_register_nodes(api) - # target is a node - node_mapping = AlbaNodeHelper._map_alba_nodes(api) + BackendSetup._discover_and_register_nodes() # Make sure all backends are registered + node_mapping = AlbaNodeHelper._map_alba_nodes() # target is a node + alba_backend_guid = BackendHelper.get_alba_backend_guid_by_name(albabackend_name) - local_stack = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name, api=api) - disk_queue = {} + backend_info = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name) + local_stack = backend_info['local_stack'] + node_slot_information = {} for disk, amount_of_osds in disks.iteritems(): disk_object = AlbaNodeHelper.get_disk_by_ip(ip=target, diskname=disk) # Get the name of the disk out of the path, only expecting one with ata- - disk_path = BackendHelper.get_local_stack_alias(disk_object) + slot_id = BackendHelper.get_local_stack_alias(disk_object) for alba_node_id, alba_node_guid in node_mapping.iteritems(): + node_info = local_stack[alba_node_id] # Check if the alba_node_id has the disk - if disk_path in local_stack['local_stack'][alba_node_id]: - if alba_node_guid not in disk_queue: - disk_queue[alba_node_guid] = {} - # Initialize disk: - BackendSetup.LOGGER.info( - 'Adding {0} to disk queue for providing {1} asds.'.format(disk_path, amount_of_osds)) - disk_queue[alba_node_guid][disk_path] = amount_of_osds - for alba_node_guid, queue in disk_queue.iteritems(): - BackendSetup.LOGGER.info( - 'Posting disk queue {0} for alba_node_guid {1}'.format(disk_queue[alba_node_guid], alba_node_guid)) - result = BackendSetup._initialize_disk( - alba_node_guid=alba_node_guid, - queue=disk_queue[alba_node_guid], - api=api) - BackendSetup.LOGGER.info('Claiming disks was succesfull. Result = {0}'.format(result)) + if slot_id in node_info: + slot_information = node_slot_information.get(alba_node_guid, []) + BackendSetup.LOGGER.info('Adding {0} to disk queue for providing {1} osds.'.format(slot_id, amount_of_osds)) + slot_information.append({'count': amount_of_osds, + 'slot_id': slot_id, + 'osd_type': 'ASD', + 'alba_backend_guid': alba_backend_guid}) + + node_slot_information[alba_node_guid] = slot_information + for alba_node_guid, slot_information in node_slot_information.iteritems(): + BackendSetup.LOGGER.info('Posting {0} for alba_node_guid {1}'.format(slot_information, alba_node_guid)) + BackendSetup._fill_slots(alba_node_guid=alba_node_guid, slot_information=slot_information) # Local stack should sync with the new disks BackendSetup.LOGGER.info('Sleeping for {0} seconds to let local stack sync.'.format(BackendSetup.LOCAL_STACK_SYNC)) time.sleep(BackendSetup.LOCAL_STACK_SYNC) # Restarting iteration to avoid too many local stack calls: - local_stack = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name, api=api) - asd_queue = {} + node_osds_to_claim = {} for disk, amount_of_osds in disks.iteritems(): disk_object = AlbaNodeHelper.get_disk_by_ip(ip=target, diskname=disk) # Get the name of the disk out of the path - disk_path = BackendHelper.get_local_stack_alias(disk_object) + slot_id = BackendHelper.get_local_stack_alias(disk_object) for alba_node_id, alba_node_guid in node_mapping.iteritems(): - # Check if the alba_node_id has the disk - if disk_path in local_stack['local_stack'][alba_node_id]: - # Claim asds - for asd_id, asd_info in local_stack['local_stack'][alba_node_id][disk_path]['asds'].iteritems(): - # If the asd is not available, fetch local_stack again after 5s to wait for albamgr to claim it - current_retry = 0 - while asd_info['status'] != 'available': - current_retry += 1 - BackendSetup.LOGGER.info('ASD {0} for Alba node {1} was not available. Waiting 5 seconds' - ' to retry (currently {2} retries left).'.format(asd_id, alba_node_id, claim_retries - current_retry)) - if current_retry >= claim_retries: - raise RuntimeError('ASD {0} for Alba node {1} did come available after {2} seconds'.format(asd_id, alba_node_id, current_retry * 5)) - time.sleep(5) - local_stack = BackendHelper.get_backend_local_stack(albabackend_name=albabackend_name, api=api) - asd_info = local_stack['local_stack'][alba_node_id][disk_path]['asds'][asd_id] - BackendSetup.LOGGER.info('Adding asd {0} for disk {1} to claim queue'.format(asd_id, - local_stack[ - 'local_stack'][ - alba_node_id][ - disk_path][ - 'guid'])) - asd_queue[asd_id] = local_stack['local_stack'][alba_node_id][disk_path]['guid'] - if len(asd_queue.keys()) != 0: - BackendSetup.LOGGER.info('Posting asd queue {0}'.format(asd_queue)) - BackendSetup._claim_asd( - alba_backend_name=albabackend_name, - api=api, - queue=asd_queue - ) - else: - BackendSetup.LOGGER.info('No asds have to claimed for {0}'.format(albabackend_name)) - - @staticmethod - def _discover_and_register_nodes(api): + albanode = AlbaNodeHelper.get_albanode(alba_node_guid) + # Claim asds + if slot_id not in albanode.stack: + continue + osds = albanode.stack[slot_id]['osds'] + for osd_id, osd_info in osds.iteritems(): + # If the asd is not available, fetch local_stack again after 5s to wait for albamgr to claim it + current_retry = 0 + while osd_info['status'] not in ['available', 'ok']: + current_retry += 1 + BackendSetup.LOGGER.info('ASD {0} for Alba node {1} was not available. Waiting 5 seconds ' + 'to retry (currently {2} retries left).'.format(osd_id, alba_node_id, claim_retries - current_retry)) + if current_retry >= claim_retries: + raise RuntimeError('ASD {0} for Alba node {1} did come available after {2} seconds'.format(osd_id, alba_node_id, current_retry * 5)) + time.sleep(5) + albanode.invalidate_dynamics('stack') + osd_info = albanode.stack[slot_id][osd_id] + BackendSetup.LOGGER.info('Adding asd {0} for slot {1} to claim queue'.format(osd_id, slot_id)) + osds_to_claim = node_osds_to_claim.get(alba_node_guid, []) + osds_to_claim.append({'osd_type': 'ASD', + 'ips': osd_info['ips'], + 'port': osd_info['port'], + 'slot_id': slot_id}) + node_osds_to_claim[alba_node_guid] = osds_to_claim + for alba_node_guid, osds_to_claim in node_osds_to_claim.iteritems(): + BackendSetup.LOGGER.info('Posting {0} for alba_node_guid {1}'.format(osds_to_claim, alba_node_guid)) + BackendSetup._claim_osds(alba_backend_name=albabackend_name, alba_node_guid=alba_node_guid, osds=osds_to_claim) + + @classmethod + def _discover_and_register_nodes(cls, *args, **kwargs): """ Will discover and register potential nodes to the DAL/Alba - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient """ options = { @@ -299,127 +272,109 @@ def _discover_and_register_nodes(api): 'contents': 'node_id,_relations', 'discover': True } - response = api.get( + response = cls.api.get( api='alba/nodes', params=options ) for node in response['data']: - api.post( + cls.api.post( api='alba/nodes', data={'node_id': {'node_id': node['node_id']}} ) - @staticmethod - def _map_alba_nodes(api): + @classmethod + def _map_alba_nodes(cls, *args, **kwargs): """ Will map the alba_node_id with its guid counterpart and return the map dict - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient """ mapping = {} options = { 'contents': 'node_id,_relations', } - response = api.get( + response = cls.api.get( api='alba/nodes', params=options ) for node in response['data']: - print node mapping[node['node_id']] = node['guid'] return mapping - @staticmethod - def get_backend_local_stack(alba_backend_name, api): + @classmethod + def get_backend_local_stack(cls, alba_backend_name, *args, **kwargs): """ Fetches the local stack property of a backend :param alba_backend_name: backend name :type alba_backend_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient """ options = { 'contents': 'local_stack', } - return api.get(api='/alba/backends/{0}/'.format(BackendHelper.get_alba_backend_guid_by_name(alba_backend_name)), + return cls.api.get(api='/alba/backends/{0}/'.format(BackendHelper.get_alba_backend_guid_by_name(alba_backend_name)), params={'queryparams': options} ) - @staticmethod - def _initialize_disk(alba_node_guid, api, timeout=INITIALIZE_DISK_TIMEOUT, diskname=None, amount_of_osds=None, - queue=None): + @classmethod + def _fill_slots(cls, alba_node_guid, slot_information, timeout=INITIALIZE_DISK_TIMEOUT, *args, **kwargs): """ Initializes a disk to create osds :param alba_node_guid: - :param diskname: - :param amount_of_osds: - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: timeout counter in seconds - :param queue: queue of disks - :type queue: dict + :param slot_information: list of slots to fill + :type slot_information: list :type timeout: int :return: """ - if queue is None and (diskname and amount_of_osds is not None): - queue = {'disks': {diskname: amount_of_osds}} - - data = {'disks': queue} - task_guid = api.post( - api='/alba/nodes/{0}/initialize_disks/'.format(alba_node_guid), + data = {'slot_information': slot_information} + task_guid = cls.api.post( + api='/alba/nodes/{0}/fill_slots/'.format(alba_node_guid), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = "Initialize disk `{0}` for alba node `{1}` has failed".format(queue, alba_node_guid) + error_msg = "Initialize disk `{0}` for alba node `{1}` has failed".format(data, alba_node_guid) BackendSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: - BackendSetup.LOGGER.info("Successfully initialized '{0}'".format(queue)) + BackendSetup.LOGGER.info("Successfully initialized '{0}'".format(data)) return task_result[0] - @staticmethod - def _claim_asd(alba_backend_name, api, timeout=CLAIM_ASD_TIMEOUT, asd_id=None, disk_guid=None, queue=None): + @classmethod + def _claim_osds(cls, alba_backend_name, alba_node_guid, osds, timeout=CLAIM_ASD_TIMEOUT, *args, **kwargs): """ Claims a asd :param alba_backend_name: backend name :type alba_backend_name: str - :param asd_id: id of the asd - :type asd_id: str - :param disk_guid: guid of the disk - :type disk_guid: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient + :param alba_node_guid: guid of the alba node on which the osds are available + :type alba_node_guid: str + :param osds: list of osds to claim + :type osds: list :param timeout: timeout counter in seconds :type timeout: int - :param queue: queue of asds - :type queue: dict :return: """ - if queue is None and (asd_id and disk_guid is not None): - queue = {asd_id: disk_guid} - data = {'osds': queue} - task_guid = api.post( - api='/alba/backends/{0}/add_units/'.format(BackendHelper.get_alba_backend_guid_by_name(alba_backend_name)), + data = {'alba_node_guid': alba_node_guid, + 'osds': osds} + task_guid = cls.api.post( + api='/alba/backends/{0}/add_osds/'.format(BackendHelper.get_alba_backend_guid_by_name(alba_backend_name)), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = "Claim ASD `{0}` for alba backend `{1}` has failed with error '{2}'".format(queue, alba_backend_name, task_result[1]) + error_msg = "Claim ASD `{0}` for alba backend `{1}` has failed with error '{2}'".format(osds, alba_backend_name, task_result[1]) BackendSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: - BackendSetup.LOGGER.info("Succesfully claimed '{0}'".format(queue)) + BackendSetup.LOGGER.info("Succesfully claimed '{0}'".format(osds)) return task_result[0] - @staticmethod + @classmethod @required_preset @required_backend @check_linked_backend - def link_backend(albabackend_name, globalbackend_name, preset_name, api, timeout=LINK_BACKEND_TIMEOUT): + def link_backend(cls, albabackend_name, globalbackend_name, preset_name, timeout=LINK_BACKEND_TIMEOUT, *args, **kwargs): """ Link a LOCAL backend to a GLOBAL backend @@ -429,12 +384,11 @@ def link_backend(albabackend_name, globalbackend_name, preset_name, api, timeout :type globalbackend_name: str :param preset_name: name of the preset available in the LOCAL alba backend :type preset_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: timeout counter in seconds :type timeout: int :return: """ + local_albabackend = BackendHelper.get_albabackend_by_name(albabackend_name) data = { @@ -453,14 +407,13 @@ def link_backend(albabackend_name, globalbackend_name, preset_name, api, timeout } } } - task_guid = api.post( + task_guid = cls.api.post( api='/alba/backends/{0}/link_alba_backends' .format(BackendHelper.get_alba_backend_guid_by_name(globalbackend_name)), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) - + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Linking backend `{0}` to global backend `{1}` has failed with error '{2}'".format( albabackend_name, globalbackend_name, task_result[1]) diff --git a/setup/celery.py b/setup/celery.py index fd7b5db..df94d2b 100644 --- a/setup/celery.py +++ b/setup/celery.py @@ -15,9 +15,9 @@ # but WITHOUT ANY WARRANTY of any kind. from ovs.extensions.generic.configuration import Configuration +from ovs.log.log_handler import LogHandler from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.services.servicefactory import ServiceFactory -from ovs.log.log_handler import LogHandler from ..helpers.storagerouter import StoragerouterHelper diff --git a/setup/domain.py b/setup/domain.py index fae2b81..5419abe 100644 --- a/setup/domain.py +++ b/setup/domain.py @@ -16,34 +16,32 @@ from ovs.log.log_handler import LogHandler from ..helpers.backend import BackendHelper +from ..helpers.ci_constants import CIConstants from ..helpers.domain import DomainHelper from ..helpers.storagerouter import StoragerouterHelper from ..validate.decorators import required_backend -class DomainSetup(object): +class DomainSetup(CIConstants): LOGGER = LogHandler.get(source="setup", name="ci_domain_setup") def __init__(self): pass - @staticmethod - def add_domain(domain_name, api): + @classmethod + def add_domain(cls, domain_name, *args, **kwargs): """ Add a new (recovery) domain to the cluster - :param domain_name: name of a new domain + :param domain_name: name of a new domain to add :type domain_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :return: """ - # check if domain already exists if not DomainHelper.get_domain_by_name(domain_name): data = {"name": domain_name} - api.post( + cls.api.post( api='/domains/', data=data ) @@ -58,22 +56,18 @@ def add_domain(domain_name, api): else: return - @staticmethod - def link_domains_to_storagerouter(domain_details, storagerouter_ip, api): + @classmethod + def link_domains_to_storagerouter(cls, domain_details, storagerouter_ip, *args, **kwargs): """ Link a existing domain(s) and/or recovery (domains) to a storagerouter - :param domain_details: domain details of a storagerouter example: {"domain_guids":["Gravelines"],"recovery_domain_guids":["Roubaix", "Strasbourg"]} :type domain_details: dict :param storagerouter_ip: ip address of a storage router :type storagerouter_ip: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :return: """ - storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip) domain_guids = [] recovery_domain_guids = [] # translate domain names to domain guids @@ -86,25 +80,27 @@ def link_domains_to_storagerouter(domain_details, storagerouter_ip, api): data = {"domain_guids": domain_guids, "recovery_domain_guids": recovery_domain_guids} - api.post( + + storagerouter_guid = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip).guid + cls.api.post( api='/storagerouters/{0}/set_domains/'.format(storagerouter_guid), data=data ) - storagerouter = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip) + storagerouter = StoragerouterHelper.get_storagerouter_by_guid(storagerouter_guid=storagerouter_guid) if len(set(domain_guids) - set(storagerouter.regular_domains)) != 0 or \ len(set(recovery_domain_guids) - set(storagerouter.recovery_domains)) != 0: - error_msg = "Failed to link (recovery) domain(s) to storagerouter `{0}`".format(storagerouter_ip) + error_msg = "Failed to link (recovery) domain(s) to storagerouter `{0}`".format(storagerouter_guid) DomainSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: DomainSetup.LOGGER.info("Successfully linked domain (recovery) domain(s) to storagerouter `{0}`" - .format(storagerouter_ip)) + .format(storagerouter_guid)) return - @staticmethod + @classmethod @required_backend - def link_domains_to_backend(domain_details, albabackend_name, api): + def link_domains_to_backend(cls, domain_details, albabackend_name, *args, **kwargs): """ Link a existing domain(s) and/or recovery (domains) to a storagerouter @@ -113,9 +109,6 @@ def link_domains_to_backend(domain_details, albabackend_name, api): :type domain_details: dict :param albabackend_name: name of a existing alba backend :type albabackend_name: str - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient - :return: """ albabackend_guid = BackendHelper.get_backend_guid_by_name(albabackend_name) @@ -126,7 +119,7 @@ def link_domains_to_backend(domain_details, albabackend_name, api): domain_guids.append(DomainHelper.get_domainguid_by_name(domain_name)) data = {"domain_guids": domain_guids} - api.post( + cls.api.post( api='/backends/{0}/set_domains/'.format(albabackend_guid), data=data ) diff --git a/setup/proxy.py b/setup/proxy.py index 9467707..2595f55 100644 --- a/setup/proxy.py +++ b/setup/proxy.py @@ -1,4 +1,5 @@ # Copyright (C) 2016 iNuron NV +# Copyright (C) 2016 iNuron NV # # This file is part of Open vStorage Open Source Edition (OSE), # as available from @@ -16,9 +17,10 @@ import json from ovs.dal.lists.vpoollist import VPoolList from ovs.extensions.generic.configuration import Configuration +from ovs.log.log_handler import LogHandler +from ovs_extensions.generic.toolbox import ExtensionsToolbox from ovs.extensions.services.servicefactory import ServiceFactory from ovs.lib.helpers.toolbox import Toolbox -from ovs.log.log_handler import LogHandler from ovs.dal.hybrids.service import Service from ovs.extensions.generic.sshclient import SSHClient @@ -48,7 +50,7 @@ def configure_proxy(backend_name, proxy_configuration): faulty_keys = [key for key in proxy_configuration.keys() if key not in ProxySetup.PARAMS] if len(faulty_keys) > 0: raise ValueError('{0} are unsupported keys for proxy configuration.'.format(', '.join(faulty_keys))) - Toolbox.verify_required_params(ProxySetup.PARAMS, proxy_configuration) + ExtensionsToolbox.verify_required_params(ProxySetup.PARAMS, proxy_configuration) vpools = VPoolList.get_vpools() service_manager = ServiceFactory.get_manager() with open('/root/old_proxies', 'w') as backup_file: diff --git a/setup/roles.py b/setup/roles.py index a3873b9..553bf72 100644 --- a/setup/roles.py +++ b/setup/roles.py @@ -15,11 +15,12 @@ # but WITHOUT ANY WARRANTY of any kind. from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants from ..helpers.storagerouter import StoragerouterHelper from ..validate.decorators import check_role_on_disk -class RoleSetup(object): +class RoleSetup(CIConstants): LOGGER = LogHandler.get(source="setup", name="ci_role_setup") CONFIGURE_DISK_TIMEOUT = 300 @@ -29,28 +30,27 @@ class RoleSetup(object): def __init__(self): pass - @staticmethod + @classmethod @check_role_on_disk - def add_disk_role(storagerouter_ip, diskname, roles, api, min_size=MIN_PARTITION_SIZE): + def add_disk_role(cls, storagerouter_ip, diskname, roles, min_size=MIN_PARTITION_SIZE, *args, **kwargs): + """ Partition and adds roles to a disk - :param storagerouter_ip: ip address of a existing storagerouter + :param storagerouter_ip: guid of an existing storagerouter :type storagerouter_ip: str :param diskname: shortname of a disk (e.g. sdb) :type diskname: str :param roles: list of roles you want to add to the disk :type roles: list - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param min_size: minimum total_partition_size that is required to allocate the disk role :type min_size: int :return: """ # Fetch information - storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip) - disk = StoragerouterHelper.get_disk_by_ip(storagerouter_ip, diskname) + storagerouter_guid = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip).guid + disk = StoragerouterHelper.get_disk_by_name(storagerouter_guid, diskname) # Check if there are any partitions on the disk, if so check if there is enough space unused_partitions = [] if len(disk.partitions) > 0: @@ -60,7 +60,7 @@ def add_disk_role(storagerouter_ip, diskname, roles, api, min_size=MIN_PARTITION # Check if the partition is in use - could possibly write role on unused partition if partition.mountpoint is None: # Means no output -> partition not mounted - # @Todo support partitions that are not sequentional + # @Todo support partitions that are not sequential unused_partitions.append(partition) # Elect biggest unused partition as potential candidate @@ -72,21 +72,21 @@ def add_disk_role(storagerouter_ip, diskname, roles, api, min_size=MIN_PARTITION if ((disk.size-total_partition_size)/1024**3) > min_size: # disk is still large enough, let the partitioning begin and apply some roles! RoleSetup.configure_disk(storagerouter_guid=storagerouter_guid, disk_guid=disk.guid, offset=total_partition_size + 1, - size=(disk.size-total_partition_size)-1, roles=roles, api=api) + size=(disk.size-total_partition_size)-1, roles=roles) elif biggest_unused_partition is not None and (biggest_unused_partition.size/1024**3) > min_size: RoleSetup.configure_disk(storagerouter_guid=storagerouter_guid, disk_guid=disk.guid, offset=biggest_unused_partition.offset, - size=biggest_unused_partition.size, roles=roles, api=api, partition_guid=biggest_unused_partition.guid) + size=biggest_unused_partition.size, roles=roles, partition_guid=biggest_unused_partition.guid) else: # disk is too small raise RuntimeError("Disk `{0}` on node `{1}` is too small for role(s) `{2}`, min. total_partition_size is `{3}`" - .format(diskname, storagerouter_ip, roles, min_size)) + .format(diskname, storagerouter_guid, roles, min_size)) else: # there are no partitions on the disk, go nuke it! - RoleSetup.configure_disk(storagerouter_guid, disk.guid, 0, disk.size, roles, api) + RoleSetup.configure_disk(storagerouter_guid, disk.guid, 0, disk.size, roles) - @staticmethod - def configure_disk(storagerouter_guid, disk_guid, offset, size, roles, api, partition_guid=None, - timeout=CONFIGURE_DISK_TIMEOUT): + @classmethod + def configure_disk(cls, storagerouter_guid, disk_guid, offset, size, roles, partition_guid=None, + timeout=CONFIGURE_DISK_TIMEOUT, *args, **kwargs): """ Partition a disk and add roles to it @@ -100,8 +100,6 @@ def configure_disk(storagerouter_guid, disk_guid, offset, size, roles, api, part :type size: int :param roles: roles to add to a partition (e.g. ['DB', 'WRITE']) :type roles: list - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :param partition_guid: guid of the partition @@ -116,12 +114,11 @@ def configure_disk(storagerouter_guid, disk_guid, offset, size, roles, api, part 'roles': roles, 'partition_guid': partition_guid } - task_guid = api.post( + task_guid = cls.api.post( api='/storagerouters/{0}/configure_disk/'.format(storagerouter_guid), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) - + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Adjusting disk `{0}` has failed on storagerouter `{1}` with error '{2}'" \ .format(disk_guid, storagerouter_guid, task_result[1]) diff --git a/setup/storagedriver.py b/setup/storagedriver.py index 34ae470..4a0be5d 100644 --- a/setup/storagedriver.py +++ b/setup/storagedriver.py @@ -13,8 +13,8 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. -from ovs.lib.helpers.toolbox import Toolbox from ovs.log.log_handler import LogHandler +from ovs_extensions.generic.toolbox import ExtensionsToolbox from ..helpers.storagedriver import StoragedriverHelper from ..helpers.vpool import VPoolHelper @@ -32,7 +32,7 @@ def change_config(vpool_name, vpool_details, storagerouter_ip, *args, **kwargs): # Settings volumedriver storagedriver_config = vpool_details.get('storagedriver') if storagedriver_config is not None: - Toolbox.verify_required_params(StoragedriverSetup.STORAGEDRIVER_PARAMS, storagedriver_config) + ExtensionsToolbox.verify_required_params(StoragedriverSetup.STORAGEDRIVER_PARAMS, storagedriver_config) StoragedriverSetup.LOGGER.info('Updating volumedriver configuration of vPool `{0}` on storagerouter `{1}`.'.format(vpool_name, storagerouter_ip)) vpool = VPoolHelper.get_vpool_by_name(vpool_name) storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter.ip == storagerouter_ip][0] diff --git a/setup/vdisk.py b/setup/vdisk.py index af51bf7..1914370 100644 --- a/setup/vdisk.py +++ b/setup/vdisk.py @@ -15,14 +15,14 @@ # but WITHOUT ANY WARRANTY of any kind. from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants from ..helpers.storagerouter import StoragerouterHelper from ..helpers.vdisk import VDiskHelper from ..helpers.vpool import VPoolHelper from ..validate.decorators import required_vdisk, required_snapshot, required_vtemplate -class VDiskSetup(object): - +class VDiskSetup(CIConstants): LOGGER = LogHandler.get(source="setup", name="ci_vdisk_setup") CREATE_SNAPSHOT_TIMEOUT = 60 CREATE_VDISK_TIMEOUT = 60 @@ -34,9 +34,9 @@ class VDiskSetup(object): def __init__(self): pass - @staticmethod - def create_snapshot(snapshot_name, vdisk_name, vpool_name, api, consistent=True, sticky=True, - timeout=CREATE_SNAPSHOT_TIMEOUT): + @classmethod + def create_snapshot(cls, snapshot_name, vdisk_name, vpool_name, consistent=True, sticky=True, + timeout=CREATE_SNAPSHOT_TIMEOUT, *args, **kwargs): """ Create a new snapshot for a vdisk @@ -49,8 +49,6 @@ def create_snapshot(snapshot_name, vdisk_name, vpool_name, api, consistent=True, :type consistent: bool :param sticky: let this snapshot stick forever? :type sticky: bool - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :param vpool_name: name of a existing vpool @@ -66,11 +64,11 @@ def create_snapshot(snapshot_name, vdisk_name, vpool_name, api, consistent=True, 'sticky': sticky } - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/create_snapshot/'.format(vdisk_guid), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Creating snapshot `{0}` for vdisk `{1}` on vPool `{2}` has failed"\ @@ -82,8 +80,8 @@ def create_snapshot(snapshot_name, vdisk_name, vpool_name, api, consistent=True, .format(snapshot_name, vdisk_name, vpool_name)) return task_result[1] - @staticmethod - def create_vdisk(vdisk_name, vpool_name, size, storagerouter_ip, api, timeout=CREATE_VDISK_TIMEOUT): + @classmethod + def create_vdisk(cls, vdisk_name, vpool_name, size, storagerouter_ip, timeout=CREATE_VDISK_TIMEOUT, *args, **kwargs): """ Create a new vDisk on a certain vPool/storagerouter :param vdisk_name: location of a vdisk on a vpool (e.g. /mnt/vpool/test.raw = test.raw, /mnt/vpool/volumes/test.raw = volumes/test.raw ) @@ -94,8 +92,6 @@ def create_vdisk(vdisk_name, vpool_name, size, storagerouter_ip, api, timeout=CR :type size: int :param storagerouter_ip: ip address of a existing storagerouter :type storagerouter_ip: str - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :param vpool_name: name of a existing vpool @@ -117,11 +113,11 @@ def create_vdisk(vdisk_name, vpool_name, size, storagerouter_ip, api, timeout=CR "vpool_guid": vpool_guid, "storagerouter_guid": storagerouter_guid} - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/', data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Creating vdisk `{0}` on vPool `{1}` on storagerouter `{2}` has failed with error {3}"\ @@ -133,24 +129,23 @@ def create_vdisk(vdisk_name, vpool_name, size, storagerouter_ip, api, timeout=CR .format(vdisk_name, vpool_name, storagerouter_ip)) return task_result[1] - @staticmethod + @classmethod @required_vdisk - def move_vdisk(vdisk_guid, target_storagerouter_guid, api, timeout=60): + def move_vdisk(cls, vdisk_guid, target_storagerouter_guid, timeout=60, *args, **kwargs): """ Moves a vdisk :param vdisk_guid: guid of the vdisk :param target_storagerouter_guid: guid of the storuagerouter to move to - :param api: instance of ovs client :param timeout: timeout in seconds :return: """ data = {"target_storagerouter_guid": target_storagerouter_guid} - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/move/'.format(vdisk_guid), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Moving vdisk {0} to {1} has failed with {2}.".format( @@ -162,11 +157,11 @@ def move_vdisk(vdisk_guid, target_storagerouter_guid, api, timeout=60): "Vdisk {0} should have been moved to {1}.".format(vdisk_guid, target_storagerouter_guid)) return task_result[1] - @staticmethod + @classmethod @required_vdisk @required_snapshot - def create_clone(vdisk_name, vpool_name, new_vdisk_name, storagerouter_ip, api, snapshot_id=None, - timeout=CREATE_CLONE_TIMEOUT): + def create_clone(cls, vdisk_name, vpool_name, new_vdisk_name, storagerouter_ip, snapshot_id=None, + timeout=CREATE_CLONE_TIMEOUT, *args, **kwargs): """ Create a new vDisk on a certain vPool/storagerouter :param vdisk_name: location of a vdisk on a vpool (e.g. /mnt/vpool/test.raw = test.raw, /mnt/vpool/volumes/test.raw = volumes/test.raw ) @@ -179,8 +174,6 @@ def create_clone(vdisk_name, vpool_name, new_vdisk_name, storagerouter_ip, api, :type storagerouter_ip: str :param snapshot_id: GUID of a existing snapshot (DEFAULT=None -> will create new snapshot) :type snapshot_id: str - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :param vpool_name: name of a existing vpool @@ -210,11 +203,11 @@ def create_clone(vdisk_name, vpool_name, new_vdisk_name, storagerouter_ip, api, "storagerouter_guid": storagerouter_guid, "snapshot_id": snapshot_id} - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/clone'.format(vdisk.guid), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Creating clone `{0}` with snapshot_id `{4}` on vPool `{1}` on storagerouter `{2}` " \ @@ -228,9 +221,9 @@ def create_clone(vdisk_name, vpool_name, new_vdisk_name, storagerouter_ip, api, snapshot_id)) return task_result[1] - @staticmethod + @classmethod @required_vdisk - def set_vdisk_as_template(vdisk_name, vpool_name, api, timeout=SET_VDISK_AS_TEMPLATE_TIMEOUT): + def set_vdisk_as_template(cls, vdisk_name, vpool_name, timeout=SET_VDISK_AS_TEMPLATE_TIMEOUT, *args, **kwargs): """ Create a new vDisk on a certain vPool/storagerouter Set a existing vDisk as vTemplate @@ -240,17 +233,15 @@ def set_vdisk_as_template(vdisk_name, vpool_name, api, timeout=SET_VDISK_AS_TEMP :type vdisk_name: str :param vpool_name: name of a existing vpool :type vpool_name: str - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete """ # fetch the requirements vdisk = VDiskHelper.get_vdisk_by_name(vdisk_name, vpool_name) - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/set_as_template'.format(vdisk.guid) ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Creating vTemplate `{0}` has failed with error {1}".format(vdisk_name, task_result[1]) @@ -260,10 +251,10 @@ def set_vdisk_as_template(vdisk_name, vpool_name, api, timeout=SET_VDISK_AS_TEMP VDiskSetup.LOGGER.info("Creating vTemplate `{0}` should have succeeded".format(vdisk_name)) return task_result[1] - @staticmethod + @classmethod @required_vtemplate - def create_from_template(vdisk_name, vpool_name, new_vdisk_name, storagerouter_ip, api, - timeout=SET_VDISK_AS_TEMPLATE_TIMEOUT): + def create_from_template(cls, vdisk_name, vpool_name, new_vdisk_name, storagerouter_ip, + timeout=SET_VDISK_AS_TEMPLATE_TIMEOUT, *args, **kwargs): """ Create a new vDisk on a certain vPool/storagerouter Set a existing vDisk as vTemplate @@ -275,8 +266,6 @@ def create_from_template(vdisk_name, vpool_name, new_vdisk_name, storagerouter_i :type new_vdisk_name: str :param storagerouter_ip: ip address of a existing storagerouter where the clone will be deployed :type storagerouter_ip: str - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete :return: dict with info about the new vdisk {'vdisk_guid': new_vdisk.guid, 'name': new_vdisk.name, 'backingdevice': devicename} :rtype: dict @@ -294,11 +283,11 @@ def create_from_template(vdisk_name, vpool_name, new_vdisk_name, storagerouter_i data = {"name": official_new_vdisk_name, "storagerouter_guid": storagerouter_guid} - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/create_from_template'.format(vdisk.guid), data=data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Creating vTemplate `{0}` has failed with error {1}".format(vdisk_name, task_result[1]) @@ -308,9 +297,9 @@ def create_from_template(vdisk_name, vpool_name, new_vdisk_name, storagerouter_i VDiskSetup.LOGGER.info("Creating vTemplate `{0}` should have succeeded".format(vdisk_name)) return task_result[1] - @staticmethod + @classmethod @required_vdisk - def rollback_to_snapshot(vdisk_name, vpool_name, snapshot_id, api, timeout=ROLLBACK_VDISK_TIMEOUT): + def rollback_to_snapshot(cls, vdisk_name, vpool_name, snapshot_id, timeout=ROLLBACK_VDISK_TIMEOUT, *args, **kwargs): """ Rollback a vdisk to a certain snapshot @@ -321,8 +310,6 @@ def rollback_to_snapshot(vdisk_name, vpool_name, snapshot_id, api, timeout=ROLLB :type vpool_name: str :param snapshot_id: guid of a snapshot for the chosen vdisk :type snapshot_id: str - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete """ @@ -331,11 +318,11 @@ def rollback_to_snapshot(vdisk_name, vpool_name, snapshot_id, api, timeout=ROLLB snapshot = VDiskHelper.get_snapshot_by_guid(snapshot_guid=snapshot_id, vdisk_name=vdisk_name, vpool_name=vpool_name) - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/rollback'.format(vdisk_guid), data={"timestamp": snapshot['timestamp']} ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Rollback vDisk `{0}` has failed with error {1}".format(vdisk_name, task_result[1]) @@ -345,9 +332,9 @@ def rollback_to_snapshot(vdisk_name, vpool_name, snapshot_id, api, timeout=ROLLB VDiskSetup.LOGGER.info("Rollback vDisk `{0}` should have succeeded".format(vdisk_name)) return task_result[1] - @staticmethod + @classmethod @required_vdisk - def set_config_params(vdisk_name, vpool_name, config, api, timeout=SET_CONFIG_VDISK_TIMEOUT): + def set_config_params(cls, vdisk_name, vpool_name, config, timeout=SET_CONFIG_VDISK_TIMEOUT, *args, **kwargs): """ Rollback a vdisk to a certain snapshot @@ -366,8 +353,6 @@ def set_config_params(vdisk_name, vpool_name, config, api, timeout=SET_CONFIG_VD ] } :type config: dict - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :rtype: dict @@ -377,11 +362,11 @@ def set_config_params(vdisk_name, vpool_name, config, api, timeout=SET_CONFIG_VD # fetch the requirements vdisk_guid = VDiskHelper.get_vdisk_by_name(vdisk_name=vdisk_name, vpool_name=vpool_name).guid - task_guid = api.post( + task_guid = cls.api.post( api='/vdisks/{0}/set_config_params'.format(vdisk_guid), data={"new_config_params": config} ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Setting config vDisk `{0}` has failed with error {1}".format(vdisk_name, task_result[1]) diff --git a/setup/vpool.py b/setup/vpool.py index 0b17c54..5dccbc8 100644 --- a/setup/vpool.py +++ b/setup/vpool.py @@ -13,15 +13,16 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. +from ovs.extensions.generic.configuration import Configuration from ovs.lib.generic import GenericController from ovs.log.log_handler import LogHandler from ..helpers.backend import BackendHelper from ..helpers.storagerouter import StoragerouterHelper +from ..helpers.vpool import VPoolHelper from ..validate.decorators import required_roles, check_vpool class VPoolSetup(object): - LOGGER = LogHandler.get(source='setup', name='ci_vpool_setup') ADD_VPOOL_TIMEOUT = 500 REQUIRED_VPOOL_ROLES = ['DB', 'WRITE', 'DTL'] @@ -32,7 +33,8 @@ def __init__(self): @staticmethod @check_vpool @required_roles(REQUIRED_VPOOL_ROLES, 'LOCAL') - def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, timeout=ADD_VPOOL_TIMEOUT, *args, **kwargs): + def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, timeout=ADD_VPOOL_TIMEOUT, *args, + **kwargs): """ Adds a VPool to a storagerouter @@ -55,8 +57,9 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, # Build ADD_VPOOL parameters call_parameters = { 'vpool_name': vpool_name, - 'backend_info': {'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['backend_name']).guid, - 'preset': vpool_details['preset']}, + 'backend_info': { + 'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['backend_name']).guid, + 'preset': vpool_details['preset']}, 'connection_info': {'host': '', 'port': '', 'client_id': '', 'client_secret': ''}, 'storage_ip': vpool_details['storage_ip'], 'storagerouter_ip': storagerouter_ip, @@ -72,13 +75,10 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, } api_data = {'call_parameters': call_parameters} - # Setting for mds_safety - if vpool_details.get('mds_safety') is not None: - call_parameters['mds_config_params'] = {'mds_safety': vpool_details['mds_safety']} - # Setting possible alba accelerated alba if vpool_details['fragment_cache']['location'] == 'backend': - call_parameters['backend_info_aa'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['fragment_cache']['backend']['name']).guid, + call_parameters['backend_info_aa'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name( + vpool_details['fragment_cache']['backend']['name']).guid, 'preset': vpool_details['fragment_cache']['backend']['preset']} call_parameters['connection_info_aa'] = {'host': '', 'port': '', 'client_id': '', 'client_secret': ''} elif vpool_details['fragment_cache']['location'] == 'disk': @@ -93,7 +93,8 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, call_parameters['block_cache_on_read'] = vpool_details['block_cache']['strategy']['cache_on_read'] call_parameters['block_cache_on_write'] = vpool_details['block_cache']['strategy']['cache_on_write'] if vpool_details['block_cache']['location'] == 'backend': - call_parameters['backend_info_bc'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['block_cache']['backend']['name']).guid, + call_parameters['backend_info_bc'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name( + vpool_details['block_cache']['backend']['name']).guid, 'preset': vpool_details['block_cache']['backend']['preset']} call_parameters['connection_info_bc'] = {'host': '', 'port': '', 'client_id': '', 'client_secret': ''} elif vpool_details['block_cache']['location'] == 'disk': # Ignore disk @@ -103,19 +104,37 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, error_msg = 'Wrong `block_cache->location` in vPool configuration, it should be `disk` or `backend`' VPoolSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) - + task_guid = api.post( api='/storagerouters/{0}/add_vpool/'.format( - StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip)), + StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip)), data=api_data ) task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = 'vPool {0} has failed to create on storagerouter {1} because: {2}'.format(vpool_name, storagerouter_ip, task_result[1]) + error_msg = 'vPool {0} has failed to create on storagerouter {1} because: {2}'.format(vpool_name, + storagerouter_ip, + task_result[1]) VPoolSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: - VPoolSetup.LOGGER.info('Creation of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) + VPoolSetup.LOGGER.info( + 'Creation of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, + storagerouter_ip)) + + if vpool_details.get('mds_safety') is not None: + vpool = VPoolHelper.get_vpool_by_name(vpool_name) + + if vpool: + mds_config_path = '/ovs/vpools/{0}/mds_config'.format(vpool.guid) + mds_config = Configuration.get(mds_config_path) + mds_config['mds_safety'] = vpool_details.get('mds_safety') + Configuration.set(mds_config_path, mds_config) + else: + error_msg = 'Unable to change the mds safety of vPool {0}.'.format(vpool_name) + VPoolSetup.LOGGER.error(error_msg) + raise RuntimeError(error_msg) + return storagerouter_ip, '/mnt/{0}'.format(vpool_name) @staticmethod diff --git a/testrail/containers/casetype.py b/testrail/containers/casetype.py new file mode 100644 index 0000000..24740b2 --- /dev/null +++ b/testrail/containers/casetype.py @@ -0,0 +1,50 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +Case Module +""" +from ..testrailobject import TestRailObject + + +class CaseType(TestRailObject): + + _VERIFY_PARAMS = {'is_default': (bool, None), + 'name': (str, None)} + + def __init__(self, id=None, is_default=None, name=None, client=None, *args, **kwargs): + """ + Testrail Case + :param id: ID of the CaseType + :type id: int + :param is_default: The ID of the section the test case should be added to (required) + :type is_default: bool + :param name: The name of the case type + :type name: str + """ + self.is_default = is_default + self.name = name + + super(CaseType, self).__init__(id=id, client=client, load=False, *args, **kwargs) + + def save(self): + """ + Saves the current object on the backend + :return: None + :rtype: NoneType + """ + raise NotImplemented('CaseTypes cannot be created. Only listed') + diff --git a/testrail/lists/casetypelist.py b/testrail/lists/casetypelist.py new file mode 100644 index 0000000..3abfafe --- /dev/null +++ b/testrail/lists/casetypelist.py @@ -0,0 +1,38 @@ +# Copyright (C) 2017 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +""" +ProjectList module +""" +from ..testraillist import TestRailList +from ..containers.casetype import CaseType + + +class CaseTypeList(TestRailList): + def __init__(self, client, *args, **kwargs): + """ + Initializes a TestList + :param run_id: ID of the Run to list tests for + :type run_id: int + """ + super(CaseTypeList, self).__init__(client=client, object_type=CaseType, plural='case_types', *args, **kwargs) + + def get_casetype_by_name(self, casetype_name): + for casetype in self.load(): + if casetype.name.lower() == casetype_name.lower(): + return casetype + + raise LookupError('CaseType `{0}` not found.'.format(casetype_name)) diff --git a/validate/backend.py b/validate/backend.py index 7a43849..69eda64 100644 --- a/validate/backend.py +++ b/validate/backend.py @@ -17,7 +17,6 @@ import ast from ovs.log.log_handler import LogHandler from ..helpers.albanode import AlbaNodeHelper -from ..helpers.asdmanager import ASDManagerClient from ..helpers.backend import BackendHelper from ..helpers.exceptions import AlbaBackendNotFoundError, PresetNotFoundError, AlbaNodeNotFoundError @@ -53,7 +52,7 @@ def check_preset_on_backend(preset_name, albabackend_name): @staticmethod def check_policies_on_preset(preset_name, albabackend_name, policies): """ - Check if a preset is available on a backend + Check if given policies match with the specified backend :param preset_name: name of a preset :type preset_name: str @@ -111,31 +110,33 @@ def check_available_osds_on_asdmanager(ip, disks): :return: dict with available disks :rtype: dict """ - albanode = AlbaNodeHelper.get_albanode_by_ip(ip) - - if albanode is not None: - # compare requested disks with available disks - fetched_disks = ASDManagerClient(node=albanode).get_disks().values() - available_disks = {} - for disk, amount_asds in disks.iteritems(): - try: - # check if requested disk is present and available in fetched_disks - next(fetched_disk for fetched_disk in fetched_disks if - fetched_disk['device'].split('/')[2] == disk and fetched_disk['available']) - - # add disk to available disks - available_disks[disk] = amount_asds - BackendValidation.LOGGER.info("Disk `{0}` is available on node `{1}`!".format(ip, disk)) - except StopIteration: - BackendValidation.LOGGER.error("Disk `{0}` is NOT available on node `{1}`!".format(ip, disk)) - - BackendValidation.LOGGER.info("The following disks are available for use on `{0}`: {1}" - .format(ip, available_disks)) - - return available_disks - else: + if albanode is None: error_msg = "Alba node with ip `{0}` was not found!".format(ip) BackendValidation.LOGGER.error(error_msg) raise AlbaNodeNotFoundError(error_msg) - + # compare requested disks with available disks + fetched_slots = albanode.client.get_stack().values() + slot_map = {} + for fetched_disk in fetched_slots: + # Ignore other slots than disks + if any(key not in fetched_disk for key in ("device", "available")): + continue + disk_name = fetched_disk['device'].rsplit('/', 1)[-1] + slot_map[disk_name] = fetched_disk + available_disks = {} + for disk, amount_asds in disks.iteritems(): + + # check if requested disk is present and available in fetched_disks + if disk not in slot_map: + BackendValidation.LOGGER.error("Disk `{0}` was NOT found on node `{1}`!".format(disk, ip)) + continue + if slot_map[disk]['available'] is False: + BackendValidation.LOGGER.error("Disk `{0}` is NOT available on node `{1}`!".format(disk, ip)) + continue + # add disk to available disks + available_disks[disk] = amount_asds + BackendValidation.LOGGER.info("Disk `{0}` is available on node `{1}`!".format(disk, ip)) + BackendValidation.LOGGER.info("The following disks are available for use on `{0}`: {1}".format(ip, available_disks)) + + return available_disks From 7024f843a741aa450a70551a880b7393c83ba05c Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Wed, 11 Jul 2018 17:35:21 +0200 Subject: [PATCH 5/5] Implemented bug fixes + added new tests (create 500 vdisks from a template) --- helpers/backend.py | 2 +- helpers/ci_constants.py | 3 +- helpers/service.py | 52 +++++++++++++++++++++++++++ helpers/vdisk.py | 49 +++++++++++++++++-------- remove/vdisk.py | 8 ++--- setup/arakoon.py | 12 ++++--- setup/backend.py | 14 ++++++-- setup/vdisk.py | 4 +-- setup/vpool.py | 80 +++++++++++++++++++++-------------------- validate/decorators.py | 21 ----------- validate/roles.py | 4 ++- 11 files changed, 158 insertions(+), 91 deletions(-) create mode 100644 helpers/service.py diff --git a/helpers/backend.py b/helpers/backend.py index e147d77..43cdee3 100644 --- a/helpers/backend.py +++ b/helpers/backend.py @@ -183,7 +183,7 @@ def get_preset_by_albabackend(preset_name, albabackend_name): """ Fetches preset by albabackend_guid - :param preset_name: name of a existing preset + :param preset_name: name of an existing preset :type preset_name: str :param albabackend_name: name of a existing alba backend :type albabackend_name: str diff --git a/helpers/ci_constants.py b/helpers/ci_constants.py index 540498c..618d6f9 100644 --- a/helpers/ci_constants.py +++ b/helpers/ci_constants.py @@ -42,7 +42,8 @@ def __get__(self, cls, owner): def api(cls): return OVSClient(cls.SETUP_CFG['ci']['grid_ip'], cls.SETUP_CFG['ci']['user']['api']['username'], - cls.SETUP_CFG['ci']['user']['api']['password']) + cls.SETUP_CFG['ci']['user']['api']['password'], + ) @classmethod def get_vpool_names(cls): diff --git a/helpers/service.py b/helpers/service.py new file mode 100644 index 0000000..4f73fe6 --- /dev/null +++ b/helpers/service.py @@ -0,0 +1,52 @@ +# Copyright (C) 2016 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. + +from ovs.dal.hybrids.service import Service +from ovs.extensions.generic.sshclient import SSHClient +from ovs.extensions.services.servicefactory import ServiceFactory +from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants + + +class ServiceHelper(CIConstants): + """ + ServiceHelper class + """ + + LOGGER = LogHandler.get(source="setup", name="ci_service_setup") + SERVICE_MANAGER = ServiceFactory.get_manager() + + @staticmethod + def get_service(guid): + """ + Fetch Service object by service guid + :param guid: guid of the service + :return: a Service object + """ + return Service(guid) + + @staticmethod + def restart_service(guid, storagerouter): + """ + Restart a service based on the guid located on storagerouter + :param guid: guid of the service + :param storagerouter: storagerouter where the service is running + :type storagerouter: StorageRouter + :return: + """ + client = SSHClient(storagerouter, username='root') + service = ServiceHelper.get_service(guid) + return ServiceHelper.SERVICE_MANAGER.restart_service(service.name, client=client) diff --git a/helpers/vdisk.py b/helpers/vdisk.py index 31760cf..60cde66 100644 --- a/helpers/vdisk.py +++ b/helpers/vdisk.py @@ -13,14 +13,16 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. -from ovs.log.log_handler import LogHandler + from ovs.dal.hybrids.vdisk import VDisk from ovs.dal.lists.vdisklist import VDiskList from ovs.dal.lists.vpoollist import VPoolList +from ovs.log.log_handler import LogHandler +from ..helpers.ci_constants import CIConstants from ..helpers.exceptions import VPoolNotFoundError, VDiskNotFoundError -class VDiskHelper(object): +class VDiskHelper(CIConstants): """ vDiskHelper class """ @@ -105,8 +107,8 @@ def get_snapshot_by_guid(snapshot_guid, vdisk_name, vpool_name): raise RuntimeError("Did not find snapshot with guid `{0}` on vdisk `{1}` on vpool `{2}`" .format(snapshot_guid, vdisk_name, vpool_name)) - @staticmethod - def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIMEOUT): + @classmethod + def get_config_params(cls, vdisk_name, vpool_name, timeout=GET_CONFIG_PARAMS_TIMEOUT, *args, **kwargs): """ Fetch the config parameters of a vDisk @@ -115,8 +117,6 @@ def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIM :type vdisk_name: str :param vpool_name: name of a existing vpool :type vpool_name: str - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :return: a dict with config parameters, e.g. @@ -133,8 +133,8 @@ def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIM """ vdisk = VDiskHelper.get_vdisk_by_name(vdisk_name=vdisk_name, vpool_name=vpool_name) - task_guid = api.get(api='/vdisks/{0}/get_config_params'.format(vdisk.guid)) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_guid = cls.api.get(api='/vdisks/{0}/get_config_params'.format(vdisk.guid)) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Setting config vDisk `{0}` has failed with error {1}".format(vdisk_name, task_result[1]) @@ -144,23 +144,21 @@ def get_config_params(vdisk_name, vpool_name, api, timeout=GET_CONFIG_PARAMS_TIM VDiskHelper.LOGGER.info("Setting config vDisk `{0}` should have succeeded".format(vdisk_name)) return task_result[1] - @staticmethod - def scrub_vdisk(vdisk_guid, api, timeout=15 * 60, wait=True): + @classmethod + def scrub_vdisk(cls, vdisk_guid, timeout=15 * 60, wait=True, *args, **kwargs): """ Scrub a specific vdisk :param vdisk_guid: guid of the vdisk to scrub :type vdisk_guid: str - :param api: specify a valid api connection to the setup - :type api: ci.helpers.api.OVSClient :param timeout: time to wait for the task to complete :type timeout: int :param wait: wait for task to finish or not :type wait: bool - :return: + :return: """ - task_guid = api.post(api='/vdisks/{0}/scrub/'.format(vdisk_guid), data={}) + task_guid = cls.api.post(api='/vdisks/{0}/scrub/'.format(vdisk_guid), data={}) if wait is True: - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: error_msg = "Scrubbing vDisk `{0}` has failed with error {1}".format(vdisk_guid, task_result[1]) VDiskHelper.LOGGER.error(error_msg) @@ -170,3 +168,24 @@ def scrub_vdisk(vdisk_guid, api, timeout=15 * 60, wait=True): return task_result[1] else: return task_guid + + @classmethod + def delete_vdisk(cls, vdisk_guid, timeout=GET_CONFIG_PARAMS_TIMEOUT): + """ + Delete a specific vdisk + :param vdisk_guid: guid of the vdisk to delete + :type vdisk_guid: str + :param timeout: time to wait for the task to complete + :type timeout: int + :return: bool + """ + task_guid = cls.api.delete(api='/vdisks/{0}'.format(vdisk_guid)) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) + + if not task_result[0]: + error_msg = "Deleting of vDisk `{0}` has failed with error {1}".format(vdisk_guid, task_result[1]) + VDiskHelper.LOGGER.error(error_msg) + raise RuntimeError(error_msg) + else: + VDiskHelper.LOGGER.info("Deleting of vDisk `{0}` should have succeeded".format(vdisk_guid)) + return True diff --git a/remove/vdisk.py b/remove/vdisk.py index 7c8664a..e71efc9 100644 --- a/remove/vdisk.py +++ b/remove/vdisk.py @@ -75,7 +75,7 @@ def remove_snapshot(cls, snapshot_guid, vdisk_name, vpool_name, timeout=REMOVE_S task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = "Deleting snapshot `{0}` for vdisk `{1}` has failed".format(snapshot_guid, vdisk_name) + error_msg = "Task `{0}`: deleting snapshot `{1}` for vdisk `{2}` has failed".format(task_guid, snapshot_guid, vdisk_name) VDiskRemover.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: @@ -97,11 +97,11 @@ def remove_vdisk(cls, vdisk_guid, timeout=REMOVE_VDISK_TIMEOUT, *args, **kwargs) task_guid = cls.api.post(api='vdisks/{0}/delete'.format(vdisk_guid)) task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = "Deleting vDisk `{0}` has failed".format(vdisk_guid) + error_msg = "Task `{0}`: deleting vDisk `{1}` has failed".format(task_guid, vdisk_guid) VDiskRemover.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: - VDiskRemover.LOGGER.info("Deleting vDisk `{0}` should have succeeded".format(vdisk_guid)) + VDiskRemover.LOGGER.info("Task `{0}`: deleting vDisk `{1}` should have succeeded".format(task_guid, vdisk_guid)) return True @classmethod @@ -139,7 +139,7 @@ def remove_vtemplate_by_name(cls, vdisk_name, vpool_name, timeout=REMOVE_VTEMPLA task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = "Deleting vTemplate `{0}` has failed".format(vdisk_name) + error_msg = "Task `{0}`: deleting vTemplate `{1}` has failed".format(task_guid, vdisk_name) VDiskRemover.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: diff --git a/setup/arakoon.py b/setup/arakoon.py index a0381f1..77e4867 100644 --- a/setup/arakoon.py +++ b/setup/arakoon.py @@ -77,7 +77,10 @@ def add_arakoon(cluster_name, storagerouter_ip, cluster_basedir, base_dir=cluster_basedir, plugins=plugins, locked=False, - internal=False) + internal=False, + log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server'), + crash_log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server_crash') + ) if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: client.run(['ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)]) elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: @@ -128,7 +131,9 @@ def extend_arakoon(cluster_name, master_storagerouter_ip, storagerouter_ip, clus arakoon_installer.load() arakoon_installer.extend_cluster(new_ip=storagerouter_ip, base_dir=cluster_basedir, - locked=False) + locked=False, + log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server'), + crash_log_sinks=LogHandler.get_sink_path('automation_lib_arakoon_server_crash')) if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: client.run(['ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)]) elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: @@ -194,6 +199,5 @@ def setup_external_arakoons(backend): clustered_nodes=external_arakoon_mapping[arakoon_name]['all']) return external_arakoon_mapping else: - ArakoonSetup.LOGGER.info("Skipping external arakoon creation because backend `{0}` already exists" - .format(backend['name'])) + ArakoonSetup.LOGGER.info("Skipping external arakoon creation because backend `{0}` already exists".format(backend['name'])) return diff --git a/setup/backend.py b/setup/backend.py index 64179b7..2487429 100644 --- a/setup/backend.py +++ b/setup/backend.py @@ -18,8 +18,8 @@ from ..helpers.albanode import AlbaNodeHelper from ..helpers.backend import BackendHelper from ..helpers.ci_constants import CIConstants -from ..validate.decorators import required_roles, required_backend, required_preset, check_backend, check_preset, \ - check_linked_backend, filter_osds +from ..helpers.exceptions import PresetNotFoundError +from ..validate.decorators import required_roles, required_backend, required_preset, check_backend, check_linked_backend, filter_osds class BackendSetup(CIConstants): @@ -95,7 +95,6 @@ def add_backend(cls, backend_name, scaling='LOCAL', timeout=BACKEND_TIMEOUT, max return False @classmethod - @check_preset @required_backend def add_preset(cls, albabackend_name, preset_details, timeout=ADD_PRESET_TIMEOUT, *args, **kwargs): """ @@ -120,6 +119,15 @@ def add_preset(cls, albabackend_name, preset_details, timeout=ADD_PRESET_TIMEOUT :return: success or not :rtype: bool """ + + # CHECK PRESET + try: + BackendHelper.get_preset_by_albabackend(preset_details['name'], albabackend_name) + BackendSetup.LOGGER.info("Preset `{0}` already exists.".format(preset_details['name'])) + return True + except PresetNotFoundError: + pass + # BUILD_PRESET preset = {'name': preset_details['name'], 'policies': preset_details['policies'], diff --git a/setup/vdisk.py b/setup/vdisk.py index 1914370..d47a6bc 100644 --- a/setup/vdisk.py +++ b/setup/vdisk.py @@ -290,11 +290,11 @@ def create_from_template(cls, vdisk_name, vpool_name, new_vdisk_name, storagerou task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = "Creating vTemplate `{0}` has failed with error {1}".format(vdisk_name, task_result[1]) + error_msg = "Creating `{0}` from vTemplate `{1}` has failed with error {2}".format(official_new_vdisk_name, vdisk_name, task_result[1]) VDiskSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: - VDiskSetup.LOGGER.info("Creating vTemplate `{0}` should have succeeded".format(vdisk_name)) + VDiskSetup.LOGGER.info("Creating `{0}` from vTemplate `{1}` should have succeeded".format(official_new_vdisk_name, vdisk_name)) return task_result[1] @classmethod diff --git a/setup/vpool.py b/setup/vpool.py index 5dccbc8..d2a1843 100644 --- a/setup/vpool.py +++ b/setup/vpool.py @@ -13,28 +13,35 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. -from ovs.extensions.generic.configuration import Configuration -from ovs.lib.generic import GenericController + from ovs.log.log_handler import LogHandler +from ovs_extensions.generic.toolbox import ExtensionsToolbox +from ovs.lib.generic import GenericController from ..helpers.backend import BackendHelper +from ..helpers.ci_constants import CIConstants from ..helpers.storagerouter import StoragerouterHelper +from ..helpers.storagedriver import StoragedriverHelper from ..helpers.vpool import VPoolHelper from ..validate.decorators import required_roles, check_vpool -class VPoolSetup(object): +class VPoolSetup(CIConstants): + LOGGER = LogHandler.get(source='setup', name='ci_vpool_setup') ADD_VPOOL_TIMEOUT = 500 REQUIRED_VPOOL_ROLES = ['DB', 'WRITE', 'DTL'] + # These will be all possible settings for the StorageDriver. Messing them up is their own responsibility (they should not bypass the API by default!!) + STORAGEDRIVER_PARAMS = {"volume_manager": (dict, None, False), + "backend_connection_manager": (dict, None, False)} + def __init__(self): pass - @staticmethod + @classmethod @check_vpool @required_roles(REQUIRED_VPOOL_ROLES, 'LOCAL') - def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, timeout=ADD_VPOOL_TIMEOUT, *args, - **kwargs): + def add_vpool(cls, vpool_name, vpool_details, storagerouter_ip, proxy_amount=2, timeout=ADD_VPOOL_TIMEOUT, *args, **kwargs): """ Adds a VPool to a storagerouter @@ -44,8 +51,6 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, :type vpool_details: dict :param timeout: specify a timeout :type timeout: int - :param api: specify a valid api connection to the setup - :type api: helpers.api.OVSClient :param storagerouter_ip: ip of the storagerouter to add the vpool too :type storagerouter_ip: str :param proxy_amount: amount of proxies for this vpool @@ -57,9 +62,8 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, # Build ADD_VPOOL parameters call_parameters = { 'vpool_name': vpool_name, - 'backend_info': { - 'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['backend_name']).guid, - 'preset': vpool_details['preset']}, + 'backend_info': {'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['backend_name']).guid, + 'preset': vpool_details['preset']}, 'connection_info': {'host': '', 'port': '', 'client_id': '', 'client_secret': ''}, 'storage_ip': vpool_details['storage_ip'], 'storagerouter_ip': storagerouter_ip, @@ -75,10 +79,13 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, } api_data = {'call_parameters': call_parameters} + # Setting for mds_safety + if vpool_details.get('mds_safety') is not None: + call_parameters['mds_config_params'] = {'mds_safety': vpool_details['mds_safety']} + # Setting possible alba accelerated alba if vpool_details['fragment_cache']['location'] == 'backend': - call_parameters['backend_info_aa'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name( - vpool_details['fragment_cache']['backend']['name']).guid, + call_parameters['backend_info_aa'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['fragment_cache']['backend']['name']).guid, 'preset': vpool_details['fragment_cache']['backend']['preset']} call_parameters['connection_info_aa'] = {'host': '', 'port': '', 'client_id': '', 'client_secret': ''} elif vpool_details['fragment_cache']['location'] == 'disk': @@ -93,8 +100,7 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, call_parameters['block_cache_on_read'] = vpool_details['block_cache']['strategy']['cache_on_read'] call_parameters['block_cache_on_write'] = vpool_details['block_cache']['strategy']['cache_on_write'] if vpool_details['block_cache']['location'] == 'backend': - call_parameters['backend_info_bc'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name( - vpool_details['block_cache']['backend']['name']).guid, + call_parameters['backend_info_bc'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['block_cache']['backend']['name']).guid, 'preset': vpool_details['block_cache']['backend']['preset']} call_parameters['connection_info_bc'] = {'host': '', 'port': '', 'client_id': '', 'client_secret': ''} elif vpool_details['block_cache']['location'] == 'disk': # Ignore disk @@ -105,37 +111,33 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, VPoolSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) - task_guid = api.post( + task_guid = cls.api.post( api='/storagerouters/{0}/add_vpool/'.format( - StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip)), + StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip).guid), data=api_data ) - task_result = api.wait_for_task(task_id=task_guid, timeout=timeout) + task_result = cls.api.wait_for_task(task_id=task_guid, timeout=timeout) if not task_result[0]: - error_msg = 'vPool {0} has failed to create on storagerouter {1} because: {2}'.format(vpool_name, - storagerouter_ip, - task_result[1]) + error_msg = 'vPool {0} has failed to create on storagerouter {1} because: {2}'.format(vpool_name, storagerouter_ip, task_result[1]) VPoolSetup.LOGGER.error(error_msg) raise RuntimeError(error_msg) else: - VPoolSetup.LOGGER.info( - 'Creation of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, - storagerouter_ip)) - - if vpool_details.get('mds_safety') is not None: - vpool = VPoolHelper.get_vpool_by_name(vpool_name) - - if vpool: - mds_config_path = '/ovs/vpools/{0}/mds_config'.format(vpool.guid) - mds_config = Configuration.get(mds_config_path) - mds_config['mds_safety'] = vpool_details.get('mds_safety') - Configuration.set(mds_config_path, mds_config) - else: - error_msg = 'Unable to change the mds safety of vPool {0}.'.format(vpool_name) - VPoolSetup.LOGGER.error(error_msg) - raise RuntimeError(error_msg) - - return storagerouter_ip, '/mnt/{0}'.format(vpool_name) + VPoolSetup.LOGGER.info('Creation of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) + + # Settings volumedriver + storagedriver_config = vpool_details.get('storagedriver') + if storagedriver_config is not None: + ExtensionsToolbox.verify_required_params(VPoolSetup.STORAGEDRIVER_PARAMS, storagedriver_config) + VPoolSetup.LOGGER.info('Updating volumedriver configuration of vPool `{0}` on storagerouter `{1}`.'.format(vpool_name, storagerouter_ip)) + vpool = VPoolHelper.get_vpool_by_name(vpool_name) + storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter.ip == storagerouter_ip][0] + if not storagedriver: + error_msg = 'Unable to find the storagedriver of vPool {0} on storagerouter {1}'.format(vpool_name, storagerouter_ip) + raise RuntimeError(error_msg) + StoragedriverHelper.change_config(storagedriver, storagedriver_config) + VPoolSetup.LOGGER.info('Updating volumedriver config of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) + + return storagerouter_ip, '/mnt/{0}'.format(vpool_name) @staticmethod def execute_scrubbing(): diff --git a/validate/decorators.py b/validate/decorators.py index 29cd17a..240d6e8 100644 --- a/validate/decorators.py +++ b/validate/decorators.py @@ -317,24 +317,3 @@ def validate(*args, **kwargs): else: raise AttributeError("Missing parameter: backend_name") return validate - - -def check_preset(func): - """ - Check if a preset is already present on a backend - - :param func: function - :type func: Function - """ - - def validate(*args, **kwargs): - if kwargs['albabackend_name'] and kwargs['preset_details']: - if not BackendValidation.check_preset_on_backend(preset_name=kwargs['preset_details']['name'], - albabackend_name=kwargs['albabackend_name']): - # if the preset is not yet created, create it - return func(*args, **kwargs) - else: - return - else: - raise AttributeError("Missing parameter: backend_name") - return validate diff --git a/validate/roles.py b/validate/roles.py index e832057..8bd288f 100644 --- a/validate/roles.py +++ b/validate/roles.py @@ -15,6 +15,7 @@ # but WITHOUT ANY WARRANTY of any kind. from ovs.log.log_handler import LogHandler from ..helpers.disk import DiskHelper +from ..helpers.storagerouter import StoragerouterHelper class RoleValidation(object): @@ -43,7 +44,8 @@ def check_required_roles(roles, storagerouter_ip=None, location="GLOBAL"): # fetch availabe roles if location == "LOCAL": # LOCAL - available_roles = DiskHelper.get_roles_from_disks(storagerouter_ip=storagerouter_ip) + storagerouter_guid = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip).guid + available_roles = DiskHelper.get_roles_from_disks(storagerouter_guid=storagerouter_guid) else: # GLOBAL available_roles = DiskHelper.get_roles_from_disks()