diff --git a/.gitignore b/.gitignore index b411a8fab..959b64ce2 100644 --- a/.gitignore +++ b/.gitignore @@ -121,4 +121,4 @@ venv_/ .vscode Pipfile -Pipfile.lock \ No newline at end of file +Pipfile.lock diff --git a/atlassian/rest_client.py b/atlassian/rest_client.py index 523d5056b..0b4a763c0 100644 --- a/atlassian/rest_client.py +++ b/atlassian/rest_client.py @@ -2,8 +2,8 @@ import logging from json import dumps +import random import requests - try: from oauthlib.oauth1.rfc5849 import SIGNATURE_RSA_SHA512 as SIGNATURE_RSA except ImportError: @@ -60,7 +60,46 @@ def __init__( cloud=False, proxies=None, token=None, + backoff_and_retry=False, + retry_error_matches=[(429, "Too Many Requests"), + (429, "Unknown Status Code")], + max_backoff_seconds=1800, + max_backoff_retries=1000, ): + """ + init function for the AtlassianRestAPI object. + + Args: + url (str): The url to be used in the request. + username (str, optional): Username Defaults to None. + password (sstr, optional): Password. Defaults to None. + timeout (int, optional): Request timeout. Defaults to 75. + api_root (str, optional): Root for the api requests. Defaults to "rest/api". + api_version (str, optional): Version of the API to use. Defaults to "latest". + verify_ssl (bool, optional): Turn on / off SSL verification. Defaults to True. + session ([type], optional): Pass an existing Python requests session object. Defaults to None. + oauth ([type], optional): oauth. Defaults to None. + oauth2 ([type], optional): oauth2. Defaults to None. + cookies ([type], optional): Cookies to send with the request. Defaults to None. + advanced_mode ([type], optional): Return results in advanced mode. Defaults to None. + kerberos ([type], optional): Kerberos. Defaults to None. + cloud (bool, optional): Specify if using Atlassian Cloud. Defaults to False. + proxies ([type], optional): Specify proxies to use. Defaults to None. + token ([type], optional): Atlassian / Jira auth token. Defaults to None. + backoff_and_retry (bool, optional): Enable exponential backoff and retry. + This will retry the request if there is a predefined failure. Primarily + designed for Atlassian Cloud where API limits are commonly hit if doing + operations on many issues, and the limits require a cooling off period. + The wait period before the next request increases exponentially with each + failed retry. Defaults to False. + retry_error_matches (list, optional): Errors to match, passed as a list of tuples + containing the response code and the response text to match (exact match). + Defaults to the rate limit error from Atlassian Cloud - [(429, 'Too Many Requests')]. + max_backoff_seconds (int, optional): Max backoff seconds. When backing off, requests won't + wait any longer than this. Defaults to 1800. + max_backoff_retries (int, optional): Maximum number of retries to try before + continuing. Defaults to 1000. + """ self.url = url self.username = username self.password = password @@ -72,6 +111,10 @@ def __init__( self.advanced_mode = advanced_mode self.cloud = cloud self.proxies = proxies + self.backoff_and_retry = backoff_and_retry + self.retry_error_matches = retry_error_matches + self.max_backoff_seconds = max_backoff_seconds + self.max_backoff_retries = max_backoff_retries if session is None: self._session = requests.Session() else: @@ -236,17 +279,35 @@ def request( data=data if data else json_dump, ) headers = headers or self.default_headers - response = self._session.request( - method=method, - url=url, - headers=headers, - data=data, - json=json, - timeout=self.timeout, - verify=self.verify_ssl, - files=files, - proxies=self.proxies, - ) + + backoff = 1 + retries = 0 + while True: + response = self._session.request( + method=method, + url=url, + headers=headers, + data=data, + json=json, + timeout=self.timeout, + verify=self.verify_ssl, + files=files, + proxies=self.proxies, + ) + if self.backoff_and_retry: + for em in self.retry_error_matches: + if retries > self.max_backoff_retries: + log.warning("Hit max backoff retry limit of {0}, no more retries.".format(self.max_backoff_retries)) + responseloop = False + break + if response.status_code == em[0] and response.reason == em[1]: + log.warning('Backing off due to error "{0}: {1}" for {2}s'.format(em[0], em[1], backoff)) + time.sleep(backoff + (random.random() * backoff / 10)) + backoff = min(2 * backoff, self.max_backoff_seconds) + retries += 1 + else: + break + response.encoding = "utf-8" log.debug("HTTP: %s %s -> %s %s", method, path, response.status_code, response.reason)