|
@@ -0,0 +1,123 @@
|
|
|
|
|
+import functools
|
|
|
|
|
+import random
|
|
|
|
|
+import time
|
|
|
|
|
+from functools import partial
|
|
|
|
|
+
|
|
|
|
|
+from module.logger import logger as logging_logger
|
|
|
|
|
+
|
|
|
|
|
+"""
|
|
|
|
|
+Copied from Alas
|
|
|
|
|
+"""
|
|
|
|
|
+
|
|
|
|
|
+try:
|
|
|
|
|
+ from decorator import decorator
|
|
|
|
|
+except ImportError:
|
|
|
|
|
+ def decorator(caller):
|
|
|
|
|
+ """ Turns caller into a decorator.
|
|
|
|
|
+ Unlike decorator module, function signature is not preserved.
|
|
|
|
|
+
|
|
|
|
|
+ :param caller: caller(f, *args, **kwargs)
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ def decor(f):
|
|
|
|
|
+ @functools.wraps(f)
|
|
|
|
|
+ def wrapper(*args, **kwargs):
|
|
|
|
|
+ return caller(f, *args, **kwargs)
|
|
|
|
|
+
|
|
|
|
|
+ return wrapper
|
|
|
|
|
+
|
|
|
|
|
+ return decor
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def __retry_internal(f, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0,
|
|
|
|
|
+ logger=logging_logger):
|
|
|
|
|
+ """
|
|
|
|
|
+ Executes a function and retries it if it failed.
|
|
|
|
|
+
|
|
|
|
|
+ :param f: the function to execute.
|
|
|
|
|
+ :param exceptions: an exception or a tuple of exceptions to catch. default: Exception.
|
|
|
|
|
+ :param tries: the maximum number of attempts. default: -1 (infinite).
|
|
|
|
|
+ :param delay: initial delay between attempts. default: 0.
|
|
|
|
|
+ :param max_delay: the maximum value of delay. default: None (no limit).
|
|
|
|
|
+ :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff).
|
|
|
|
|
+ :param jitter: extra seconds added to delay between attempts. default: 0.
|
|
|
|
|
+ fixed if a number, random if a range tuple (min, max)
|
|
|
|
|
+ :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
|
|
|
|
|
+ default: retry.logging_logger. if None, logging is disabled.
|
|
|
|
|
+ :returns: the result of the f function.
|
|
|
|
|
+ """
|
|
|
|
|
+ _tries, _delay = tries, delay
|
|
|
|
|
+ while _tries:
|
|
|
|
|
+ try:
|
|
|
|
|
+ return f()
|
|
|
|
|
+ except exceptions as e:
|
|
|
|
|
+ _tries -= 1
|
|
|
|
|
+ if not _tries:
|
|
|
|
|
+ # Difference, raise same exception
|
|
|
|
|
+ raise e
|
|
|
|
|
+
|
|
|
|
|
+ if logger is not None:
|
|
|
|
|
+ # Difference, show exception
|
|
|
|
|
+ logger.exception(e)
|
|
|
|
|
+ logger.warning(f'{type(e).__name__}({e}), retrying in {_delay} seconds...')
|
|
|
|
|
+
|
|
|
|
|
+ time.sleep(_delay)
|
|
|
|
|
+ _delay *= backoff
|
|
|
|
|
+
|
|
|
|
|
+ if isinstance(jitter, tuple):
|
|
|
|
|
+ _delay += random.uniform(*jitter)
|
|
|
|
|
+ else:
|
|
|
|
|
+ _delay += jitter
|
|
|
|
|
+
|
|
|
|
|
+ if max_delay is not None:
|
|
|
|
|
+ _delay = min(_delay, max_delay)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def retry(exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, logger=logging_logger):
|
|
|
|
|
+ """Returns a retry decorator.
|
|
|
|
|
+
|
|
|
|
|
+ :param exceptions: an exception or a tuple of exceptions to catch. default: Exception.
|
|
|
|
|
+ :param tries: the maximum number of attempts. default: -1 (infinite).
|
|
|
|
|
+ :param delay: initial delay between attempts. default: 0.
|
|
|
|
|
+ :param max_delay: the maximum value of delay. default: None (no limit).
|
|
|
|
|
+ :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff).
|
|
|
|
|
+ :param jitter: extra seconds added to delay between attempts. default: 0.
|
|
|
|
|
+ fixed if a number, random if a range tuple (min, max)
|
|
|
|
|
+ :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
|
|
|
|
|
+ default: retry.logging_logger. if None, logging is disabled.
|
|
|
|
|
+ :returns: a retry decorator.
|
|
|
|
|
+ """
|
|
|
|
|
+
|
|
|
|
|
+ @decorator
|
|
|
|
|
+ def retry_decorator(f, *fargs, **fkwargs):
|
|
|
|
|
+ args = fargs if fargs else list()
|
|
|
|
|
+ kwargs = fkwargs if fkwargs else dict()
|
|
|
|
|
+ return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter,
|
|
|
|
|
+ logger)
|
|
|
|
|
+
|
|
|
|
|
+ return retry_decorator
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def retry_call(f, fargs=None, fkwargs=None, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1,
|
|
|
|
|
+ jitter=0,
|
|
|
|
|
+ logger=logging_logger):
|
|
|
|
|
+ """
|
|
|
|
|
+ Calls a function and re-executes it if it failed.
|
|
|
|
|
+
|
|
|
|
|
+ :param f: the function to execute.
|
|
|
|
|
+ :param fargs: the positional arguments of the function to execute.
|
|
|
|
|
+ :param fkwargs: the named arguments of the function to execute.
|
|
|
|
|
+ :param exceptions: an exception or a tuple of exceptions to catch. default: Exception.
|
|
|
|
|
+ :param tries: the maximum number of attempts. default: -1 (infinite).
|
|
|
|
|
+ :param delay: initial delay between attempts. default: 0.
|
|
|
|
|
+ :param max_delay: the maximum value of delay. default: None (no limit).
|
|
|
|
|
+ :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff).
|
|
|
|
|
+ :param jitter: extra seconds added to delay between attempts. default: 0.
|
|
|
|
|
+ fixed if a number, random if a range tuple (min, max)
|
|
|
|
|
+ :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
|
|
|
|
|
+ default: retry.logging_logger. if None, logging is disabled.
|
|
|
|
|
+ :returns: the result of the f function.
|
|
|
|
|
+ """
|
|
|
|
|
+ args = fargs if fargs else list()
|
|
|
|
|
+ kwargs = fkwargs if fkwargs else dict()
|
|
|
|
|
+ return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter, logger)
|