Module arvados.retry

class RetryLoop(object):
    """Coordinate limited retries of code.

    RetryLoop coordinates a loop that runs until it records a
    successful result or tries too many times, whichever comes first.
    Typical use looks like:

        loop = RetryLoop(num_retries=2)
        for tries_left in loop:
            try:
                result = do_something()
            except TemporaryError as error:
                log("error: {} ({} tries left)".format(error, tries_left))
            else:
                loop.save_result(result)
        if loop.success():
            return loop.last_result()
    """
    def __init__(self, num_retries, success_check=lambda r: True,
                 backoff_start=0, backoff_growth=2, save_results=1,
                 max_wait=60):
        """Construct a new RetryLoop.

        Arguments:
        * num_retries: The maximum number of times to retry the loop if it
          doesn't succeed.  This means the loop could run at most 1+N times.
        * success_check: This is a function that will be called each
          time the loop saves a result.  The function should return
          True if the result indicates loop success, False if it
          represents a permanent failure state, and None if the loop
          should continue.  If no function is provided, the loop will
          end as soon as it records any result.
        * backoff_start: The number of seconds that must pass before the
          loop's second iteration.  Default 0, which disables all waiting.
        * backoff_growth: The wait time multiplier after each iteration.
          Default 2 (i.e., double the wait time each time).
        * save_results: Specify a number to save the last N results
          that the loop recorded.  These records are available through
          the results attribute, oldest first.  Default 1.
        * max_wait: Maximum number of seconds to wait between retries.
        """
        self.tries_left = num_retries + 1
        self.check_result = success_check
        self.backoff_wait = backoff_start
        self.backoff_growth = backoff_growth
        self.max_wait = max_wait
        self.next_start_time = 0
        self.results = deque(maxlen=save_results)
        self._attempts = 0
        self._running = None
        self._success = None

    def __iter__(self):
        return self

    def running(self):
        return self._running and (self._success is None)

    def __next__(self):
        if self._running is None:
            self._running = True
        if (self.tries_left < 1) or not self.running():
            self._running = False
            raise StopIteration
        else:
            wait_time = max(0, self.next_start_time - time.time())
            time.sleep(wait_time)
            self.backoff_wait *= self.backoff_growth
            if self.backoff_wait > self.max_wait:
                self.backoff_wait = self.max_wait
        self.next_start_time = time.time() + self.backoff_wait
        self.tries_left -= 1
        return self.tries_left

    def save_result(self, result):
        """Record a loop result.

        Save the given result, and end the loop if it indicates
        success or permanent failure.  See __init__'s documentation
        about success_check to learn how to make that indication.
        """
        if not self.running():
            raise arvados.errors.AssertionError(
                "recorded a loop result after the loop finished")
        self.results.append(result)
        self._success = self.check_result(result)
        self._attempts += 1

    def success(self):
        """Return the loop's end state.

        Returns True if the loop obtained a successful result, False if it
        encountered permanent failure, or else None.
        """
        return self._success

    def last_result(self):
        """Return the most recent result the loop recorded."""
        try:
            return self.results[-1]
        except IndexError:
            raise arvados.errors.AssertionError(
                "queried loop results before any were recorded")

    def attempts(self):
        """Return the number of attempts that have been made.

        Includes successes and failures."""
        return self._attempts

    def attempts_str(self):
        """Human-readable attempts(): 'N attempts' or '1 attempt'"""
        if self._attempts == 1:
            return '1 attempt'
        else:
            return '{} attempts'.format(self._attempts)