1
2
3
4
5 from builtins import range
6 from builtins import object
7 import functools
8 import inspect
9 import pycurl
10 import time
11
12 from collections import deque
13
14 import arvados.errors
15
16 _HTTP_SUCCESSES = set(range(200, 300))
17 _HTTP_CAN_RETRY = set([408, 409, 422, 423, 500, 502, 503, 504])
20 """Coordinate limited retries of code.
21
22 RetryLoop coordinates a loop that runs until it records a
23 successful result or tries too many times, whichever comes first.
24 Typical use looks like:
25
26 loop = RetryLoop(num_retries=2)
27 for tries_left in loop:
28 try:
29 result = do_something()
30 except TemporaryError as error:
31 log("error: {} ({} tries left)".format(error, tries_left))
32 else:
33 loop.save_result(result)
34 if loop.success():
35 return loop.last_result()
36 """
37 - def __init__(self, num_retries, success_check=lambda r: True,
38 backoff_start=0, backoff_growth=2, save_results=1,
39 max_wait=60):
40 """Construct a new RetryLoop.
41
42 Arguments:
43 * num_retries: The maximum number of times to retry the loop if it
44 doesn't succeed. This means the loop could run at most 1+N times.
45 * success_check: This is a function that will be called each
46 time the loop saves a result. The function should return
47 True if the result indicates loop success, False if it
48 represents a permanent failure state, and None if the loop
49 should continue. If no function is provided, the loop will
50 end as soon as it records any result.
51 * backoff_start: The number of seconds that must pass before the
52 loop's second iteration. Default 0, which disables all waiting.
53 * backoff_growth: The wait time multiplier after each iteration.
54 Default 2 (i.e., double the wait time each time).
55 * save_results: Specify a number to save the last N results
56 that the loop recorded. These records are available through
57 the results attribute, oldest first. Default 1.
58 * max_wait: Maximum number of seconds to wait between retries.
59 """
60 self.tries_left = num_retries + 1
61 self.check_result = success_check
62 self.backoff_wait = backoff_start
63 self.backoff_growth = backoff_growth
64 self.max_wait = max_wait
65 self.next_start_time = 0
66 self.results = deque(maxlen=save_results)
67 self._running = None
68 self._success = None
69
72
74 return self._running and (self._success is None)
75
77 if self._running is None:
78 self._running = True
79 if (self.tries_left < 1) or not self.running():
80 self._running = False
81 raise StopIteration
82 else:
83 wait_time = max(0, self.next_start_time - time.time())
84 time.sleep(wait_time)
85 self.backoff_wait *= self.backoff_growth
86 if self.backoff_wait > self.max_wait:
87 self.backoff_wait = self.max_wait
88 self.next_start_time = time.time() + self.backoff_wait
89 self.tries_left -= 1
90 return self.tries_left
91
93 """Record a loop result.
94
95 Save the given result, and end the loop if it indicates
96 success or permanent failure. See __init__'s documentation
97 about success_check to learn how to make that indication.
98 """
99 if not self.running():
100 raise arvados.errors.AssertionError(
101 "recorded a loop result after the loop finished")
102 self.results.append(result)
103 self._success = self.check_result(result)
104
106 """Return the loop's end state.
107
108 Returns True if the loop obtained a successful result, False if it
109 encountered permanent failure, or else None.
110 """
111 return self._success
112
114 """Return the most recent result the loop recorded."""
115 try:
116 return self.results[-1]
117 except IndexError:
118 raise arvados.errors.AssertionError(
119 "queried loop results before any were recorded")
120
123 """Convert an HTTP status code to a loop control flag.
124
125 Pass this method a numeric HTTP status code. It returns True if
126 the code indicates success, None if it indicates temporary
127 failure, and False otherwise. You can use this as the
128 success_check for a RetryLoop.
129
130 Implementation details:
131 * Any 2xx result returns True.
132 * A select few status codes, or any malformed responses, return None.
133 422 Unprocessable Entity is in this category. This may not meet the
134 letter of the HTTP specification, but the Arvados API server will
135 use it for various server-side problems like database connection
136 errors.
137 * Everything else returns False. Note that this includes 1xx and
138 3xx status codes. They don't indicate success, and you can't
139 retry those requests verbatim.
140 """
141 if status_code in _HTTP_SUCCESSES:
142 return True
143 elif status_code in _HTTP_CAN_RETRY:
144 return None
145 elif 100 <= status_code < 600:
146 return False
147 else:
148 return None
149
151 """Provide a default value for a method's num_retries argument.
152
153 This is a decorator for instance and class methods that accept a
154 num_retries argument, with a None default. When the method is called
155 without a value for num_retries, it will be set from the underlying
156 instance or class' num_retries attribute.
157 """
158 @functools.wraps(orig_func)
159 def num_retries_setter(self, *args, **kwargs):
160 if kwargs.get('num_retries') is None:
161 kwargs['num_retries'] = self.num_retries
162 return orig_func(self, *args, **kwargs)
163 return num_retries_setter
164