gvsig-scripting / org.gvsig.scripting / trunk / org.gvsig.scripting / org.gvsig.scripting.app / org.gvsig.scripting.app.mainplugin / src / main / resources-plugin / scripting / lib / requests / packages / urllib3 / util / retry.py @ 564
History | View | Annotate | Download (9.75 KB)
1 |
from __future__ import absolute_import |
---|---|
2 |
import time |
3 |
import logging |
4 |
|
5 |
from ..exceptions import ( |
6 |
ConnectTimeoutError, |
7 |
MaxRetryError, |
8 |
ProtocolError, |
9 |
ReadTimeoutError, |
10 |
ResponseError, |
11 |
) |
12 |
from ..packages import six |
13 |
|
14 |
|
15 |
log = logging.getLogger(__name__) |
16 |
|
17 |
|
18 |
class Retry(object): |
19 |
""" Retry configuration.
|
20 |
|
21 |
Each retry attempt will create a new Retry object with updated values, so
|
22 |
they can be safely reused.
|
23 |
|
24 |
Retries can be defined as a default for a pool::
|
25 |
|
26 |
retries = Retry(connect=5, read=2, redirect=5)
|
27 |
http = PoolManager(retries=retries)
|
28 |
response = http.request('GET', 'http://example.com/')
|
29 |
|
30 |
Or per-request (which overrides the default for the pool)::
|
31 |
|
32 |
response = http.request('GET', 'http://example.com/', retries=Retry(10))
|
33 |
|
34 |
Retries can be disabled by passing ``False``::
|
35 |
|
36 |
response = http.request('GET', 'http://example.com/', retries=False)
|
37 |
|
38 |
Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
|
39 |
retries are disabled, in which case the causing exception will be raised.
|
40 |
|
41 |
:param int total:
|
42 |
Total number of retries to allow. Takes precedence over other counts.
|
43 |
|
44 |
Set to ``None`` to remove this constraint and fall back on other
|
45 |
counts. It's a good idea to set this to some sensibly-high value to
|
46 |
account for unexpected edge cases and avoid infinite retry loops.
|
47 |
|
48 |
Set to ``0`` to fail on the first retry.
|
49 |
|
50 |
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
51 |
|
52 |
:param int connect:
|
53 |
How many connection-related errors to retry on.
|
54 |
|
55 |
These are errors raised before the request is sent to the remote server,
|
56 |
which we assume has not triggered the server to process the request.
|
57 |
|
58 |
Set to ``0`` to fail on the first retry of this type.
|
59 |
|
60 |
:param int read:
|
61 |
How many times to retry on read errors.
|
62 |
|
63 |
These errors are raised after the request was sent to the server, so the
|
64 |
request may have side-effects.
|
65 |
|
66 |
Set to ``0`` to fail on the first retry of this type.
|
67 |
|
68 |
:param int redirect:
|
69 |
How many redirects to perform. Limit this to avoid infinite redirect
|
70 |
loops.
|
71 |
|
72 |
A redirect is a HTTP response with a status code 301, 302, 303, 307 or
|
73 |
308.
|
74 |
|
75 |
Set to ``0`` to fail on the first retry of this type.
|
76 |
|
77 |
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
78 |
|
79 |
:param iterable method_whitelist:
|
80 |
Set of uppercased HTTP method verbs that we should retry on.
|
81 |
|
82 |
By default, we only retry on methods which are considered to be
|
83 |
indempotent (multiple requests with the same parameters end with the
|
84 |
same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
|
85 |
|
86 |
:param iterable status_forcelist:
|
87 |
A set of HTTP status codes that we should force a retry on.
|
88 |
|
89 |
By default, this is disabled with ``None``.
|
90 |
|
91 |
:param float backoff_factor:
|
92 |
A backoff factor to apply between attempts. urllib3 will sleep for::
|
93 |
|
94 |
{backoff factor} * (2 ^ ({number of total retries} - 1))
|
95 |
|
96 |
seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
|
97 |
for [0.1s, 0.2s, 0.4s, ...] between retries. It will never be longer
|
98 |
than :attr:`Retry.BACKOFF_MAX`.
|
99 |
|
100 |
By default, backoff is disabled (set to 0).
|
101 |
|
102 |
:param bool raise_on_redirect: Whether, if the number of redirects is
|
103 |
exhausted, to raise a MaxRetryError, or to return a response with a
|
104 |
response code in the 3xx range.
|
105 |
"""
|
106 |
|
107 |
DEFAULT_METHOD_WHITELIST = frozenset([
|
108 |
'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) |
109 |
|
110 |
#: Maximum backoff time.
|
111 |
BACKOFF_MAX = 120
|
112 |
|
113 |
def __init__(self, total=10, connect=None, read=None, redirect=None, |
114 |
method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
|
115 |
backoff_factor=0, raise_on_redirect=True, _observed_errors=0): |
116 |
|
117 |
self.total = total
|
118 |
self.connect = connect
|
119 |
self.read = read
|
120 |
|
121 |
if redirect is False or total is False: |
122 |
redirect = 0
|
123 |
raise_on_redirect = False
|
124 |
|
125 |
self.redirect = redirect
|
126 |
self.status_forcelist = status_forcelist or set() |
127 |
self.method_whitelist = method_whitelist
|
128 |
self.backoff_factor = backoff_factor
|
129 |
self.raise_on_redirect = raise_on_redirect
|
130 |
self._observed_errors = _observed_errors # TODO: use .history instead? |
131 |
|
132 |
def new(self, **kw): |
133 |
params = dict(
|
134 |
total=self.total,
|
135 |
connect=self.connect, read=self.read, redirect=self.redirect, |
136 |
method_whitelist=self.method_whitelist,
|
137 |
status_forcelist=self.status_forcelist,
|
138 |
backoff_factor=self.backoff_factor,
|
139 |
raise_on_redirect=self.raise_on_redirect,
|
140 |
_observed_errors=self._observed_errors,
|
141 |
) |
142 |
params.update(kw) |
143 |
return type(self)(**params) |
144 |
|
145 |
@classmethod
|
146 |
def from_int(cls, retries, redirect=True, default=None): |
147 |
""" Backwards-compatibility for the old retries format."""
|
148 |
if retries is None: |
149 |
retries = default if default is not None else cls.DEFAULT |
150 |
|
151 |
if isinstance(retries, Retry): |
152 |
return retries
|
153 |
|
154 |
redirect = bool(redirect) and None |
155 |
new_retries = cls(retries, redirect=redirect) |
156 |
log.debug("Converted retries value: %r -> %r" % (retries, new_retries))
|
157 |
return new_retries
|
158 |
|
159 |
def get_backoff_time(self): |
160 |
""" Formula for computing the current backoff
|
161 |
|
162 |
:rtype: float
|
163 |
"""
|
164 |
if self._observed_errors <= 1: |
165 |
return 0 |
166 |
|
167 |
backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1)) |
168 |
return min(self.BACKOFF_MAX, backoff_value) |
169 |
|
170 |
def sleep(self): |
171 |
""" Sleep between retry attempts using an exponential backoff.
|
172 |
|
173 |
By default, the backoff factor is 0 and this method will return
|
174 |
immediately.
|
175 |
"""
|
176 |
backoff = self.get_backoff_time()
|
177 |
if backoff <= 0: |
178 |
return
|
179 |
time.sleep(backoff) |
180 |
|
181 |
def _is_connection_error(self, err): |
182 |
""" Errors when we're fairly sure that the server did not receive the
|
183 |
request, so it should be safe to retry.
|
184 |
"""
|
185 |
return isinstance(err, ConnectTimeoutError) |
186 |
|
187 |
def _is_read_error(self, err): |
188 |
""" Errors that occur after the request has been started, so we should
|
189 |
assume that the server began processing it.
|
190 |
"""
|
191 |
return isinstance(err, (ReadTimeoutError, ProtocolError)) |
192 |
|
193 |
def is_forced_retry(self, method, status_code): |
194 |
""" Is this method/status code retryable? (Based on method/codes whitelists)
|
195 |
"""
|
196 |
if self.method_whitelist and method.upper() not in self.method_whitelist: |
197 |
return False |
198 |
|
199 |
return self.status_forcelist and status_code in self.status_forcelist |
200 |
|
201 |
def is_exhausted(self): |
202 |
""" Are we out of retries? """
|
203 |
retry_counts = (self.total, self.connect, self.read, self.redirect) |
204 |
retry_counts = list(filter(None, retry_counts)) |
205 |
if not retry_counts: |
206 |
return False |
207 |
|
208 |
return min(retry_counts) < 0 |
209 |
|
210 |
def increment(self, method=None, url=None, response=None, error=None, |
211 |
_pool=None, _stacktrace=None): |
212 |
""" Return a new Retry object with incremented retry counters.
|
213 |
|
214 |
:param response: A response object, or None, if the server did not
|
215 |
return a response.
|
216 |
:type response: :class:`~urllib3.response.HTTPResponse`
|
217 |
:param Exception error: An error encountered during the request, or
|
218 |
None if the response was received successfully.
|
219 |
|
220 |
:return: A new ``Retry`` object.
|
221 |
"""
|
222 |
if self.total is False and error: |
223 |
# Disabled, indicate to re-raise the error.
|
224 |
raise six.reraise(type(error), error, _stacktrace) |
225 |
|
226 |
total = self.total
|
227 |
if total is not None: |
228 |
total -= 1
|
229 |
|
230 |
_observed_errors = self._observed_errors
|
231 |
connect = self.connect
|
232 |
read = self.read
|
233 |
redirect = self.redirect
|
234 |
cause = 'unknown'
|
235 |
|
236 |
if error and self._is_connection_error(error): |
237 |
# Connect retry?
|
238 |
if connect is False: |
239 |
raise six.reraise(type(error), error, _stacktrace) |
240 |
elif connect is not None: |
241 |
connect -= 1
|
242 |
_observed_errors += 1
|
243 |
|
244 |
elif error and self._is_read_error(error): |
245 |
# Read retry?
|
246 |
if read is False: |
247 |
raise six.reraise(type(error), error, _stacktrace) |
248 |
elif read is not None: |
249 |
read -= 1
|
250 |
_observed_errors += 1
|
251 |
|
252 |
elif response and response.get_redirect_location(): |
253 |
# Redirect retry?
|
254 |
if redirect is not None: |
255 |
redirect -= 1
|
256 |
cause = 'too many redirects'
|
257 |
|
258 |
else:
|
259 |
# Incrementing because of a server error like a 500 in
|
260 |
# status_forcelist and a the given method is in the whitelist
|
261 |
_observed_errors += 1
|
262 |
cause = ResponseError.GENERIC_ERROR |
263 |
if response and response.status: |
264 |
cause = ResponseError.SPECIFIC_ERROR.format( |
265 |
status_code=response.status) |
266 |
|
267 |
new_retry = self.new(
|
268 |
total=total, |
269 |
connect=connect, read=read, redirect=redirect, |
270 |
_observed_errors=_observed_errors) |
271 |
|
272 |
if new_retry.is_exhausted():
|
273 |
raise MaxRetryError(_pool, url, error or ResponseError(cause)) |
274 |
|
275 |
log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry))
|
276 |
|
277 |
return new_retry
|
278 |
|
279 |
def __repr__(self): |
280 |
return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' |
281 |
'read={self.read}, redirect={self.redirect})').format(
|
282 |
cls=type(self), self=self) |
283 |
|
284 |
|
285 |
# For backwards compatibility (equivalent to pre-v1.9):
|
286 |
Retry.DEFAULT = Retry(3)
|