venv/lib/python3.11/site-packages/pip/_vendor/urllib3/util/retry.py

   1 from __future__ import absolute_import
   2
   3 import email
   4 import logging
   5 import re
   6 import time
   7 import warnings
   8 from collections import namedtuple
   9 from itertools import takewhile
  10
  11 from ..exceptions import (
  12     ConnectTimeoutError,
  13     InvalidHeader,
  14     MaxRetryError,
  15     ProtocolError,
  16     ProxyError,
  17     ReadTimeoutError,
  18     ResponseError,
  19 )
  20 from ..packages import six
  21
  22 log = logging.getLogger(__name__)
  23
  24
  25 # Data structure for representing the metadata of requests that result in a retry.
  26 RequestHistory = namedtuple(
  27     "RequestHistory", ["method", "url", "error", "status", "redirect_location"]
  28 )
  29
  30
  31 # TODO: In v2 we can remove this sentinel and metaclass with deprecated options.
  32 _Default = object()
  33
  34
  35 class _RetryMeta(type):
  36     @property
  37     def DEFAULT_METHOD_WHITELIST(cls):
  38         warnings.warn(
  39             "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
  40             "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
  41             DeprecationWarning,
  42         )
  43         return cls.DEFAULT_ALLOWED_METHODS
  44
  45     @DEFAULT_METHOD_WHITELIST.setter
  46     def DEFAULT_METHOD_WHITELIST(cls, value):
  47         warnings.warn(
  48             "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
  49             "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
  50             DeprecationWarning,
  51         )
  52         cls.DEFAULT_ALLOWED_METHODS = value
  53
  54     @property
  55     def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls):
  56         warnings.warn(
  57             "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
  58             "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
  59             DeprecationWarning,
  60         )
  61         return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
  62
  63     @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter
  64     def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value):
  65         warnings.warn(
  66             "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
  67             "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
  68             DeprecationWarning,
  69         )
  70         cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value
  71
  72     @property
  73     def BACKOFF_MAX(cls):
  74         warnings.warn(
  75             "Using 'Retry.BACKOFF_MAX' is deprecated and "
  76             "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
  77             DeprecationWarning,
  78         )
  79         return cls.DEFAULT_BACKOFF_MAX
  80
  81     @BACKOFF_MAX.setter
  82     def BACKOFF_MAX(cls, value):
  83         warnings.warn(
  84             "Using 'Retry.BACKOFF_MAX' is deprecated and "
  85             "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
  86             DeprecationWarning,
  87         )
  88         cls.DEFAULT_BACKOFF_MAX = value
  89
  90
  91 @six.add_metaclass(_RetryMeta)
  92 class Retry(object):
  93     """Retry configuration.
  94
  95     Each retry attempt will create a new Retry object with updated values, so
  96     they can be safely reused.
  97
  98     Retries can be defined as a default for a pool::
  99
 100         retries = Retry(connect=5, read=2, redirect=5)
 101         http = PoolManager(retries=retries)
 102         response = http.request('GET', 'http://example.com/')
 103
 104     Or per-request (which overrides the default for the pool)::
 105
 106         response = http.request('GET', 'http://example.com/', retries=Retry(10))
 107
 108     Retries can be disabled by passing ``False``::
 109
 110         response = http.request('GET', 'http://example.com/', retries=False)
 111
 112     Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
 113     retries are disabled, in which case the causing exception will be raised.
 114
 115     :param int total:
 116         Total number of retries to allow. Takes precedence over other counts.
 117
 118         Set to ``None`` to remove this constraint and fall back on other
 119         counts.
 120
 121         Set to ``0`` to fail on the first retry.
 122
 123         Set to ``False`` to disable and imply ``raise_on_redirect=False``.
 124
 125     :param int connect:
 126         How many connection-related errors to retry on.
 127
 128         These are errors raised before the request is sent to the remote server,
 129         which we assume has not triggered the server to process the request.
 130
 131         Set to ``0`` to fail on the first retry of this type.
 132
 133     :param int read:
 134         How many times to retry on read errors.
 135
 136         These errors are raised after the request was sent to the server, so the
 137         request may have side-effects.
 138
 139         Set to ``0`` to fail on the first retry of this type.
 140
 141     :param int redirect:
 142         How many redirects to perform. Limit this to avoid infinite redirect
 143         loops.
 144
 145         A redirect is a HTTP response with a status code 301, 302, 303, 307 or
 146         308.
 147
 148         Set to ``0`` to fail on the first retry of this type.
 149
 150         Set to ``False`` to disable and imply ``raise_on_redirect=False``.
 151
 152     :param int status:
 153         How many times to retry on bad status codes.
 154
 155         These are retries made on responses, where status code matches
 156         ``status_forcelist``.
 157
 158         Set to ``0`` to fail on the first retry of this type.
 159
 160     :param int other:
 161         How many times to retry on other errors.
 162
 163         Other errors are errors that are not connect, read, redirect or status errors.
 164         These errors might be raised after the request was sent to the server, so the
 165         request might have side-effects.
 166
 167         Set to ``0`` to fail on the first retry of this type.
 168
 169         If ``total`` is not set, it's a good idea to set this to 0 to account
 170         for unexpected edge cases and avoid infinite retry loops.
 171
 172     :param iterable allowed_methods:
 173         Set of uppercased HTTP method verbs that we should retry on.
 174
 175         By default, we only retry on methods which are considered to be
 176         idempotent (multiple requests with the same parameters end with the
 177         same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
 178
 179         Set to a ``False`` value to retry on any verb.
 180
 181         .. warning::
 182
 183             Previously this parameter was named ``method_whitelist``, that
 184             usage is deprecated in v1.26.0 and will be removed in v2.0.
 185
 186     :param iterable status_forcelist:
 187         A set of integer HTTP status codes that we should force a retry on.
 188         A retry is initiated if the request method is in ``allowed_methods``
 189         and the response status code is in ``status_forcelist``.
 190
 191         By default, this is disabled with ``None``.
 192
 193     :param float backoff_factor:
 194         A backoff factor to apply between attempts after the second try
 195         (most errors are resolved immediately by a second try without a
 196         delay). urllib3 will sleep for::
 197
 198             {backoff factor} * (2 ** ({number of total retries} - 1))
 199
 200         seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
 201         for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
 202         than :attr:`Retry.DEFAULT_BACKOFF_MAX`.
 203
 204         By default, backoff is disabled (set to 0).
 205
 206     :param bool raise_on_redirect: Whether, if the number of redirects is
 207         exhausted, to raise a MaxRetryError, or to return a response with a
 208         response code in the 3xx range.
 209
 210     :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
 211         whether we should raise an exception, or return a response,
 212         if status falls in ``status_forcelist`` range and retries have
 213         been exhausted.
 214
 215     :param tuple history: The history of the request encountered during
 216         each call to :meth:`~Retry.increment`. The list is in the order
 217         the requests occurred. Each list item is of class :class:`RequestHistory`.
 218
 219     :param bool respect_retry_after_header:
 220         Whether to respect Retry-After header on status codes defined as
 221         :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
 222
 223     :param iterable remove_headers_on_redirect:
 224         Sequence of headers to remove from the request when a response
 225         indicating a redirect is returned before firing off the redirected
 226         request.
 227     """
 228
 229     #: Default methods to be used for ``allowed_methods``
 230     DEFAULT_ALLOWED_METHODS = frozenset(
 231         ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
 232     )
 233
 234     #: Default status codes to be used for ``status_forcelist``
 235     RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
 236
 237     #: Default headers to be used for ``remove_headers_on_redirect``
 238     DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])
 239
 240     #: Maximum backoff time.
 241     DEFAULT_BACKOFF_MAX = 120
 242
 243     def __init__(
 244         self,
 245         total=10,
 246         connect=None,
 247         read=None,
 248         redirect=None,
 249         status=None,
 250         other=None,
 251         allowed_methods=_Default,
 252         status_forcelist=None,
 253         backoff_factor=0,
 254         raise_on_redirect=True,
 255         raise_on_status=True,
 256         history=None,
 257         respect_retry_after_header=True,
 258         remove_headers_on_redirect=_Default,
 259         # TODO: Deprecated, remove in v2.0
 260         method_whitelist=_Default,
 261     ):
 262
 263         if method_whitelist is not _Default:
 264             if allowed_methods is not _Default:
 265                 raise ValueError(
 266                     "Using both 'allowed_methods' and "
 267                     "'method_whitelist' together is not allowed. "
 268                     "Instead only use 'allowed_methods'"
 269                 )
 270             warnings.warn(
 271                 "Using 'method_whitelist' with Retry is deprecated and "
 272                 "will be removed in v2.0. Use 'allowed_methods' instead",
 273                 DeprecationWarning,
 274                 stacklevel=2,
 275             )
 276             allowed_methods = method_whitelist
 277         if allowed_methods is _Default:
 278             allowed_methods = self.DEFAULT_ALLOWED_METHODS
 279         if remove_headers_on_redirect is _Default:
 280             remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
 281
 282         self.total = total
 283         self.connect = connect
 284         self.read = read
 285         self.status = status
 286         self.other = other
 287
 288         if redirect is False or total is False:
 289             redirect = 0
 290             raise_on_redirect = False
 291
 292         self.redirect = redirect
 293         self.status_forcelist = status_forcelist or set()
 294         self.allowed_methods = allowed_methods
 295         self.backoff_factor = backoff_factor
 296         self.raise_on_redirect = raise_on_redirect
 297         self.raise_on_status = raise_on_status
 298         self.history = history or tuple()
 299         self.respect_retry_after_header = respect_retry_after_header
 300         self.remove_headers_on_redirect = frozenset(
 301             [h.lower() for h in remove_headers_on_redirect]
 302         )
 303
 304     def new(self, **kw):
 305         params = dict(
 306             total=self.total,
 307             connect=self.connect,
 308             read=self.read,
 309             redirect=self.redirect,
 310             status=self.status,
 311             other=self.other,
 312             status_forcelist=self.status_forcelist,
 313             backoff_factor=self.backoff_factor,
 314             raise_on_redirect=self.raise_on_redirect,
 315             raise_on_status=self.raise_on_status,
 316             history=self.history,
 317             remove_headers_on_redirect=self.remove_headers_on_redirect,
 318             respect_retry_after_header=self.respect_retry_after_header,
 319         )
 320
 321         # TODO: If already given in **kw we use what's given to us
 322         # If not given we need to figure out what to pass. We decide
 323         # based on whether our class has the 'method_whitelist' property
 324         # and if so we pass the deprecated 'method_whitelist' otherwise
 325         # we use 'allowed_methods'. Remove in v2.0
 326         if "method_whitelist" not in kw and "allowed_methods" not in kw:
 327             if "method_whitelist" in self.__dict__:
 328                 warnings.warn(
 329                     "Using 'method_whitelist' with Retry is deprecated and "
 330                     "will be removed in v2.0. Use 'allowed_methods' instead",
 331                     DeprecationWarning,
 332                 )
 333                 params["method_whitelist"] = self.allowed_methods
 334             else:
 335                 params["allowed_methods"] = self.allowed_methods
 336
 337         params.update(kw)
 338         return type(self)(**params)
 339
 340     @classmethod
 341     def from_int(cls, retries, redirect=True, default=None):
 342         """Backwards-compatibility for the old retries format."""
 343         if retries is None:
 344             retries = default if default is not None else cls.DEFAULT
 345
 346         if isinstance(retries, Retry):
 347             return retries
 348
 349         redirect = bool(redirect) and None
 350         new_retries = cls(retries, redirect=redirect)
 351         log.debug("Converted retries value: %r -> %r", retries, new_retries)
 352         return new_retries
 353
 354     def get_backoff_time(self):
 355         """Formula for computing the current backoff
 356
 357         :rtype: float
 358         """
 359         # We want to consider only the last consecutive errors sequence (Ignore redirects).
 360         consecutive_errors_len = len(
 361             list(
 362                 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
 363             )
 364         )
 365         if consecutive_errors_len <= 1:
 366             return 0
 367
 368         backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
 369         return min(self.DEFAULT_BACKOFF_MAX, backoff_value)
 370
 371     def parse_retry_after(self, retry_after):
 372         # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
 373         if re.match(r"^\s*[0-9]+\s*$", retry_after):
 374             seconds = int(retry_after)
 375         else:
 376             retry_date_tuple = email.utils.parsedate_tz(retry_after)
 377             if retry_date_tuple is None:
 378                 raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
 379             if retry_date_tuple[9] is None:  # Python 2
 380                 # Assume UTC if no timezone was specified
 381                 # On Python2.7, parsedate_tz returns None for a timezone offset
 382                 # instead of 0 if no timezone is given, where mktime_tz treats
 383                 # a None timezone offset as local time.
 384                 retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:]
 385
 386             retry_date = email.utils.mktime_tz(retry_date_tuple)
 387             seconds = retry_date - time.time()
 388
 389         if seconds < 0:
 390             seconds = 0
 391
 392         return seconds
 393
 394     def get_retry_after(self, response):
 395         """Get the value of Retry-After in seconds."""
 396
 397         retry_after = response.headers.get("Retry-After")
 398
 399         if retry_after is None:
 400             return None
 401
 402         return self.parse_retry_after(retry_after)
 403
 404     def sleep_for_retry(self, response=None):
 405         retry_after = self.get_retry_after(response)
 406         if retry_after:
 407             time.sleep(retry_after)
 408             return True
 409
 410         return False
 411
 412     def _sleep_backoff(self):
 413         backoff = self.get_backoff_time()
 414         if backoff <= 0:
 415             return
 416         time.sleep(backoff)
 417
 418     def sleep(self, response=None):
 419         """Sleep between retry attempts.
 420
 421         This method will respect a server's ``Retry-After`` response header
 422         and sleep the duration of the time requested. If that is not present, it
 423         will use an exponential backoff. By default, the backoff factor is 0 and
 424         this method will return immediately.
 425         """
 426
 427         if self.respect_retry_after_header and response:
 428             slept = self.sleep_for_retry(response)
 429             if slept:
 430                 return
 431
 432         self._sleep_backoff()
 433
 434     def _is_connection_error(self, err):
 435         """Errors when we're fairly sure that the server did not receive the
 436         request, so it should be safe to retry.
 437         """
 438         if isinstance(err, ProxyError):
 439             err = err.original_error
 440         return isinstance(err, ConnectTimeoutError)
 441
 442     def _is_read_error(self, err):
 443         """Errors that occur after the request has been started, so we should
 444         assume that the server began processing it.
 445         """
 446         return isinstance(err, (ReadTimeoutError, ProtocolError))
 447
 448     def _is_method_retryable(self, method):
 449         """Checks if a given HTTP method should be retried upon, depending if
 450         it is included in the allowed_methods
 451         """
 452         # TODO: For now favor if the Retry implementation sets its own method_whitelist
 453         # property outside of our constructor to avoid breaking custom implementations.
 454         if "method_whitelist" in self.__dict__:
 455             warnings.warn(
 456                 "Using 'method_whitelist' with Retry is deprecated and "
 457                 "will be removed in v2.0. Use 'allowed_methods' instead",
 458                 DeprecationWarning,
 459             )
 460             allowed_methods = self.method_whitelist
 461         else:
 462             allowed_methods = self.allowed_methods
 463
 464         if allowed_methods and method.upper() not in allowed_methods:
 465             return False
 466         return True
 467
 468     def is_retry(self, method, status_code, has_retry_after=False):
 469         """Is this method/status code retryable? (Based on allowlists and control
 470         variables such as the number of total retries to allow, whether to
 471         respect the Retry-After header, whether this header is present, and
 472         whether the returned status code is on the list of status codes to
 473         be retried upon on the presence of the aforementioned header)
 474         """
 475         if not self._is_method_retryable(method):
 476             return False
 477
 478         if self.status_forcelist and status_code in self.status_forcelist:
 479             return True
 480
 481         return (
 482             self.total
 483             and self.respect_retry_after_header
 484             and has_retry_after
 485             and (status_code in self.RETRY_AFTER_STATUS_CODES)
 486         )
 487
 488     def is_exhausted(self):
 489         """Are we out of retries?"""
 490         retry_counts = (
 491             self.total,
 492             self.connect,
 493             self.read,
 494             self.redirect,
 495             self.status,
 496             self.other,
 497         )
 498         retry_counts = list(filter(None, retry_counts))
 499         if not retry_counts:
 500             return False
 501
 502         return min(retry_counts) < 0
 503
 504     def increment(
 505         self,
 506         method=None,
 507         url=None,
 508         response=None,
 509         error=None,
 510         _pool=None,
 511         _stacktrace=None,
 512     ):
 513         """Return a new Retry object with incremented retry counters.
 514
 515         :param response: A response object, or None, if the server did not
 516             return a response.
 517         :type response: :class:`~urllib3.response.HTTPResponse`
 518         :param Exception error: An error encountered during the request, or
 519             None if the response was received successfully.
 520
 521         :return: A new ``Retry`` object.
 522         """
 523         if self.total is False and error:
 524             # Disabled, indicate to re-raise the error.
 525             raise six.reraise(type(error), error, _stacktrace)
 526
 527         total = self.total
 528         if total is not None:
 529             total -= 1
 530
 531         connect = self.connect
 532         read = self.read
 533         redirect = self.redirect
 534         status_count = self.status
 535         other = self.other
 536         cause = "unknown"
 537         status = None
 538         redirect_location = None
 539
 540         if error and self._is_connection_error(error):
 541             # Connect retry?
 542             if connect is False:
 543                 raise six.reraise(type(error), error, _stacktrace)
 544             elif connect is not None:
 545                 connect -= 1
 546
 547         elif error and self._is_read_error(error):
 548             # Read retry?
 549             if read is False or not self._is_method_retryable(method):
 550                 raise six.reraise(type(error), error, _stacktrace)
 551             elif read is not None:
 552                 read -= 1
 553
 554         elif error:
 555             # Other retry?
 556             if other is not None:
 557                 other -= 1
 558
 559         elif response and response.get_redirect_location():
 560             # Redirect retry?
 561             if redirect is not None:
 562                 redirect -= 1
 563             cause = "too many redirects"
 564             redirect_location = response.get_redirect_location()
 565             status = response.status
 566
 567         else:
 568             # Incrementing because of a server error like a 500 in
 569             # status_forcelist and the given method is in the allowed_methods
 570             cause = ResponseError.GENERIC_ERROR
 571             if response and response.status:
 572                 if status_count is not None:
 573                     status_count -= 1
 574                 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
 575                 status = response.status
 576
 577         history = self.history + (
 578             RequestHistory(method, url, error, status, redirect_location),
 579         )
 580
 581         new_retry = self.new(
 582             total=total,
 583             connect=connect,
 584             read=read,
 585             redirect=redirect,
 586             status=status_count,
 587             other=other,
 588             history=history,
 589         )
 590
 591         if new_retry.is_exhausted():
 592             raise MaxRetryError(_pool, url, error or ResponseError(cause))
 593
 594         log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
 595
 596         return new_retry
 597
 598     def __repr__(self):
 599         return (
 600             "{cls.__name__}(total={self.total}, connect={self.connect}, "
 601             "read={self.read}, redirect={self.redirect}, status={self.status})"
 602         ).format(cls=type(self), self=self)
 603
 604     def __getattr__(self, item):
 605         if item == "method_whitelist":
 606             # TODO: Remove this deprecated alias in v2.0
 607             warnings.warn(
 608                 "Using 'method_whitelist' with Retry is deprecated and "
 609                 "will be removed in v2.0. Use 'allowed_methods' instead",
 610                 DeprecationWarning,
 611             )
 612             return self.allowed_methods
 613         try:
 614             return getattr(super(Retry, self), item)
 615         except AttributeError:
 616             return getattr(Retry, item)
 617
 618
 619 # For backwards compatibility (equivalent to pre-v1.9):
 620 Retry.DEFAULT = Retry(3)