venv/lib/python3.11/site-packages/pip/_vendor/urllib3/response.py

   1 from __future__ import absolute_import
   2
   3 import io
   4 import logging
   5 import sys
   6 import warnings
   7 import zlib
   8 from contextlib import contextmanager
   9 from socket import error as SocketError
  10 from socket import timeout as SocketTimeout
  11
  12 brotli = None
  13
  14 from . import util
  15 from ._collections import HTTPHeaderDict
  16 from .connection import BaseSSLError, HTTPException
  17 from .exceptions import (
  18     BodyNotHttplibCompatible,
  19     DecodeError,
  20     HTTPError,
  21     IncompleteRead,
  22     InvalidChunkLength,
  23     InvalidHeader,
  24     ProtocolError,
  25     ReadTimeoutError,
  26     ResponseNotChunked,
  27     SSLError,
  28 )
  29 from .packages import six
  30 from .util.response import is_fp_closed, is_response_to_head
  31
  32 log = logging.getLogger(__name__)
  33
  34
  35 class DeflateDecoder(object):
  36     def __init__(self):
  37         self._first_try = True
  38         self._data = b""
  39         self._obj = zlib.decompressobj()
  40
  41     def __getattr__(self, name):
  42         return getattr(self._obj, name)
  43
  44     def decompress(self, data):
  45         if not data:
  46             return data
  47
  48         if not self._first_try:
  49             return self._obj.decompress(data)
  50
  51         self._data += data
  52         try:
  53             decompressed = self._obj.decompress(data)
  54             if decompressed:
  55                 self._first_try = False
  56                 self._data = None
  57             return decompressed
  58         except zlib.error:
  59             self._first_try = False
  60             self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
  61             try:
  62                 return self.decompress(self._data)
  63             finally:
  64                 self._data = None
  65
  66
  67 class GzipDecoderState(object):
  68
  69     FIRST_MEMBER = 0
  70     OTHER_MEMBERS = 1
  71     SWALLOW_DATA = 2
  72
  73
  74 class GzipDecoder(object):
  75     def __init__(self):
  76         self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
  77         self._state = GzipDecoderState.FIRST_MEMBER
  78
  79     def __getattr__(self, name):
  80         return getattr(self._obj, name)
  81
  82     def decompress(self, data):
  83         ret = bytearray()
  84         if self._state == GzipDecoderState.SWALLOW_DATA or not data:
  85             return bytes(ret)
  86         while True:
  87             try:
  88                 ret += self._obj.decompress(data)
  89             except zlib.error:
  90                 previous_state = self._state
  91                 # Ignore data after the first error
  92                 self._state = GzipDecoderState.SWALLOW_DATA
  93                 if previous_state == GzipDecoderState.OTHER_MEMBERS:
  94                     # Allow trailing garbage acceptable in other gzip clients
  95                     return bytes(ret)
  96                 raise
  97             data = self._obj.unused_data
  98             if not data:
  99                 return bytes(ret)
 100             self._state = GzipDecoderState.OTHER_MEMBERS
 101             self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
 102
 103
 104 if brotli is not None:
 105
 106     class BrotliDecoder(object):
 107         # Supports both 'brotlipy' and 'Brotli' packages
 108         # since they share an import name. The top branches
 109         # are for 'brotlipy' and bottom branches for 'Brotli'
 110         def __init__(self):
 111             self._obj = brotli.Decompressor()
 112             if hasattr(self._obj, "decompress"):
 113                 self.decompress = self._obj.decompress
 114             else:
 115                 self.decompress = self._obj.process
 116
 117         def flush(self):
 118             if hasattr(self._obj, "flush"):
 119                 return self._obj.flush()
 120             return b""
 121
 122
 123 class MultiDecoder(object):
 124     """
 125     From RFC7231:
 126         If one or more encodings have been applied to a representation, the
 127         sender that applied the encodings MUST generate a Content-Encoding
 128         header field that lists the content codings in the order in which
 129         they were applied.
 130     """
 131
 132     def __init__(self, modes):
 133         self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]
 134
 135     def flush(self):
 136         return self._decoders[0].flush()
 137
 138     def decompress(self, data):
 139         for d in reversed(self._decoders):
 140             data = d.decompress(data)
 141         return data
 142
 143
 144 def _get_decoder(mode):
 145     if "," in mode:
 146         return MultiDecoder(mode)
 147
 148     if mode == "gzip":
 149         return GzipDecoder()
 150
 151     if brotli is not None and mode == "br":
 152         return BrotliDecoder()
 153
 154     return DeflateDecoder()
 155
 156
 157 class HTTPResponse(io.IOBase):
 158     """
 159     HTTP Response container.
 160
 161     Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is
 162     loaded and decoded on-demand when the ``data`` property is accessed.  This
 163     class is also compatible with the Python standard library's :mod:`io`
 164     module, and can hence be treated as a readable object in the context of that
 165     framework.
 166
 167     Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:
 168
 169     :param preload_content:
 170         If True, the response's body will be preloaded during construction.
 171
 172     :param decode_content:
 173         If True, will attempt to decode the body based on the
 174         'content-encoding' header.
 175
 176     :param original_response:
 177         When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`
 178         object, it's convenient to include the original for debug purposes. It's
 179         otherwise unused.
 180
 181     :param retries:
 182         The retries contains the last :class:`~urllib3.util.retry.Retry` that
 183         was used during the request.
 184
 185     :param enforce_content_length:
 186         Enforce content length checking. Body returned by server must match
 187         value of Content-Length header, if present. Otherwise, raise error.
 188     """
 189
 190     CONTENT_DECODERS = ["gzip", "deflate"]
 191     if brotli is not None:
 192         CONTENT_DECODERS += ["br"]
 193     REDIRECT_STATUSES = [301, 302, 303, 307, 308]
 194
 195     def __init__(
 196         self,
 197         body="",
 198         headers=None,
 199         status=0,
 200         version=0,
 201         reason=None,
 202         strict=0,
 203         preload_content=True,
 204         decode_content=True,
 205         original_response=None,
 206         pool=None,
 207         connection=None,
 208         msg=None,
 209         retries=None,
 210         enforce_content_length=False,
 211         request_method=None,
 212         request_url=None,
 213         auto_close=True,
 214     ):
 215
 216         if isinstance(headers, HTTPHeaderDict):
 217             self.headers = headers
 218         else:
 219             self.headers = HTTPHeaderDict(headers)
 220         self.status = status
 221         self.version = version
 222         self.reason = reason
 223         self.strict = strict
 224         self.decode_content = decode_content
 225         self.retries = retries
 226         self.enforce_content_length = enforce_content_length
 227         self.auto_close = auto_close
 228
 229         self._decoder = None
 230         self._body = None
 231         self._fp = None
 232         self._original_response = original_response
 233         self._fp_bytes_read = 0
 234         self.msg = msg
 235         self._request_url = request_url
 236
 237         if body and isinstance(body, (six.string_types, bytes)):
 238             self._body = body
 239
 240         self._pool = pool
 241         self._connection = connection
 242
 243         if hasattr(body, "read"):
 244             self._fp = body
 245
 246         # Are we using the chunked-style of transfer encoding?
 247         self.chunked = False
 248         self.chunk_left = None
 249         tr_enc = self.headers.get("transfer-encoding", "").lower()
 250         # Don't incur the penalty of creating a list and then discarding it
 251         encodings = (enc.strip() for enc in tr_enc.split(","))
 252         if "chunked" in encodings:
 253             self.chunked = True
 254
 255         # Determine length of response
 256         self.length_remaining = self._init_length(request_method)
 257
 258         # If requested, preload the body.
 259         if preload_content and not self._body:
 260             self._body = self.read(decode_content=decode_content)
 261
 262     def get_redirect_location(self):
 263         """
 264         Should we redirect and where to?
 265
 266         :returns: Truthy redirect location string if we got a redirect status
 267             code and valid location. ``None`` if redirect status and no
 268             location. ``False`` if not a redirect status code.
 269         """
 270         if self.status in self.REDIRECT_STATUSES:
 271             return self.headers.get("location")
 272
 273         return False
 274
 275     def release_conn(self):
 276         if not self._pool or not self._connection:
 277             return
 278
 279         self._pool._put_conn(self._connection)
 280         self._connection = None
 281
 282     def drain_conn(self):
 283         """
 284         Read and discard any remaining HTTP response data in the response connection.
 285
 286         Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.
 287         """
 288         try:
 289             self.read()
 290         except (HTTPError, SocketError, BaseSSLError, HTTPException):
 291             pass
 292
 293     @property
 294     def data(self):
 295         # For backwards-compat with earlier urllib3 0.4 and earlier.
 296         if self._body:
 297             return self._body
 298
 299         if self._fp:
 300             return self.read(cache_content=True)
 301
 302     @property
 303     def connection(self):
 304         return self._connection
 305
 306     def isclosed(self):
 307         return is_fp_closed(self._fp)
 308
 309     def tell(self):
 310         """
 311         Obtain the number of bytes pulled over the wire so far. May differ from
 312         the amount of content returned by :meth:``urllib3.response.HTTPResponse.read``
 313         if bytes are encoded on the wire (e.g, compressed).
 314         """
 315         return self._fp_bytes_read
 316
 317     def _init_length(self, request_method):
 318         """
 319         Set initial length value for Response content if available.
 320         """
 321         length = self.headers.get("content-length")
 322
 323         if length is not None:
 324             if self.chunked:
 325                 # This Response will fail with an IncompleteRead if it can't be
 326                 # received as chunked. This method falls back to attempt reading
 327                 # the response before raising an exception.
 328                 log.warning(
 329                     "Received response with both Content-Length and "
 330                     "Transfer-Encoding set. This is expressly forbidden "
 331                     "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
 332                     "attempting to process response as Transfer-Encoding: "
 333                     "chunked."
 334                 )
 335                 return None
 336
 337             try:
 338                 # RFC 7230 section 3.3.2 specifies multiple content lengths can
 339                 # be sent in a single Content-Length header
 340                 # (e.g. Content-Length: 42, 42). This line ensures the values
 341                 # are all valid ints and that as long as the `set` length is 1,
 342                 # all values are the same. Otherwise, the header is invalid.
 343                 lengths = set([int(val) for val in length.split(",")])
 344                 if len(lengths) > 1:
 345                     raise InvalidHeader(
 346                         "Content-Length contained multiple "
 347                         "unmatching values (%s)" % length
 348                     )
 349                 length = lengths.pop()
 350             except ValueError:
 351                 length = None
 352             else:
 353                 if length < 0:
 354                     length = None
 355
 356         # Convert status to int for comparison
 357         # In some cases, httplib returns a status of "_UNKNOWN"
 358         try:
 359             status = int(self.status)
 360         except ValueError:
 361             status = 0
 362
 363         # Check for responses that shouldn't include a body
 364         if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
 365             length = 0
 366
 367         return length
 368
 369     def _init_decoder(self):
 370         """
 371         Set-up the _decoder attribute if necessary.
 372         """
 373         # Note: content-encoding value should be case-insensitive, per RFC 7230
 374         # Section 3.2
 375         content_encoding = self.headers.get("content-encoding", "").lower()
 376         if self._decoder is None:
 377             if content_encoding in self.CONTENT_DECODERS:
 378                 self._decoder = _get_decoder(content_encoding)
 379             elif "," in content_encoding:
 380                 encodings = [
 381                     e.strip()
 382                     for e in content_encoding.split(",")
 383                     if e.strip() in self.CONTENT_DECODERS
 384                 ]
 385                 if len(encodings):
 386                     self._decoder = _get_decoder(content_encoding)
 387
 388     DECODER_ERROR_CLASSES = (IOError, zlib.error)
 389     if brotli is not None:
 390         DECODER_ERROR_CLASSES += (brotli.error,)
 391
 392     def _decode(self, data, decode_content, flush_decoder):
 393         """
 394         Decode the data passed in and potentially flush the decoder.
 395         """
 396         if not decode_content:
 397             return data
 398
 399         try:
 400             if self._decoder:
 401                 data = self._decoder.decompress(data)
 402         except self.DECODER_ERROR_CLASSES as e:
 403             content_encoding = self.headers.get("content-encoding", "").lower()
 404             raise DecodeError(
 405                 "Received response with content-encoding: %s, but "
 406                 "failed to decode it." % content_encoding,
 407                 e,
 408             )
 409         if flush_decoder:
 410             data += self._flush_decoder()
 411
 412         return data
 413
 414     def _flush_decoder(self):
 415         """
 416         Flushes the decoder. Should only be called if the decoder is actually
 417         being used.
 418         """
 419         if self._decoder:
 420             buf = self._decoder.decompress(b"")
 421             return buf + self._decoder.flush()
 422
 423         return b""
 424
 425     @contextmanager
 426     def _error_catcher(self):
 427         """
 428         Catch low-level python exceptions, instead re-raising urllib3
 429         variants, so that low-level exceptions are not leaked in the
 430         high-level api.
 431
 432         On exit, release the connection back to the pool.
 433         """
 434         clean_exit = False
 435
 436         try:
 437             try:
 438                 yield
 439
 440             except SocketTimeout:
 441                 # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
 442                 # there is yet no clean way to get at it from this context.
 443                 raise ReadTimeoutError(self._pool, None, "Read timed out.")
 444
 445             except BaseSSLError as e:
 446                 # FIXME: Is there a better way to differentiate between SSLErrors?
 447                 if "read operation timed out" not in str(e):
 448                     # SSL errors related to framing/MAC get wrapped and reraised here
 449                     raise SSLError(e)
 450
 451                 raise ReadTimeoutError(self._pool, None, "Read timed out.")
 452
 453             except (HTTPException, SocketError) as e:
 454                 # This includes IncompleteRead.
 455                 raise ProtocolError("Connection broken: %r" % e, e)
 456
 457             # If no exception is thrown, we should avoid cleaning up
 458             # unnecessarily.
 459             clean_exit = True
 460         finally:
 461             # If we didn't terminate cleanly, we need to throw away our
 462             # connection.
 463             if not clean_exit:
 464                 # The response may not be closed but we're not going to use it
 465                 # anymore so close it now to ensure that the connection is
 466                 # released back to the pool.
 467                 if self._original_response:
 468                     self._original_response.close()
 469
 470                 # Closing the response may not actually be sufficient to close
 471                 # everything, so if we have a hold of the connection close that
 472                 # too.
 473                 if self._connection:
 474                     self._connection.close()
 475
 476             # If we hold the original response but it's closed now, we should
 477             # return the connection back to the pool.
 478             if self._original_response and self._original_response.isclosed():
 479                 self.release_conn()
 480
 481     def _fp_read(self, amt):
 482         """
 483         Read a response with the thought that reading the number of bytes
 484         larger than can fit in a 32-bit int at a time via SSL in some
 485         known cases leads to an overflow error that has to be prevented
 486         if `amt` or `self.length_remaining` indicate that a problem may
 487         happen.
 488
 489         The known cases:
 490           * 3.8 <= CPython < 3.9.7 because of a bug
 491             https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900.
 492           * urllib3 injected with pyOpenSSL-backed SSL-support.
 493           * CPython < 3.10 only when `amt` does not fit 32-bit int.
 494         """
 495         assert self._fp
 496         c_int_max = 2 ** 31 - 1
 497         if (
 498             (
 499                 (amt and amt > c_int_max)
 500                 or (self.length_remaining and self.length_remaining > c_int_max)
 501             )
 502             and not util.IS_SECURETRANSPORT
 503             and (util.IS_PYOPENSSL or sys.version_info < (3, 10))
 504         ):
 505             buffer = io.BytesIO()
 506             # Besides `max_chunk_amt` being a maximum chunk size, it
 507             # affects memory overhead of reading a response by this
 508             # method in CPython.
 509             # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum
 510             # chunk size that does not lead to an overflow error, but
 511             # 256 MiB is a compromise.
 512             max_chunk_amt = 2 ** 28
 513             while amt is None or amt != 0:
 514                 if amt is not None:
 515                     chunk_amt = min(amt, max_chunk_amt)
 516                     amt -= chunk_amt
 517                 else:
 518                     chunk_amt = max_chunk_amt
 519                 data = self._fp.read(chunk_amt)
 520                 if not data:
 521                     break
 522                 buffer.write(data)
 523                 del data  # to reduce peak memory usage by `max_chunk_amt`.
 524             return buffer.getvalue()
 525         else:
 526             # StringIO doesn't like amt=None
 527             return self._fp.read(amt) if amt is not None else self._fp.read()
 528
 529     def read(self, amt=None, decode_content=None, cache_content=False):
 530         """
 531         Similar to :meth:`http.client.HTTPResponse.read`, but with two additional
 532         parameters: ``decode_content`` and ``cache_content``.
 533
 534         :param amt:
 535             How much of the content to read. If specified, caching is skipped
 536             because it doesn't make sense to cache partial content as the full
 537             response.
 538
 539         :param decode_content:
 540             If True, will attempt to decode the body based on the
 541             'content-encoding' header.
 542
 543         :param cache_content:
 544             If True, will save the returned data such that the same result is
 545             returned despite of the state of the underlying file object. This
 546             is useful if you want the ``.data`` property to continue working
 547             after having ``.read()`` the file object. (Overridden if ``amt`` is
 548             set.)
 549         """
 550         self._init_decoder()
 551         if decode_content is None:
 552             decode_content = self.decode_content
 553
 554         if self._fp is None:
 555             return
 556
 557         flush_decoder = False
 558         fp_closed = getattr(self._fp, "closed", False)
 559
 560         with self._error_catcher():
 561             data = self._fp_read(amt) if not fp_closed else b""
 562             if amt is None:
 563                 flush_decoder = True
 564             else:
 565                 cache_content = False
 566                 if (
 567                     amt != 0 and not data
 568                 ):  # Platform-specific: Buggy versions of Python.
 569                     # Close the connection when no data is returned
 570                     #
 571                     # This is redundant to what httplib/http.client _should_
 572                     # already do.  However, versions of python released before
 573                     # December 15, 2012 (http://bugs.python.org/issue16298) do
 574                     # not properly close the connection in all cases. There is
 575                     # no harm in redundantly calling close.
 576                     self._fp.close()
 577                     flush_decoder = True
 578                     if self.enforce_content_length and self.length_remaining not in (
 579                         0,
 580                         None,
 581                     ):
 582                         # This is an edge case that httplib failed to cover due
 583                         # to concerns of backward compatibility. We're
 584                         # addressing it here to make sure IncompleteRead is
 585                         # raised during streaming, so all calls with incorrect
 586                         # Content-Length are caught.
 587                         raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
 588
 589         if data:
 590             self._fp_bytes_read += len(data)
 591             if self.length_remaining is not None:
 592                 self.length_remaining -= len(data)
 593
 594             data = self._decode(data, decode_content, flush_decoder)
 595
 596             if cache_content:
 597                 self._body = data
 598
 599         return data
 600
 601     def stream(self, amt=2 ** 16, decode_content=None):
 602         """
 603         A generator wrapper for the read() method. A call will block until
 604         ``amt`` bytes have been read from the connection or until the
 605         connection is closed.
 606
 607         :param amt:
 608             How much of the content to read. The generator will return up to
 609             much data per iteration, but may return less. This is particularly
 610             likely when using compressed data. However, the empty string will
 611             never be returned.
 612
 613         :param decode_content:
 614             If True, will attempt to decode the body based on the
 615             'content-encoding' header.
 616         """
 617         if self.chunked and self.supports_chunked_reads():
 618             for line in self.read_chunked(amt, decode_content=decode_content):
 619                 yield line
 620         else:
 621             while not is_fp_closed(self._fp):
 622                 data = self.read(amt=amt, decode_content=decode_content)
 623
 624                 if data:
 625                     yield data
 626
 627     @classmethod
 628     def from_httplib(ResponseCls, r, **response_kw):
 629         """
 630         Given an :class:`http.client.HTTPResponse` instance ``r``, return a
 631         corresponding :class:`urllib3.response.HTTPResponse` object.
 632
 633         Remaining parameters are passed to the HTTPResponse constructor, along
 634         with ``original_response=r``.
 635         """
 636         headers = r.msg
 637
 638         if not isinstance(headers, HTTPHeaderDict):
 639             if six.PY2:
 640                 # Python 2.7
 641                 headers = HTTPHeaderDict.from_httplib(headers)
 642             else:
 643                 headers = HTTPHeaderDict(headers.items())
 644
 645         # HTTPResponse objects in Python 3 don't have a .strict attribute
 646         strict = getattr(r, "strict", 0)
 647         resp = ResponseCls(
 648             body=r,
 649             headers=headers,
 650             status=r.status,
 651             version=r.version,
 652             reason=r.reason,
 653             strict=strict,
 654             original_response=r,
 655             **response_kw
 656         )
 657         return resp
 658
 659     # Backwards-compatibility methods for http.client.HTTPResponse
 660     def getheaders(self):
 661         warnings.warn(
 662             "HTTPResponse.getheaders() is deprecated and will be removed "
 663             "in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.",
 664             category=DeprecationWarning,
 665             stacklevel=2,
 666         )
 667         return self.headers
 668
 669     def getheader(self, name, default=None):
 670         warnings.warn(
 671             "HTTPResponse.getheader() is deprecated and will be removed "
 672             "in urllib3 v2.1.0. Instead use HTTPResponse.headers.get(name, default).",
 673             category=DeprecationWarning,
 674             stacklevel=2,
 675         )
 676         return self.headers.get(name, default)
 677
 678     # Backwards compatibility for http.cookiejar
 679     def info(self):
 680         return self.headers
 681
 682     # Overrides from io.IOBase
 683     def close(self):
 684         if not self.closed:
 685             self._fp.close()
 686
 687         if self._connection:
 688             self._connection.close()
 689
 690         if not self.auto_close:
 691             io.IOBase.close(self)
 692
 693     @property
 694     def closed(self):
 695         if not self.auto_close:
 696             return io.IOBase.closed.__get__(self)
 697         elif self._fp is None:
 698             return True
 699         elif hasattr(self._fp, "isclosed"):
 700             return self._fp.isclosed()
 701         elif hasattr(self._fp, "closed"):
 702             return self._fp.closed
 703         else:
 704             return True
 705
 706     def fileno(self):
 707         if self._fp is None:
 708             raise IOError("HTTPResponse has no file to get a fileno from")
 709         elif hasattr(self._fp, "fileno"):
 710             return self._fp.fileno()
 711         else:
 712             raise IOError(
 713                 "The file-like object this HTTPResponse is wrapped "
 714                 "around has no file descriptor"
 715             )
 716
 717     def flush(self):
 718         if (
 719             self._fp is not None
 720             and hasattr(self._fp, "flush")
 721             and not getattr(self._fp, "closed", False)
 722         ):
 723             return self._fp.flush()
 724
 725     def readable(self):
 726         # This method is required for `io` module compatibility.
 727         return True
 728
 729     def readinto(self, b):
 730         # This method is required for `io` module compatibility.
 731         temp = self.read(len(b))
 732         if len(temp) == 0:
 733             return 0
 734         else:
 735             b[: len(temp)] = temp
 736             return len(temp)
 737
 738     def supports_chunked_reads(self):
 739         """
 740         Checks if the underlying file-like object looks like a
 741         :class:`http.client.HTTPResponse` object. We do this by testing for
 742         the fp attribute. If it is present we assume it returns raw chunks as
 743         processed by read_chunked().
 744         """
 745         return hasattr(self._fp, "fp")
 746
 747     def _update_chunk_length(self):
 748         # First, we'll figure out length of a chunk and then
 749         # we'll try to read it from socket.
 750         if self.chunk_left is not None:
 751             return
 752         line = self._fp.fp.readline()
 753         line = line.split(b";", 1)[0]
 754         try:
 755             self.chunk_left = int(line, 16)
 756         except ValueError:
 757             # Invalid chunked protocol response, abort.
 758             self.close()
 759             raise InvalidChunkLength(self, line)
 760
 761     def _handle_chunk(self, amt):
 762         returned_chunk = None
 763         if amt is None:
 764             chunk = self._fp._safe_read(self.chunk_left)
 765             returned_chunk = chunk
 766             self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
 767             self.chunk_left = None
 768         elif amt < self.chunk_left:
 769             value = self._fp._safe_read(amt)
 770             self.chunk_left = self.chunk_left - amt
 771             returned_chunk = value
 772         elif amt == self.chunk_left:
 773             value = self._fp._safe_read(amt)
 774             self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
 775             self.chunk_left = None
 776             returned_chunk = value
 777         else:  # amt > self.chunk_left
 778             returned_chunk = self._fp._safe_read(self.chunk_left)
 779             self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
 780             self.chunk_left = None
 781         return returned_chunk
 782
 783     def read_chunked(self, amt=None, decode_content=None):
 784         """
 785         Similar to :meth:`HTTPResponse.read`, but with an additional
 786         parameter: ``decode_content``.
 787
 788         :param amt:
 789             How much of the content to read. If specified, caching is skipped
 790             because it doesn't make sense to cache partial content as the full
 791             response.
 792
 793         :param decode_content:
 794             If True, will attempt to decode the body based on the
 795             'content-encoding' header.
 796         """
 797         self._init_decoder()
 798         # FIXME: Rewrite this method and make it a class with a better structured logic.
 799         if not self.chunked:
 800             raise ResponseNotChunked(
 801                 "Response is not chunked. "
 802                 "Header 'transfer-encoding: chunked' is missing."
 803             )
 804         if not self.supports_chunked_reads():
 805             raise BodyNotHttplibCompatible(
 806                 "Body should be http.client.HTTPResponse like. "
 807                 "It should have have an fp attribute which returns raw chunks."
 808             )
 809
 810         with self._error_catcher():
 811             # Don't bother reading the body of a HEAD request.
 812             if self._original_response and is_response_to_head(self._original_response):
 813                 self._original_response.close()
 814                 return
 815
 816             # If a response is already read and closed
 817             # then return immediately.
 818             if self._fp.fp is None:
 819                 return
 820
 821             while True:
 822                 self._update_chunk_length()
 823                 if self.chunk_left == 0:
 824                     break
 825                 chunk = self._handle_chunk(amt)
 826                 decoded = self._decode(
 827                     chunk, decode_content=decode_content, flush_decoder=False
 828                 )
 829                 if decoded:
 830                     yield decoded
 831
 832             if decode_content:
 833                 # On CPython and PyPy, we should never need to flush the
 834                 # decoder. However, on Jython we *might* need to, so
 835                 # lets defensively do it anyway.
 836                 decoded = self._flush_decoder()
 837                 if decoded:  # Platform-specific: Jython.
 838                     yield decoded
 839
 840             # Chunk content ends with \r\n: discard it.
 841             while True:
 842                 line = self._fp.fp.readline()
 843                 if not line:
 844                     # Some sites may not end with '\r\n'.
 845                     break
 846                 if line == b"\r\n":
 847                     break
 848
 849             # We read everything; close the "file".
 850             if self._original_response:
 851                 self._original_response.close()
 852
 853     def geturl(self):
 854         """
 855         Returns the URL that was the source of this response.
 856         If the request that generated this response redirected, this method
 857         will return the final redirect location.
 858         """
 859         if self.retries is not None and len(self.retries.history):
 860             return self.retries.history[-1].redirect_location
 861         else:
 862             return self._request_url
 863
 864     def __iter__(self):
 865         buffer = []
 866         for chunk in self.stream(decode_content=True):
 867             if b"\n" in chunk:
 868                 chunk = chunk.split(b"\n")
 869                 yield b"".join(buffer) + chunk[0] + b"\n"
 870                 for x in chunk[1:-1]:
 871                     yield x + b"\n"
 872                 if chunk[-1]:
 873                     buffer = [chunk[-1]]
 874                 else:
 875                     buffer = []
 876             else:
 877                 buffer.append(chunk)
 878         if buffer:
 879             yield b"".join(buffer)