5 This module provides a Session object to manage and persist settings across
6 requests (cookies, auth, proxies).
11 from collections
import OrderedDict
12 from datetime
import timedelta
14 from ._internal
_utils
import to_native_string
15 from .adapters
import HTTPAdapter
16 from .auth
import _basic_auth_str
17 from .compat
import Mapping
, cookielib
, urljoin
, urlparse
18 from .cookies
import (
21 extract_cookies_to_jar
,
24 from .exceptions
import (
30 from .hooks
import default_hooks
, dispatch_hook
32 # formerly defined here, reexposed here for backward compatibility
33 from .models
import ( # noqa: F401
34 DEFAULT_REDIRECT_LIMIT
,
39 from .status_codes
import codes
40 from .structures
import CaseInsensitiveDict
41 from .utils
import ( # noqa: F401
50 should_bypass_proxies
,
54 # Preferred clock, based on which one is more accurate on a given system.
55 if sys
.platform
== "win32":
56 preferred_clock
= time
.perf_counter
58 preferred_clock
= time
.time
61 def merge_setting(request_setting
, session_setting
, dict_class
=OrderedDict
):
62 """Determines appropriate setting for a given request, taking into account
63 the explicit setting on that request, and the setting in the session. If a
64 setting is a dictionary, they will be merged together using `dict_class`
67 if session_setting
is None:
68 return request_setting
70 if request_setting
is None:
71 return session_setting
73 # Bypass if not a dictionary (e.g. verify)
75 isinstance(session_setting
, Mapping
) and isinstance(request_setting
, Mapping
)
77 return request_setting
79 merged_setting
= dict_class(to_key_val_list(session_setting
))
80 merged_setting
.update(to_key_val_list(request_setting
))
82 # Remove keys that are set to None. Extract keys first to avoid altering
83 # the dictionary during iteration.
84 none_keys
= [k
for (k
, v
) in merged_setting
.items() if v
is None]
86 del merged_setting
[key
]
91 def merge_hooks(request_hooks
, session_hooks
, dict_class
=OrderedDict
):
92 """Properly merges both requests and session hooks.
94 This is necessary because when request_hooks == {'response': []}, the
95 merge breaks Session hooks entirely.
97 if session_hooks
is None or session_hooks
.get("response") == []:
100 if request_hooks
is None or request_hooks
.get("response") == []:
103 return merge_setting(request_hooks
, session_hooks
, dict_class
)
106 class SessionRedirectMixin
:
107 def get_redirect_target(self
, resp
):
108 """Receives a Response. Returns a redirect URI or ``None``"""
109 # Due to the nature of how requests processes redirects this method will
110 # be called at least once upon the original response and at least twice
111 # on each subsequent redirect response (if any).
112 # If a custom mixin is used to handle this logic, it may be advantageous
113 # to cache the redirect location onto the response object as a private
116 location
= resp
.headers
["location"]
117 # Currently the underlying http module on py3 decode headers
118 # in latin1, but empirical evidence suggests that latin1 is very
119 # rarely used with non-ASCII characters in HTTP headers.
120 # It is more likely to get UTF8 header rather than latin1.
121 # This causes incorrect handling of UTF8 encoded location headers.
122 # To solve this, we re-encode the location in latin1.
123 location
= location
.encode("latin1")
124 return to_native_string(location
, "utf8")
127 def should_strip_auth(self
, old_url
, new_url
):
128 """Decide whether Authorization header should be removed when redirecting"""
129 old_parsed
= urlparse(old_url
)
130 new_parsed
= urlparse(new_url
)
131 if old_parsed
.hostname
!= new_parsed
.hostname
:
133 # Special case: allow http -> https redirect when using the standard
134 # ports. This isn't specified by RFC 7235, but is kept to avoid
135 # breaking backwards compatibility with older versions of requests
136 # that allowed any redirects on the same host.
138 old_parsed
.scheme
== "http"
139 and old_parsed
.port
in (80, None)
140 and new_parsed
.scheme
== "https"
141 and new_parsed
.port
in (443, None)
145 # Handle default port usage corresponding to scheme.
146 changed_port
= old_parsed
.port
!= new_parsed
.port
147 changed_scheme
= old_parsed
.scheme
!= new_parsed
.scheme
148 default_port
= (DEFAULT_PORTS
.get(old_parsed
.scheme
, None), None)
151 and old_parsed
.port
in default_port
152 and new_parsed
.port
in default_port
156 # Standard case: root URI must match
157 return changed_port
or changed_scheme
159 def resolve_redirects(
168 yield_requests
=False,
171 """Receives a Response. Returns a generator of Responses or Requests."""
173 hist
= [] # keep track of history
175 url
= self
.get_redirect_target(resp
)
176 previous_fragment
= urlparse(req
.url
).fragment
178 prepared_request
= req
.copy()
180 # Update history and keep track of redirects.
181 # resp.history must ignore the original request in this loop
183 resp
.history
= hist
[1:]
186 resp
.content
# Consume socket so it can be released
187 except (ChunkedEncodingError
, ContentDecodingError
, RuntimeError):
188 resp
.raw
.read(decode_content
=False)
190 if len(resp
.history
) >= self
.max_redirects
:
191 raise TooManyRedirects(
192 f
"Exceeded {self.max_redirects} redirects.", response
=resp
195 # Release the connection back into the pool.
198 # Handle redirection without scheme (see: RFC 1808 Section 4)
199 if url
.startswith("//"):
200 parsed_rurl
= urlparse(resp
.url
)
201 url
= ":".join([to_native_string(parsed_rurl
.scheme
), url
])
203 # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2)
204 parsed
= urlparse(url
)
205 if parsed
.fragment
== "" and previous_fragment
:
206 parsed
= parsed
._replace
(fragment
=previous_fragment
)
207 elif parsed
.fragment
:
208 previous_fragment
= parsed
.fragment
209 url
= parsed
.geturl()
211 # Facilitate relative 'location' headers, as allowed by RFC 7231.
212 # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
213 # Compliant with RFC3986, we percent encode the url.
214 if not parsed
.netloc
:
215 url
= urljoin(resp
.url
, requote_uri(url
))
217 url
= requote_uri(url
)
219 prepared_request
.url
= to_native_string(url
)
221 self
.rebuild_method(prepared_request
, resp
)
223 # https://github.com/psf/requests/issues/1084
224 if resp
.status_code
not in (
225 codes
.temporary_redirect
,
226 codes
.permanent_redirect
,
228 # https://github.com/psf/requests/issues/3490
229 purged_headers
= ("Content-Length", "Content-Type", "Transfer-Encoding")
230 for header
in purged_headers
:
231 prepared_request
.headers
.pop(header
, None)
232 prepared_request
.body
= None
234 headers
= prepared_request
.headers
235 headers
.pop("Cookie", None)
237 # Extract any cookies sent on the response to the cookiejar
238 # in the new request. Because we've mutated our copied prepared
239 # request, use the old one that we haven't yet touched.
240 extract_cookies_to_jar(prepared_request
._cookies
, req
, resp
.raw
)
241 merge_cookies(prepared_request
._cookies
, self
.cookies
)
242 prepared_request
.prepare_cookies(prepared_request
._cookies
)
244 # Rebuild auth and proxy information.
245 proxies
= self
.rebuild_proxies(prepared_request
, proxies
)
246 self
.rebuild_auth(prepared_request
, resp
)
248 # A failed tell() sets `_body_position` to `object()`. This non-None
249 # value ensures `rewindable` will be True, allowing us to raise an
250 # UnrewindableBodyError, instead of hanging the connection.
251 rewindable
= prepared_request
._body
_position
is not None and (
252 "Content-Length" in headers
or "Transfer-Encoding" in headers
255 # Attempt to rewind consumed file-like object.
257 rewind_body(prepared_request
)
259 # Override the original request.
260 req
= prepared_request
273 allow_redirects
=False,
277 extract_cookies_to_jar(self
.cookies
, prepared_request
, resp
.raw
)
279 # extract redirect url, if any, for the next loop
280 url
= self
.get_redirect_target(resp
)
283 def rebuild_auth(self
, prepared_request
, response
):
284 """When being redirected we may want to strip authentication from the
285 request to avoid leaking credentials. This method intelligently removes
286 and reapplies authentication where possible to avoid credential loss.
288 headers
= prepared_request
.headers
289 url
= prepared_request
.url
291 if "Authorization" in headers
and self
.should_strip_auth(
292 response
.request
.url
, url
294 # If we get redirected to a new host, we should strip out any
295 # authentication headers.
296 del headers
["Authorization"]
298 # .netrc might have more auth for us on our new host.
299 new_auth
= get_netrc_auth(url
) if self
.trust_env
else None
300 if new_auth
is not None:
301 prepared_request
.prepare_auth(new_auth
)
303 def rebuild_proxies(self
, prepared_request
, proxies
):
304 """This method re-evaluates the proxy configuration by considering the
305 environment variables. If we are redirected to a URL covered by
306 NO_PROXY, we strip the proxy configuration. Otherwise, we set missing
307 proxy keys for this URL (in case they were stripped by a previous
310 This method also replaces the Proxy-Authorization header where
315 headers
= prepared_request
.headers
316 scheme
= urlparse(prepared_request
.url
).scheme
317 new_proxies
= resolve_proxies(prepared_request
, proxies
, self
.trust_env
)
319 if "Proxy-Authorization" in headers
:
320 del headers
["Proxy-Authorization"]
323 username
, password
= get_auth_from_url(new_proxies
[scheme
])
325 username
, password
= None, None
327 # urllib3 handles proxy authorization for us in the standard adapter.
328 # Avoid appending this to TLS tunneled requests where it may be leaked.
329 if not scheme
.startswith('https') and username
and password
:
330 headers
["Proxy-Authorization"] = _basic_auth_str(username
, password
)
334 def rebuild_method(self
, prepared_request
, response
):
335 """When being redirected we may want to change the method of the request
336 based on certain specs or browser behavior.
338 method
= prepared_request
.method
340 # https://tools.ietf.org/html/rfc7231#section-6.4.4
341 if response
.status_code
== codes
.see_other
and method
!= "HEAD":
344 # Do what the browsers do, despite standards...
345 # First, turn 302s into GETs.
346 if response
.status_code
== codes
.found
and method
!= "HEAD":
349 # Second, if a POST is responded to with a 301, turn it into a GET.
350 # This bizarre behaviour is explained in Issue 1704.
351 if response
.status_code
== codes
.moved
and method
== "POST":
354 prepared_request
.method
= method
357 class Session(SessionRedirectMixin
):
358 """A Requests session.
360 Provides cookie persistence, connection-pooling, and configuration.
365 >>> s = requests.Session()
366 >>> s.get('https://httpbin.org/get')
369 Or as a context manager::
371 >>> with requests.Session() as s:
372 ... s.get('https://httpbin.org/get')
393 #: A case-insensitive dictionary of headers to be sent on each
394 #: :class:`Request <Request>` sent from this
395 #: :class:`Session <Session>`.
396 self
.headers
= default_headers()
398 #: Default Authentication tuple or object to attach to
399 #: :class:`Request <Request>`.
402 #: Dictionary mapping protocol or protocol and host to the URL of the proxy
403 #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to
404 #: be used on each :class:`Request <Request>`.
407 #: Event-handling hooks.
408 self
.hooks
= default_hooks()
410 #: Dictionary of querystring data to attach to each
411 #: :class:`Request <Request>`. The dictionary values may be lists for
412 #: representing multivalued query parameters.
415 #: Stream response content default.
418 #: SSL Verification default.
419 #: Defaults to `True`, requiring requests to verify the TLS certificate at the
421 #: If verify is set to `False`, requests will accept any TLS certificate
422 #: presented by the server, and will ignore hostname mismatches and/or
423 #: expired certificates, which will make your application vulnerable to
424 #: man-in-the-middle (MitM) attacks.
425 #: Only set this to `False` for testing.
428 #: SSL client certificate default, if String, path to ssl client
429 #: cert file (.pem). If Tuple, ('cert', 'key') pair.
432 #: Maximum number of redirects allowed. If the request exceeds this
433 #: limit, a :class:`TooManyRedirects` exception is raised.
434 #: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is
436 self
.max_redirects
= DEFAULT_REDIRECT_LIMIT
438 #: Trust environment settings for proxy configuration, default
439 #: authentication and similar.
440 self
.trust_env
= True
442 #: A CookieJar containing all currently outstanding cookies set on this
443 #: session. By default it is a
444 #: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but
445 #: may be any other ``cookielib.CookieJar`` compatible object.
446 self
.cookies
= cookiejar_from_dict({})
448 # Default connection adapters.
449 self
.adapters
= OrderedDict()
450 self
.mount("https://", HTTPAdapter())
451 self
.mount("http://", HTTPAdapter())
456 def __exit__(self
, *args
):
459 def prepare_request(self
, request
):
460 """Constructs a :class:`PreparedRequest <PreparedRequest>` for
461 transmission and returns it. The :class:`PreparedRequest` has settings
462 merged from the :class:`Request <Request>` instance and those of the
465 :param request: :class:`Request` instance to prepare with this
467 :rtype: requests.PreparedRequest
469 cookies
= request
.cookies
or {}
471 # Bootstrap CookieJar.
472 if not isinstance(cookies
, cookielib
.CookieJar
):
473 cookies
= cookiejar_from_dict(cookies
)
475 # Merge with session cookies
476 merged_cookies
= merge_cookies(
477 merge_cookies(RequestsCookieJar(), self
.cookies
), cookies
480 # Set environment's basic authentication if not explicitly set.
482 if self
.trust_env
and not auth
and not self
.auth
:
483 auth
= get_netrc_auth(request
.url
)
485 p
= PreparedRequest()
487 method
=request
.method
.upper(),
492 headers
=merge_setting(
493 request
.headers
, self
.headers
, dict_class
=CaseInsensitiveDict
495 params
=merge_setting(request
.params
, self
.params
),
496 auth
=merge_setting(auth
, self
.auth
),
497 cookies
=merged_cookies
,
498 hooks
=merge_hooks(request
.hooks
, self
.hooks
),
513 allow_redirects
=True,
521 """Constructs a :class:`Request <Request>`, prepares it and sends it.
522 Returns :class:`Response <Response>` object.
524 :param method: method for the new :class:`Request` object.
525 :param url: URL for the new :class:`Request` object.
526 :param params: (optional) Dictionary or bytes to be sent in the query
527 string for the :class:`Request`.
528 :param data: (optional) Dictionary, list of tuples, bytes, or file-like
529 object to send in the body of the :class:`Request`.
530 :param json: (optional) json to send in the body of the
532 :param headers: (optional) Dictionary of HTTP Headers to send with the
534 :param cookies: (optional) Dict or CookieJar object to send with the
536 :param files: (optional) Dictionary of ``'filename': file-like-objects``
537 for multipart encoding upload.
538 :param auth: (optional) Auth tuple or callable to enable
539 Basic/Digest/Custom HTTP Auth.
540 :param timeout: (optional) How long to wait for the server to send
541 data before giving up, as a float, or a :ref:`(connect timeout,
542 read timeout) <timeouts>` tuple.
543 :type timeout: float or tuple
544 :param allow_redirects: (optional) Set to True by default.
545 :type allow_redirects: bool
546 :param proxies: (optional) Dictionary mapping protocol or protocol and
547 hostname to the URL of the proxy.
548 :param stream: (optional) whether to immediately download the response
549 content. Defaults to ``False``.
550 :param verify: (optional) Either a boolean, in which case it controls whether we verify
551 the server's TLS certificate, or a string, in which case it must be a path
552 to a CA bundle to use. Defaults to ``True``. When set to
553 ``False``, requests will accept any TLS certificate presented by
554 the server, and will ignore hostname mismatches and/or expired
555 certificates, which will make your application vulnerable to
556 man-in-the-middle (MitM) attacks. Setting verify to ``False``
557 may be useful during local development or testing.
558 :param cert: (optional) if String, path to ssl client cert file (.pem).
559 If Tuple, ('cert', 'key') pair.
560 :rtype: requests.Response
562 # Create the Request.
564 method
=method
.upper(),
575 prep
= self
.prepare_request(req
)
577 proxies
= proxies
or {}
579 settings
= self
.merge_environment_settings(
580 prep
.url
, proxies
, stream
, verify
, cert
586 "allow_redirects": allow_redirects
,
588 send_kwargs
.update(settings
)
589 resp
= self
.send(prep
, **send_kwargs
)
593 def get(self
, url
, **kwargs
):
594 r
"""Sends a GET request. Returns :class:`Response` object.
596 :param url: URL for the new :class:`Request` object.
597 :param \*\*kwargs: Optional arguments that ``request`` takes.
598 :rtype: requests.Response
601 kwargs
.setdefault("allow_redirects", True)
602 return self
.request("GET", url
, **kwargs
)
604 def options(self
, url
, **kwargs
):
605 r
"""Sends a OPTIONS request. Returns :class:`Response` object.
607 :param url: URL for the new :class:`Request` object.
608 :param \*\*kwargs: Optional arguments that ``request`` takes.
609 :rtype: requests.Response
612 kwargs
.setdefault("allow_redirects", True)
613 return self
.request("OPTIONS", url
, **kwargs
)
615 def head(self
, url
, **kwargs
):
616 r
"""Sends a HEAD request. Returns :class:`Response` object.
618 :param url: URL for the new :class:`Request` object.
619 :param \*\*kwargs: Optional arguments that ``request`` takes.
620 :rtype: requests.Response
623 kwargs
.setdefault("allow_redirects", False)
624 return self
.request("HEAD", url
, **kwargs
)
626 def post(self
, url
, data
=None, json
=None, **kwargs
):
627 r
"""Sends a POST request. Returns :class:`Response` object.
629 :param url: URL for the new :class:`Request` object.
630 :param data: (optional) Dictionary, list of tuples, bytes, or file-like
631 object to send in the body of the :class:`Request`.
632 :param json: (optional) json to send in the body of the :class:`Request`.
633 :param \*\*kwargs: Optional arguments that ``request`` takes.
634 :rtype: requests.Response
637 return self
.request("POST", url
, data
=data
, json
=json
, **kwargs
)
639 def put(self
, url
, data
=None, **kwargs
):
640 r
"""Sends a PUT request. Returns :class:`Response` object.
642 :param url: URL for the new :class:`Request` object.
643 :param data: (optional) Dictionary, list of tuples, bytes, or file-like
644 object to send in the body of the :class:`Request`.
645 :param \*\*kwargs: Optional arguments that ``request`` takes.
646 :rtype: requests.Response
649 return self
.request("PUT", url
, data
=data
, **kwargs
)
651 def patch(self
, url
, data
=None, **kwargs
):
652 r
"""Sends a PATCH request. Returns :class:`Response` object.
654 :param url: URL for the new :class:`Request` object.
655 :param data: (optional) Dictionary, list of tuples, bytes, or file-like
656 object to send in the body of the :class:`Request`.
657 :param \*\*kwargs: Optional arguments that ``request`` takes.
658 :rtype: requests.Response
661 return self
.request("PATCH", url
, data
=data
, **kwargs
)
663 def delete(self
, url
, **kwargs
):
664 r
"""Sends a DELETE request. Returns :class:`Response` object.
666 :param url: URL for the new :class:`Request` object.
667 :param \*\*kwargs: Optional arguments that ``request`` takes.
668 :rtype: requests.Response
671 return self
.request("DELETE", url
, **kwargs
)
673 def send(self
, request
, **kwargs
):
674 """Send a given PreparedRequest.
676 :rtype: requests.Response
678 # Set defaults that the hooks can utilize to ensure they always have
679 # the correct parameters to reproduce the previous request.
680 kwargs
.setdefault("stream", self
.stream
)
681 kwargs
.setdefault("verify", self
.verify
)
682 kwargs
.setdefault("cert", self
.cert
)
683 if "proxies" not in kwargs
:
684 kwargs
["proxies"] = resolve_proxies(request
, self
.proxies
, self
.trust_env
)
686 # It's possible that users might accidentally send a Request object.
687 # Guard against that specific failure case.
688 if isinstance(request
, Request
):
689 raise ValueError("You can only send PreparedRequests.")
691 # Set up variables needed for resolve_redirects and dispatching of hooks
692 allow_redirects
= kwargs
.pop("allow_redirects", True)
693 stream
= kwargs
.get("stream")
694 hooks
= request
.hooks
696 # Get the appropriate adapter to use
697 adapter
= self
.get_adapter(url
=request
.url
)
699 # Start time (approximately) of the request
700 start
= preferred_clock()
703 r
= adapter
.send(request
, **kwargs
)
705 # Total elapsed time of the request (approximately)
706 elapsed
= preferred_clock() - start
707 r
.elapsed
= timedelta(seconds
=elapsed
)
709 # Response manipulation hooks
710 r
= dispatch_hook("response", hooks
, r
, **kwargs
)
715 # If the hooks create history then we want those cookies too
716 for resp
in r
.history
:
717 extract_cookies_to_jar(self
.cookies
, resp
.request
, resp
.raw
)
719 extract_cookies_to_jar(self
.cookies
, request
, r
.raw
)
721 # Resolve redirects if allowed.
723 # Redirect resolving generator.
724 gen
= self
.resolve_redirects(r
, request
, **kwargs
)
725 history
= [resp
for resp
in gen
]
729 # Shuffle things around if there's history.
731 # Insert the first (original) request at the start
733 # Get the last request made
737 # If redirects aren't being followed, store the response on the Request for Response.next().
738 if not allow_redirects
:
741 self
.resolve_redirects(r
, request
, yield_requests
=True, **kwargs
)
743 except StopIteration:
751 def merge_environment_settings(self
, url
, proxies
, stream
, verify
, cert
):
753 Check the environment and merge it with some settings.
757 # Gather clues from the surrounding environment.
759 # Set environment's proxies.
760 no_proxy
= proxies
.get("no_proxy") if proxies
is not None else None
761 env_proxies
= get_environ_proxies(url
, no_proxy
=no_proxy
)
762 for (k
, v
) in env_proxies
.items():
763 proxies
.setdefault(k
, v
)
765 # Look for requests environment configuration
766 # and be compatible with cURL.
767 if verify
is True or verify
is None:
769 os
.environ
.get("REQUESTS_CA_BUNDLE")
770 or os
.environ
.get("CURL_CA_BUNDLE")
774 # Merge all the kwargs.
775 proxies
= merge_setting(proxies
, self
.proxies
)
776 stream
= merge_setting(stream
, self
.stream
)
777 verify
= merge_setting(verify
, self
.verify
)
778 cert
= merge_setting(cert
, self
.cert
)
780 return {"proxies": proxies, "stream": stream, "verify": verify, "cert": cert}
782 def get_adapter(self
, url
):
784 Returns the appropriate connection adapter for the given URL.
786 :rtype: requests.adapters.BaseAdapter
788 for (prefix
, adapter
) in self
.adapters
.items():
790 if url
.lower().startswith(prefix
.lower()):
793 # Nothing matches :-/
794 raise InvalidSchema(f
"No connection adapters were found for {url!r}")
797 """Closes all adapters and as such the session"""
798 for v
in self
.adapters
.values():
801 def mount(self
, prefix
, adapter
):
802 """Registers a connection adapter to a prefix.
804 Adapters are sorted in descending order by prefix length.
806 self
.adapters
[prefix
] = adapter
807 keys_to_move
= [k
for k
in self
.adapters
if len(k
) < len(prefix
)]
809 for key
in keys_to_move
:
810 self
.adapters
[key
] = self
.adapters
.pop(key
)
812 def __getstate__(self
):
813 state
= {attr: getattr(self, attr, None) for attr in self.__attrs__}
816 def __setstate__(self
, state
):
817 for attr
, value
in state
.items():
818 setattr(self
, attr
, value
)
823 Returns a :class:`Session` for context-management.
825 .. deprecated:: 1.0.0
827 This method has been deprecated since version 1.0.0 and is only kept for
828 backwards compatibility. New code should use :class:`~requests.sessions.Session`
829 to create a session. This may be removed at a future date.