9 from ..dependencies
import brotli
, requests
, urllib3
10 from ..utils
import bug_reports_message
, int_or_none
, variadic
13 raise ImportError('requests module is not installed')
16 raise ImportError('urllib3 module is not installed')
18 urllib3_version
= tuple(int_or_none(x
, default
=0) for x
in urllib3
.__version__
.split('.'))
20 if urllib3_version
< (1, 26, 17):
21 raise ImportError('Only urllib3 >= 1.26.17 is supported')
23 if requests
.__build
__ < 0x023100:
24 raise ImportError('Only requests >= 2.31.0 is supported')
26 import requests
.adapters
28 import urllib3
.connection
29 import urllib3
.exceptions
31 from ._helper
import (
33 add_accept_encoding_header
,
35 create_socks_proxy_socket
,
37 make_socks_proxy_opts
,
47 from .exceptions
import (
48 CertificateVerifyError
,
56 from ..socks
import ProxyError
as SocksProxyError
58 SUPPORTED_ENCODINGS
= [
62 if brotli
is not None:
63 SUPPORTED_ENCODINGS
.append('br')
66 Override urllib3's behavior to not convert lower-case percent-encoded characters
67 to upper-case during url normalization process.
69 RFC3986 defines that the lower or upper case percent-encoded hexidecimal characters are equivalent
70 and normalizers should convert them to uppercase for consistency [1].
72 However, some sites may have an incorrect implementation where they provide
73 a percent-encoded url that is then compared case-sensitively.[2]
75 While this is a very rare case, since urllib does not do this normalization step, it
76 is best to avoid it in requests too for compatability reasons.
78 1: https://tools.ietf.org/html/rfc3986#section-2.1
79 2: https://github.com/streamlink/streamlink/pull/4003
83 class Urllib3PercentREOverride
:
84 def __init__(self
, r
: re
.Pattern
):
87 # pass through all other attribute calls to the original re
88 def __getattr__(self
, item
):
89 return self
.re
.__getattribute
__(item
)
91 def subn(self
, repl
, string
, *args
, **kwargs
):
92 return string
, self
.re
.subn(repl
, string
, *args
, **kwargs
)[1]
95 # urllib3 >= 1.25.8 uses subn:
96 # https://github.com/urllib3/urllib3/commit/a2697e7c6b275f05879b60f593c5854a816489f0
97 import urllib3
.util
.url
# noqa: E305
99 if hasattr(urllib3
.util
.url
, 'PERCENT_RE'):
100 urllib3
.util
.url
.PERCENT_RE
= Urllib3PercentREOverride(urllib3
.util
.url
.PERCENT_RE
)
101 elif hasattr(urllib3
.util
.url
, '_PERCENT_RE'): # urllib3 >= 2.0.0
102 urllib3
.util
.url
._PERCENT
_RE
= Urllib3PercentREOverride(urllib3
.util
.url
._PERCENT
_RE
)
104 warnings
.warn('Failed to patch PERCENT_RE in urllib3 (does the attribute exist?)' + bug_reports_message())
107 Workaround for issue in urllib.util.ssl_.py: ssl_wrap_context does not pass
108 server_hostname to SSLContext.wrap_socket if server_hostname is an IP,
109 however this is an issue because we set check_hostname to True in our SSLContext.
111 Monkey-patching IS_SECURETRANSPORT forces ssl_wrap_context to pass server_hostname regardless.
113 This has been fixed in urllib3 2.0+.
114 See: https://github.com/urllib3/urllib3/issues/517
117 if urllib3_version
< (2, 0, 0):
118 with contextlib
.suppress():
119 urllib3
.util
.IS_SECURETRANSPORT
= urllib3
.util
.ssl_
.IS_SECURETRANSPORT
= True
122 # Requests will not automatically handle no_proxy by default
123 # due to buggy no_proxy handling with proxy dict [1].
124 # 1. https://github.com/psf/requests/issues/5000
125 requests
.adapters
.select_proxy
= select_proxy
128 class RequestsResponseAdapter(Response
):
129 def __init__(self
, res
: requests
.models
.Response
):
131 fp
=res
.raw
, headers
=res
.headers
, url
=res
.url
,
132 status
=res
.status_code
, reason
=res
.reason
)
134 self
._requests
_response
= res
136 def read(self
, amt
: int = None):
138 # Interact with urllib3 response directly.
139 return self
.fp
.read(amt
, decode_content
=True)
141 # See urllib3.response.HTTPResponse.read() for exceptions raised on read
142 except urllib3
.exceptions
.SSLError
as e
:
143 raise SSLError(cause
=e
) from e
145 except urllib3
.exceptions
.ProtocolError
as e
:
146 # IncompleteRead is always contained within ProtocolError
147 # See urllib3.response.HTTPResponse._error_catcher()
149 (err
for err
in (e
.__context
__, e
.__cause
__, *variadic(e
.args
))
150 if isinstance(err
, http
.client
.IncompleteRead
)), None)
151 if ir_err
is not None:
152 # `urllib3.exceptions.IncompleteRead` is subclass of `http.client.IncompleteRead`
153 # but uses an `int` for its `partial` property.
154 partial
= ir_err
.partial
if isinstance(ir_err
.partial
, int) else len(ir_err
.partial
)
155 raise IncompleteRead(partial
=partial
, expected
=ir_err
.expected
) from e
156 raise TransportError(cause
=e
) from e
158 except urllib3
.exceptions
.HTTPError
as e
:
159 # catch-all for any other urllib3 response exceptions
160 raise TransportError(cause
=e
) from e
163 class RequestsHTTPAdapter(requests
.adapters
.HTTPAdapter
):
164 def __init__(self
, ssl_context
=None, proxy_ssl_context
=None, source_address
=None, **kwargs
):
167 self
._pm
_args
['ssl_context'] = ssl_context
169 self
._pm
_args
['source_address'] = (source_address
, 0)
170 self
._proxy
_ssl
_context
= proxy_ssl_context
or ssl_context
171 super().__init
__(**kwargs
)
173 def init_poolmanager(self
, *args
, **kwargs
):
174 return super().init_poolmanager(*args
, **kwargs
, **self
._pm
_args
)
176 def proxy_manager_for(self
, proxy
, **proxy_kwargs
):
178 if not proxy
.lower().startswith('socks') and self
._proxy
_ssl
_context
:
179 extra_kwargs
['proxy_ssl_context'] = self
._proxy
_ssl
_context
180 return super().proxy_manager_for(proxy
, **proxy_kwargs
, **self
._pm
_args
, **extra_kwargs
)
182 def cert_verify(*args
, **kwargs
):
183 # lean on SSLContext for cert verification
187 class RequestsSession(requests
.sessions
.Session
):
189 Ensure unified redirect method handling with our urllib redirect handler.
191 def rebuild_method(self
, prepared_request
, response
):
192 new_method
= get_redirect_method(prepared_request
.method
, response
.status_code
)
194 # HACK: requests removes headers/body on redirect unless code was a 307/308.
195 if new_method
== prepared_request
.method
:
196 response
._real
_status
_code
= response
.status_code
197 response
.status_code
= 308
199 prepared_request
.method
= new_method
201 def rebuild_auth(self
, prepared_request
, response
):
202 # HACK: undo status code change from rebuild_method, if applicable.
203 # rebuild_auth runs after requests would remove headers/body based on status code
204 if hasattr(response
, '_real_status_code'):
205 response
.status_code
= response
._real
_status
_code
206 del response
._real
_status
_code
207 return super().rebuild_auth(prepared_request
, response
)
210 class Urllib3LoggingFilter(logging
.Filter
):
212 def filter(self
, record
):
213 # Ignore HTTP request messages since HTTPConnection prints those
214 if record
.msg
== '%s://%s:%s "%s %s %s" %s %s':
219 class Urllib3LoggingHandler(logging
.Handler
):
220 """Redirect urllib3 logs to our logger"""
221 def __init__(self
, logger
, *args
, **kwargs
):
222 super().__init
__(*args
, **kwargs
)
223 self
._logger
= logger
225 def emit(self
, record
):
227 msg
= self
.format(record
)
228 if record
.levelno
>= logging
.ERROR
:
229 self
._logger
.error(msg
)
231 self
._logger
.stdout(msg
)
234 self
.handleError(record
)
238 class RequestsRH(RequestHandler
, InstanceStoreMixin
):
240 """Requests RequestHandler
241 https://github.com/psf/requests
243 _SUPPORTED_URL_SCHEMES
= ('http', 'https')
244 _SUPPORTED_ENCODINGS
= tuple(SUPPORTED_ENCODINGS
)
245 _SUPPORTED_PROXY_SCHEMES
= ('http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h')
246 _SUPPORTED_FEATURES
= (Features
.NO_PROXY
, Features
.ALL_PROXY
)
249 def __init__(self
, *args
, **kwargs
):
250 super().__init
__(*args
, **kwargs
)
252 # Forward urllib3 debug messages to our logger
253 logger
= logging
.getLogger('urllib3')
254 handler
= Urllib3LoggingHandler(logger
=self
._logger
)
255 handler
.setFormatter(logging
.Formatter('requests: %(message)s'))
256 handler
.addFilter(Urllib3LoggingFilter())
257 logger
.addHandler(handler
)
258 # TODO: Use a logger filter to suppress pool reuse warning instead
259 logger
.setLevel(logging
.ERROR
)
262 # Setting this globally is not ideal, but is easier than hacking with urllib3.
263 # It could technically be problematic for scripts embedding yt-dlp.
264 # However, it is unlikely debug traffic is used in that context in a way this will cause problems.
265 urllib3
.connection
.HTTPConnection
.debuglevel
= 1
266 logger
.setLevel(logging
.DEBUG
)
267 # this is expected if we are using --no-check-certificate
268 urllib3
.disable_warnings(urllib3
.exceptions
.InsecureRequestWarning
)
271 self
._clear
_instances
()
273 def _check_extensions(self
, extensions
):
274 super()._check
_extensions
(extensions
)
275 extensions
.pop('cookiejar', None)
276 extensions
.pop('timeout', None)
278 def _create_instance(self
, cookiejar
):
279 session
= RequestsSession()
280 http_adapter
= RequestsHTTPAdapter(
281 ssl_context
=self
._make
_sslcontext
(),
282 source_address
=self
.source_address
,
283 max_retries
=urllib3
.util
.retry
.Retry(False),
285 session
.adapters
.clear()
286 session
.headers
= requests
.models
.CaseInsensitiveDict({'Connection': 'keep-alive'}
)
287 session
.mount('https://', http_adapter
)
288 session
.mount('http://', http_adapter
)
289 session
.cookies
= cookiejar
290 session
.trust_env
= False # no need, we already load proxies from env
293 def _send(self
, request
):
295 headers
= self
._merge
_headers
(request
.headers
)
296 add_accept_encoding_header(headers
, SUPPORTED_ENCODINGS
)
298 max_redirects_exceeded
= False
300 session
= self
._get
_instance
(
301 cookiejar
=request
.extensions
.get('cookiejar') or self
.cookiejar
)
304 requests_res
= session
.request(
305 method
=request
.method
,
309 timeout
=float(request
.extensions
.get('timeout') or self
.timeout
),
310 proxies
=request
.proxies
or self
.proxies
,
311 allow_redirects
=True,
315 except requests
.exceptions
.TooManyRedirects
as e
:
316 max_redirects_exceeded
= True
317 requests_res
= e
.response
319 except requests
.exceptions
.SSLError
as e
:
320 if 'CERTIFICATE_VERIFY_FAILED' in str(e
):
321 raise CertificateVerifyError(cause
=e
) from e
322 raise SSLError(cause
=e
) from e
324 except requests
.exceptions
.ProxyError
as e
:
325 raise ProxyError(cause
=e
) from e
327 except (requests
.exceptions
.ConnectionError
, requests
.exceptions
.Timeout
) as e
:
328 raise TransportError(cause
=e
) from e
330 except urllib3
.exceptions
.HTTPError
as e
:
331 # Catch any urllib3 exceptions that may leak through
332 raise TransportError(cause
=e
) from e
334 except requests
.exceptions
.RequestException
as e
:
335 # Miscellaneous Requests exceptions. May not necessary be network related e.g. InvalidURL
336 raise RequestError(cause
=e
) from e
338 res
= RequestsResponseAdapter(requests_res
)
340 if not 200 <= res
.status
< 300:
341 raise HTTPError(res
, redirect_loop
=max_redirects_exceeded
)
346 @register_preference(RequestsRH
)
347 def requests_preference(rh
, request
):
351 # Use our socks proxy implementation with requests to avoid an extra dependency.
352 class SocksHTTPConnection(urllib3
.connection
.HTTPConnection
):
353 def __init__(self
, _socks_options
, *args
, **kwargs
): # must use _socks_options to pass PoolKey checks
354 self
._proxy
_args
= _socks_options
355 super().__init
__(*args
, **kwargs
)
359 return create_connection(
360 address
=(self
._proxy
_args
['addr'], self
._proxy
_args
['port']),
361 timeout
=self
.timeout
,
362 source_address
=self
.source_address
,
363 _create_socket_func
=functools
.partial(
364 create_socks_proxy_socket
, (self
.host
, self
.port
), self
._proxy
_args
))
365 except (socket
.timeout
, TimeoutError
) as e
:
366 raise urllib3
.exceptions
.ConnectTimeoutError(
367 self
, f
'Connection to {self.host} timed out. (connect timeout={self.timeout})') from e
368 except SocksProxyError
as e
:
369 raise urllib3
.exceptions
.ProxyError(str(e
), e
) from e
370 except (OSError, socket
.error
) as e
:
371 raise urllib3
.exceptions
.NewConnectionError(
372 self
, f
'Failed to establish a new connection: {e}') from e
375 class SocksHTTPSConnection(SocksHTTPConnection
, urllib3
.connection
.HTTPSConnection
):
379 class SocksHTTPConnectionPool(urllib3
.HTTPConnectionPool
):
380 ConnectionCls
= SocksHTTPConnection
383 class SocksHTTPSConnectionPool(urllib3
.HTTPSConnectionPool
):
384 ConnectionCls
= SocksHTTPSConnection
387 class SocksProxyManager(urllib3
.PoolManager
):
389 def __init__(self
, socks_proxy
, username
=None, password
=None, num_pools
=10, headers
=None, **connection_pool_kw
):
390 connection_pool_kw
['_socks_options'] = make_socks_proxy_opts(socks_proxy
)
391 super().__init
__(num_pools
, headers
, **connection_pool_kw
)
392 self
.pool_classes_by_scheme
= {
393 'http': SocksHTTPConnectionPool
,
394 'https': SocksHTTPSConnectionPool
398 requests
.adapters
.SOCKSProxyManager
= SocksProxyManager