]>
Commit | Line | Data |
---|---|---|
8a8b5452 | 1 | import contextlib |
2 | import functools | |
3 | import http.client | |
4 | import logging | |
5 | import re | |
6 | import socket | |
7 | import warnings | |
8 | ||
9 | from ..dependencies import brotli, requests, urllib3 | |
10 | from ..utils import bug_reports_message, int_or_none, variadic | |
11 | ||
12 | if requests is None: | |
13 | raise ImportError('requests module is not installed') | |
14 | ||
15 | if urllib3 is None: | |
16 | raise ImportError('urllib3 module is not installed') | |
17 | ||
18 | urllib3_version = tuple(int_or_none(x, default=0) for x in urllib3.__version__.split('.')) | |
19 | ||
20 | if urllib3_version < (1, 26, 17): | |
21 | raise ImportError('Only urllib3 >= 1.26.17 is supported') | |
22 | ||
23 | if requests.__build__ < 0x023100: | |
24 | raise ImportError('Only requests >= 2.31.0 is supported') | |
25 | ||
26 | import requests.adapters | |
27 | import requests.utils | |
28 | import urllib3.connection | |
29 | import urllib3.exceptions | |
30 | ||
31 | from ._helper import ( | |
32 | InstanceStoreMixin, | |
33 | add_accept_encoding_header, | |
34 | create_connection, | |
35 | create_socks_proxy_socket, | |
36 | get_redirect_method, | |
37 | make_socks_proxy_opts, | |
38 | select_proxy, | |
39 | ) | |
40 | from .common import ( | |
41 | Features, | |
42 | RequestHandler, | |
43 | Response, | |
44 | register_preference, | |
45 | register_rh, | |
46 | ) | |
47 | from .exceptions import ( | |
48 | CertificateVerifyError, | |
49 | HTTPError, | |
50 | IncompleteRead, | |
51 | ProxyError, | |
52 | RequestError, | |
53 | SSLError, | |
54 | TransportError, | |
55 | ) | |
56 | from ..socks import ProxyError as SocksProxyError | |
57 | ||
58 | SUPPORTED_ENCODINGS = [ | |
59 | 'gzip', 'deflate' | |
60 | ] | |
61 | ||
62 | if brotli is not None: | |
63 | SUPPORTED_ENCODINGS.append('br') | |
64 | ||
65 | """ | |
66 | Override urllib3's behavior to not convert lower-case percent-encoded characters | |
67 | to upper-case during url normalization process. | |
68 | ||
69 | RFC3986 defines that the lower or upper case percent-encoded hexidecimal characters are equivalent | |
70 | and normalizers should convert them to uppercase for consistency [1]. | |
71 | ||
72 | However, some sites may have an incorrect implementation where they provide | |
73 | a percent-encoded url that is then compared case-sensitively.[2] | |
74 | ||
75 | While this is a very rare case, since urllib does not do this normalization step, it | |
76 | is best to avoid it in requests too for compatability reasons. | |
77 | ||
78 | 1: https://tools.ietf.org/html/rfc3986#section-2.1 | |
79 | 2: https://github.com/streamlink/streamlink/pull/4003 | |
80 | """ | |
81 | ||
82 | ||
83 | class Urllib3PercentREOverride: | |
84 | def __init__(self, r: re.Pattern): | |
85 | self.re = r | |
86 | ||
87 | # pass through all other attribute calls to the original re | |
88 | def __getattr__(self, item): | |
89 | return self.re.__getattribute__(item) | |
90 | ||
91 | def subn(self, repl, string, *args, **kwargs): | |
92 | return string, self.re.subn(repl, string, *args, **kwargs)[1] | |
93 | ||
94 | ||
95 | # urllib3 >= 1.25.8 uses subn: | |
96 | # https://github.com/urllib3/urllib3/commit/a2697e7c6b275f05879b60f593c5854a816489f0 | |
97 | import urllib3.util.url # noqa: E305 | |
98 | ||
99 | if hasattr(urllib3.util.url, 'PERCENT_RE'): | |
100 | urllib3.util.url.PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url.PERCENT_RE) | |
101 | elif hasattr(urllib3.util.url, '_PERCENT_RE'): # urllib3 >= 2.0.0 | |
102 | urllib3.util.url._PERCENT_RE = Urllib3PercentREOverride(urllib3.util.url._PERCENT_RE) | |
103 | else: | |
104 | warnings.warn('Failed to patch PERCENT_RE in urllib3 (does the attribute exist?)' + bug_reports_message()) | |
105 | ||
106 | """ | |
107 | Workaround for issue in urllib.util.ssl_.py: ssl_wrap_context does not pass | |
108 | server_hostname to SSLContext.wrap_socket if server_hostname is an IP, | |
109 | however this is an issue because we set check_hostname to True in our SSLContext. | |
110 | ||
111 | Monkey-patching IS_SECURETRANSPORT forces ssl_wrap_context to pass server_hostname regardless. | |
112 | ||
113 | This has been fixed in urllib3 2.0+. | |
114 | See: https://github.com/urllib3/urllib3/issues/517 | |
115 | """ | |
116 | ||
117 | if urllib3_version < (2, 0, 0): | |
118 | with contextlib.suppress(): | |
119 | urllib3.util.IS_SECURETRANSPORT = urllib3.util.ssl_.IS_SECURETRANSPORT = True | |
120 | ||
121 | ||
122 | # Requests will not automatically handle no_proxy by default | |
123 | # due to buggy no_proxy handling with proxy dict [1]. | |
124 | # 1. https://github.com/psf/requests/issues/5000 | |
125 | requests.adapters.select_proxy = select_proxy | |
126 | ||
127 | ||
128 | class RequestsResponseAdapter(Response): | |
129 | def __init__(self, res: requests.models.Response): | |
130 | super().__init__( | |
131 | fp=res.raw, headers=res.headers, url=res.url, | |
132 | status=res.status_code, reason=res.reason) | |
133 | ||
134 | self._requests_response = res | |
135 | ||
136 | def read(self, amt: int = None): | |
137 | try: | |
138 | # Interact with urllib3 response directly. | |
139 | return self.fp.read(amt, decode_content=True) | |
140 | ||
141 | # See urllib3.response.HTTPResponse.read() for exceptions raised on read | |
142 | except urllib3.exceptions.SSLError as e: | |
143 | raise SSLError(cause=e) from e | |
144 | ||
8a8b5452 | 145 | except urllib3.exceptions.ProtocolError as e: |
4e38e2ae | 146 | # IncompleteRead is always contained within ProtocolError |
8a8b5452 | 147 | # See urllib3.response.HTTPResponse._error_catcher() |
148 | ir_err = next( | |
149 | (err for err in (e.__context__, e.__cause__, *variadic(e.args)) | |
150 | if isinstance(err, http.client.IncompleteRead)), None) | |
151 | if ir_err is not None: | |
4e38e2ae SS |
152 | # `urllib3.exceptions.IncompleteRead` is subclass of `http.client.IncompleteRead` |
153 | # but uses an `int` for its `partial` property. | |
154 | partial = ir_err.partial if isinstance(ir_err.partial, int) else len(ir_err.partial) | |
155 | raise IncompleteRead(partial=partial, expected=ir_err.expected) from e | |
8a8b5452 | 156 | raise TransportError(cause=e) from e |
157 | ||
158 | except urllib3.exceptions.HTTPError as e: | |
159 | # catch-all for any other urllib3 response exceptions | |
160 | raise TransportError(cause=e) from e | |
161 | ||
162 | ||
163 | class RequestsHTTPAdapter(requests.adapters.HTTPAdapter): | |
164 | def __init__(self, ssl_context=None, proxy_ssl_context=None, source_address=None, **kwargs): | |
165 | self._pm_args = {} | |
166 | if ssl_context: | |
167 | self._pm_args['ssl_context'] = ssl_context | |
168 | if source_address: | |
169 | self._pm_args['source_address'] = (source_address, 0) | |
170 | self._proxy_ssl_context = proxy_ssl_context or ssl_context | |
171 | super().__init__(**kwargs) | |
172 | ||
173 | def init_poolmanager(self, *args, **kwargs): | |
174 | return super().init_poolmanager(*args, **kwargs, **self._pm_args) | |
175 | ||
176 | def proxy_manager_for(self, proxy, **proxy_kwargs): | |
177 | extra_kwargs = {} | |
178 | if not proxy.lower().startswith('socks') and self._proxy_ssl_context: | |
179 | extra_kwargs['proxy_ssl_context'] = self._proxy_ssl_context | |
180 | return super().proxy_manager_for(proxy, **proxy_kwargs, **self._pm_args, **extra_kwargs) | |
181 | ||
182 | def cert_verify(*args, **kwargs): | |
183 | # lean on SSLContext for cert verification | |
184 | pass | |
185 | ||
186 | ||
187 | class RequestsSession(requests.sessions.Session): | |
188 | """ | |
189 | Ensure unified redirect method handling with our urllib redirect handler. | |
190 | """ | |
f9fb3ce8 | 191 | |
8a8b5452 | 192 | def rebuild_method(self, prepared_request, response): |
193 | new_method = get_redirect_method(prepared_request.method, response.status_code) | |
194 | ||
195 | # HACK: requests removes headers/body on redirect unless code was a 307/308. | |
196 | if new_method == prepared_request.method: | |
197 | response._real_status_code = response.status_code | |
198 | response.status_code = 308 | |
199 | ||
200 | prepared_request.method = new_method | |
201 | ||
202 | def rebuild_auth(self, prepared_request, response): | |
203 | # HACK: undo status code change from rebuild_method, if applicable. | |
204 | # rebuild_auth runs after requests would remove headers/body based on status code | |
205 | if hasattr(response, '_real_status_code'): | |
206 | response.status_code = response._real_status_code | |
207 | del response._real_status_code | |
208 | return super().rebuild_auth(prepared_request, response) | |
209 | ||
210 | ||
211 | class Urllib3LoggingFilter(logging.Filter): | |
212 | ||
213 | def filter(self, record): | |
214 | # Ignore HTTP request messages since HTTPConnection prints those | |
215 | if record.msg == '%s://%s:%s "%s %s %s" %s %s': | |
216 | return False | |
217 | return True | |
218 | ||
219 | ||
220 | class Urllib3LoggingHandler(logging.Handler): | |
221 | """Redirect urllib3 logs to our logger""" | |
f9fb3ce8 | 222 | |
8a8b5452 | 223 | def __init__(self, logger, *args, **kwargs): |
224 | super().__init__(*args, **kwargs) | |
225 | self._logger = logger | |
226 | ||
227 | def emit(self, record): | |
228 | try: | |
229 | msg = self.format(record) | |
230 | if record.levelno >= logging.ERROR: | |
231 | self._logger.error(msg) | |
232 | else: | |
233 | self._logger.stdout(msg) | |
234 | ||
235 | except Exception: | |
236 | self.handleError(record) | |
237 | ||
238 | ||
239 | @register_rh | |
240 | class RequestsRH(RequestHandler, InstanceStoreMixin): | |
241 | ||
242 | """Requests RequestHandler | |
243 | https://github.com/psf/requests | |
244 | """ | |
245 | _SUPPORTED_URL_SCHEMES = ('http', 'https') | |
246 | _SUPPORTED_ENCODINGS = tuple(SUPPORTED_ENCODINGS) | |
247 | _SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h') | |
248 | _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY) | |
249 | RH_NAME = 'requests' | |
250 | ||
251 | def __init__(self, *args, **kwargs): | |
252 | super().__init__(*args, **kwargs) | |
253 | ||
254 | # Forward urllib3 debug messages to our logger | |
255 | logger = logging.getLogger('urllib3') | |
256 | handler = Urllib3LoggingHandler(logger=self._logger) | |
257 | handler.setFormatter(logging.Formatter('requests: %(message)s')) | |
258 | handler.addFilter(Urllib3LoggingFilter()) | |
259 | logger.addHandler(handler) | |
b012271d SS |
260 | # TODO: Use a logger filter to suppress pool reuse warning instead |
261 | logger.setLevel(logging.ERROR) | |
8a8b5452 | 262 | |
263 | if self.verbose: | |
264 | # Setting this globally is not ideal, but is easier than hacking with urllib3. | |
265 | # It could technically be problematic for scripts embedding yt-dlp. | |
266 | # However, it is unlikely debug traffic is used in that context in a way this will cause problems. | |
267 | urllib3.connection.HTTPConnection.debuglevel = 1 | |
268 | logger.setLevel(logging.DEBUG) | |
269 | # this is expected if we are using --no-check-certificate | |
270 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) | |
271 | ||
272 | def close(self): | |
273 | self._clear_instances() | |
274 | ||
275 | def _check_extensions(self, extensions): | |
276 | super()._check_extensions(extensions) | |
277 | extensions.pop('cookiejar', None) | |
278 | extensions.pop('timeout', None) | |
279 | ||
280 | def _create_instance(self, cookiejar): | |
281 | session = RequestsSession() | |
282 | http_adapter = RequestsHTTPAdapter( | |
283 | ssl_context=self._make_sslcontext(), | |
284 | source_address=self.source_address, | |
285 | max_retries=urllib3.util.retry.Retry(False), | |
286 | ) | |
287 | session.adapters.clear() | |
288 | session.headers = requests.models.CaseInsensitiveDict({'Connection': 'keep-alive'}) | |
289 | session.mount('https://', http_adapter) | |
290 | session.mount('http://', http_adapter) | |
291 | session.cookies = cookiejar | |
292 | session.trust_env = False # no need, we already load proxies from env | |
293 | return session | |
294 | ||
295 | def _send(self, request): | |
296 | ||
297 | headers = self._merge_headers(request.headers) | |
298 | add_accept_encoding_header(headers, SUPPORTED_ENCODINGS) | |
299 | ||
300 | max_redirects_exceeded = False | |
301 | ||
302 | session = self._get_instance( | |
303 | cookiejar=request.extensions.get('cookiejar') or self.cookiejar) | |
304 | ||
305 | try: | |
306 | requests_res = session.request( | |
307 | method=request.method, | |
308 | url=request.url, | |
309 | data=request.data, | |
310 | headers=headers, | |
311 | timeout=float(request.extensions.get('timeout') or self.timeout), | |
312 | proxies=request.proxies or self.proxies, | |
313 | allow_redirects=True, | |
314 | stream=True | |
315 | ) | |
316 | ||
317 | except requests.exceptions.TooManyRedirects as e: | |
318 | max_redirects_exceeded = True | |
319 | requests_res = e.response | |
320 | ||
321 | except requests.exceptions.SSLError as e: | |
322 | if 'CERTIFICATE_VERIFY_FAILED' in str(e): | |
323 | raise CertificateVerifyError(cause=e) from e | |
324 | raise SSLError(cause=e) from e | |
325 | ||
326 | except requests.exceptions.ProxyError as e: | |
327 | raise ProxyError(cause=e) from e | |
328 | ||
329 | except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: | |
330 | raise TransportError(cause=e) from e | |
331 | ||
332 | except urllib3.exceptions.HTTPError as e: | |
333 | # Catch any urllib3 exceptions that may leak through | |
334 | raise TransportError(cause=e) from e | |
335 | ||
336 | except requests.exceptions.RequestException as e: | |
337 | # Miscellaneous Requests exceptions. May not necessary be network related e.g. InvalidURL | |
338 | raise RequestError(cause=e) from e | |
339 | ||
340 | res = RequestsResponseAdapter(requests_res) | |
341 | ||
342 | if not 200 <= res.status < 300: | |
343 | raise HTTPError(res, redirect_loop=max_redirects_exceeded) | |
344 | ||
345 | return res | |
346 | ||
347 | ||
348 | @register_preference(RequestsRH) | |
349 | def requests_preference(rh, request): | |
350 | return 100 | |
351 | ||
352 | ||
353 | # Use our socks proxy implementation with requests to avoid an extra dependency. | |
354 | class SocksHTTPConnection(urllib3.connection.HTTPConnection): | |
355 | def __init__(self, _socks_options, *args, **kwargs): # must use _socks_options to pass PoolKey checks | |
356 | self._proxy_args = _socks_options | |
357 | super().__init__(*args, **kwargs) | |
358 | ||
359 | def _new_conn(self): | |
360 | try: | |
361 | return create_connection( | |
362 | address=(self._proxy_args['addr'], self._proxy_args['port']), | |
363 | timeout=self.timeout, | |
364 | source_address=self.source_address, | |
365 | _create_socket_func=functools.partial( | |
366 | create_socks_proxy_socket, (self.host, self.port), self._proxy_args)) | |
367 | except (socket.timeout, TimeoutError) as e: | |
368 | raise urllib3.exceptions.ConnectTimeoutError( | |
369 | self, f'Connection to {self.host} timed out. (connect timeout={self.timeout})') from e | |
370 | except SocksProxyError as e: | |
371 | raise urllib3.exceptions.ProxyError(str(e), e) from e | |
f9fb3ce8 | 372 | except OSError as e: |
8a8b5452 | 373 | raise urllib3.exceptions.NewConnectionError( |
374 | self, f'Failed to establish a new connection: {e}') from e | |
375 | ||
376 | ||
377 | class SocksHTTPSConnection(SocksHTTPConnection, urllib3.connection.HTTPSConnection): | |
378 | pass | |
379 | ||
380 | ||
381 | class SocksHTTPConnectionPool(urllib3.HTTPConnectionPool): | |
382 | ConnectionCls = SocksHTTPConnection | |
383 | ||
384 | ||
385 | class SocksHTTPSConnectionPool(urllib3.HTTPSConnectionPool): | |
386 | ConnectionCls = SocksHTTPSConnection | |
387 | ||
388 | ||
389 | class SocksProxyManager(urllib3.PoolManager): | |
390 | ||
391 | def __init__(self, socks_proxy, username=None, password=None, num_pools=10, headers=None, **connection_pool_kw): | |
392 | connection_pool_kw['_socks_options'] = make_socks_proxy_opts(socks_proxy) | |
393 | super().__init__(num_pools, headers, **connection_pool_kw) | |
394 | self.pool_classes_by_scheme = { | |
395 | 'http': SocksHTTPConnectionPool, | |
396 | 'https': SocksHTTPSConnectionPool | |
397 | } | |
398 | ||
399 | ||
400 | requests.adapters.SOCKSProxyManager = SocksProxyManager |