]>
jfr.im git - yt-dlp.git/blob - yt_dlp/networking/common.py
1 from __future__
import annotations
11 import urllib
.response
12 from collections
.abc
import Iterable
, Mapping
13 from email
.message
import Message
14 from http
import HTTPStatus
16 from ._helper
import make_ssl_context
, wrap_request_errors
17 from .exceptions
import (
23 from ..compat
.types
import NoneType
24 from ..cookies
import YoutubeDLCookieJar
32 from ..utils
.networking
import HTTPHeaderDict
, normalize_url
35 def register_preference(*handlers
: type[RequestHandler
]):
36 assert all(issubclass(handler
, RequestHandler
) for handler
in handlers
)
38 def outer(preference
: Preference
):
39 @functools.wraps(preference
)
40 def inner(handler
, *args
, **kwargs
):
41 if not handlers
or isinstance(handler
, handlers
):
42 return preference(handler
, *args
, **kwargs
)
44 _RH_PREFERENCES
.add(inner
)
49 class RequestDirector
:
50 """RequestDirector class
52 Helper class that, when given a request, forward it to a RequestHandler that supports it.
54 Preference functions in the form of func(handler, request) -> int
55 can be registered into the `preferences` set. These are used to sort handlers
56 in order of preference.
58 @param logger: Logger instance.
59 @param verbose: Print debug request information to stdout.
62 def __init__(self
, logger
, verbose
=False):
63 self
.handlers
: dict[str, RequestHandler
] = {}
64 self
.preferences
: set[Preference
] = set()
65 self
.logger
= logger
# TODO(Grub4k): default logger
66 self
.verbose
= verbose
69 for handler
in self
.handlers
.values():
72 def add_handler(self
, handler
: RequestHandler
):
73 """Add a handler. If a handler of the same RH_KEY exists, it will overwrite it"""
74 assert isinstance(handler
, RequestHandler
), 'handler must be a RequestHandler'
75 self
.handlers
[handler
.RH_KEY
] = handler
77 def _get_handlers(self
, request
: Request
) -> list[RequestHandler
]:
78 """Sorts handlers by preference, given a request"""
80 rh
: sum(pref(rh
, request
) for pref
in self
.preferences
)
81 for rh
in self
.handlers
.values()
83 self
._print
_verbose
('Handler preferences for this request: %s' % ', '.join(
84 f
'{rh.RH_NAME}={pref}' for rh
, pref
in preferences
.items()))
85 return sorted(self
.handlers
.values(), key
=preferences
.get
, reverse
=True)
87 def _print_verbose(self
, msg
):
89 self
.logger
.stdout(f
'director: {msg}')
91 def send(self
, request
: Request
) -> Response
:
93 Passes a request onto a suitable RequestHandler
96 raise RequestError('No request handlers configured')
98 assert isinstance(request
, Request
)
100 unexpected_errors
= []
101 unsupported_errors
= []
102 for handler
in self
._get
_handlers
(request
):
103 self
._print
_verbose
(f
'Checking if "{handler.RH_NAME}" supports this request.')
105 handler
.validate(request
)
106 except UnsupportedRequest
as e
:
108 f
'"{handler.RH_NAME}" cannot handle this request (reason: {error_to_str(e)})')
109 unsupported_errors
.append(e
)
112 self
._print
_verbose
(f
'Sending request via "{handler.RH_NAME}"')
114 response
= handler
.send(request
)
117 except Exception as e
:
119 f
'[{handler.RH_NAME}] Unexpected error: {error_to_str(e)}{bug_reports_message()}',
121 unexpected_errors
.append(e
)
124 assert isinstance(response
, Response
)
127 raise NoSupportingHandlers(unsupported_errors
, unexpected_errors
)
130 _REQUEST_HANDLERS
= {}
133 def register_rh(handler
):
134 """Register a RequestHandler class"""
135 assert issubclass(handler
, RequestHandler
), f
'{handler} must be a subclass of RequestHandler'
136 assert handler
.RH_KEY
not in _REQUEST_HANDLERS
, f
'RequestHandler {handler.RH_KEY} already registered'
137 _REQUEST_HANDLERS
[handler
.RH_KEY
] = handler
141 class Features(enum
.Enum
):
142 ALL_PROXY
= enum
.auto()
143 NO_PROXY
= enum
.auto()
146 class RequestHandler(abc
.ABC
):
148 """Request Handler class
150 Request handlers are class that, given a Request,
151 process the request from start to finish and return a Response.
153 Concrete subclasses need to redefine the _send(request) method,
154 which handles the underlying request logic and returns a Response.
156 RH_NAME class variable may contain a display name for the RequestHandler.
157 By default, this is generated from the class name.
159 The concrete request handler MUST have "RH" as the suffix in the class name.
161 All exceptions raised by a RequestHandler should be an instance of RequestError.
162 Any other exception raised will be treated as a handler issue.
164 If a Request is not supported by the handler, an UnsupportedRequest
165 should be raised with a reason.
167 By default, some checks are done on the request in _validate() based on the following class variables:
168 - `_SUPPORTED_URL_SCHEMES`: a tuple of supported url schemes.
169 Any Request with an url scheme not in this list will raise an UnsupportedRequest.
171 - `_SUPPORTED_PROXY_SCHEMES`: a tuple of support proxy url schemes. Any Request that contains
172 a proxy url with an url scheme not in this list will raise an UnsupportedRequest.
174 - `_SUPPORTED_FEATURES`: a tuple of supported features, as defined in Features enum.
176 The above may be set to None to disable the checks.
179 @param logger: logger instance
180 @param headers: HTTP Headers to include when sending requests.
181 @param cookiejar: Cookiejar to use for requests.
182 @param timeout: Socket timeout to use when sending requests.
183 @param proxies: Proxies to use for sending requests.
184 @param source_address: Client-side IP address to bind to for requests.
185 @param verbose: Print debug request and traffic information to stdout.
186 @param prefer_system_certs: Whether to prefer system certificates over other means (e.g. certifi).
187 @param client_cert: SSL client certificate configuration.
188 dict with {client_certificate, client_certificate_key, client_certificate_password}
189 @param verify: Verify SSL certificates
190 @param legacy_ssl_support: Enable legacy SSL options such as legacy server connect and older cipher support.
192 Some configuration options may be available for individual Requests too. In this case,
193 either the Request configuration option takes precedence or they are merged.
195 Requests may have additional optional parameters defined as extensions.
196 RequestHandler subclasses may choose to support custom extensions.
198 If an extension is supported, subclasses should extend _check_extensions(extensions)
199 to pop and validate the extension.
200 - Extensions left in `extensions` are treated as unsupported and UnsupportedRequest will be raised.
202 The following extensions are defined for RequestHandler:
203 - `cookiejar`: Cookiejar to use for this request.
204 - `timeout`: socket timeout to use for this request.
205 To enable these, add extensions.pop('<extension>', None) to _check_extensions
207 Apart from the url protocol, proxies dict may contain the following keys:
208 - `all`: proxy to use for all protocols. Used as a fallback if no proxy is set for a specific protocol.
209 - `no`: comma seperated list of hostnames (optionally with port) to not use a proxy for.
210 Note: a RequestHandler may not support these, as defined in `_SUPPORTED_FEATURES`.
214 _SUPPORTED_URL_SCHEMES
= ()
215 _SUPPORTED_PROXY_SCHEMES
= ()
216 _SUPPORTED_FEATURES
= ()
220 logger
, # TODO(Grub4k): default logger
221 headers
: HTTPHeaderDict
= None,
222 cookiejar
: YoutubeDLCookieJar
= None,
223 timeout
: float |
int |
None = None,
224 proxies
: dict = None,
225 source_address
: str = None,
226 verbose
: bool = False,
227 prefer_system_certs
: bool = False,
228 client_cert
: dict[str, str |
None] = None,
230 legacy_ssl_support
: bool = False,
234 self
._logger
= logger
235 self
.headers
= headers
or {}
236 self
.cookiejar
= cookiejar
if cookiejar
is not None else YoutubeDLCookieJar()
237 self
.timeout
= float(timeout
or 20)
238 self
.proxies
= proxies
or {}
239 self
.source_address
= source_address
240 self
.verbose
= verbose
241 self
.prefer_system_certs
= prefer_system_certs
242 self
._client
_cert
= client_cert
or {}
244 self
.legacy_ssl_support
= legacy_ssl_support
247 def _make_sslcontext(self
):
248 return make_ssl_context(
250 legacy_support
=self
.legacy_ssl_support
,
251 use_certifi
=not self
.prefer_system_certs
,
255 def _merge_headers(self
, request_headers
):
256 return HTTPHeaderDict(self
.headers
, request_headers
)
258 def _check_url_scheme(self
, request
: Request
):
259 scheme
= urllib
.parse
.urlparse(request
.url
).scheme
.lower()
260 if self
._SUPPORTED
_URL
_SCHEMES
is not None and scheme
not in self
._SUPPORTED
_URL
_SCHEMES
:
261 raise UnsupportedRequest(f
'Unsupported url scheme: "{scheme}"')
262 return scheme
# for further processing
264 def _check_proxies(self
, proxies
):
265 for proxy_key
, proxy_url
in proxies
.items():
266 if proxy_url
is None:
268 if proxy_key
== 'no':
269 if self
._SUPPORTED
_FEATURES
is not None and Features
.NO_PROXY
not in self
._SUPPORTED
_FEATURES
:
270 raise UnsupportedRequest('"no" proxy is not supported')
274 and self
._SUPPORTED
_FEATURES
is not None
275 and Features
.ALL_PROXY
not in self
._SUPPORTED
_FEATURES
277 raise UnsupportedRequest('"all" proxy is not supported')
279 # Unlikely this handler will use this proxy, so ignore.
280 # This is to allow a case where a proxy may be set for a protocol
281 # for one handler in which such protocol (and proxy) is not supported by another handler.
282 if self
._SUPPORTED
_URL
_SCHEMES
is not None and proxy_key
not in (*self
._SUPPORTED
_URL
_SCHEMES
, 'all'):
285 if self
._SUPPORTED
_PROXY
_SCHEMES
is None:
286 # Skip proxy scheme checks
290 if urllib
.request
._parse
_proxy
(proxy_url
)[0] is None:
291 # Scheme-less proxies are not supported
292 raise UnsupportedRequest(f
'Proxy "{proxy_url}" missing scheme')
293 except ValueError as e
:
294 # parse_proxy may raise on some invalid proxy urls such as "/a/b/c"
295 raise UnsupportedRequest(f
'Invalid proxy url "{proxy_url}": {e}')
297 scheme
= urllib
.parse
.urlparse(proxy_url
).scheme
.lower()
298 if scheme
not in self
._SUPPORTED
_PROXY
_SCHEMES
:
299 raise UnsupportedRequest(f
'Unsupported proxy type: "{scheme}"')
301 def _check_extensions(self
, extensions
):
302 """Check extensions for unsupported extensions. Subclasses should extend this."""
303 assert isinstance(extensions
.get('cookiejar'), (YoutubeDLCookieJar
, NoneType
))
304 assert isinstance(extensions
.get('timeout'), (float, int, NoneType
))
306 def _validate(self
, request
):
307 self
._check
_url
_scheme
(request
)
308 self
._check
_proxies
(request
.proxies
or self
.proxies
)
309 extensions
= request
.extensions
.copy()
310 self
._check
_extensions
(extensions
)
312 # TODO: add support for optional extensions
313 raise UnsupportedRequest(f
'Unsupported extensions: {", ".join(extensions.keys())}')
316 def validate(self
, request
: Request
):
317 if not isinstance(request
, Request
):
318 raise TypeError('Expected an instance of Request')
319 self
._validate
(request
)
322 def send(self
, request
: Request
) -> Response
:
323 if not isinstance(request
, Request
):
324 raise TypeError('Expected an instance of Request')
325 return self
._send
(request
)
328 def _send(self
, request
: Request
):
329 """Handle a request from start to finish. Redefine in subclasses."""
337 return cls
.__name
__[:-2]
341 assert cls
.__name
__.endswith('RH'), 'RequestHandler class names must end with "RH"'
342 return cls
.__name
__[:-2]
347 def __exit__(self
, *args
):
353 Represents a request to be made.
354 Partially backwards-compatible with urllib.request.Request.
356 @param url: url to send. Will be sanitized.
357 @param data: payload data to send. Must be bytes, iterable of bytes, a file-like object or None
358 @param headers: headers to send.
359 @param proxies: proxy dict mapping of proto:proxy to use for the request and any redirects.
360 @param query: URL query parameters to update the url with.
361 @param method: HTTP method to use. If no method specified, will use POST if payload data is present else GET
362 @param extensions: Dictionary of Request extensions to add, as supported by handlers.
368 data
: RequestData
= None,
369 headers
: typing
.Mapping
= None,
370 proxies
: dict = None,
373 extensions
: dict = None
376 self
._headers
= HTTPHeaderDict()
380 url
= update_url_query(url
, query
)
385 self
.headers
= headers
386 self
.data
= data
# note: must be done after setting headers
387 self
.proxies
= proxies
or {}
388 self
.extensions
= extensions
or {}
396 if not isinstance(url
, str):
397 raise TypeError('url must be a string')
398 elif url
.startswith('//'):
400 self
._url
= normalize_url(url
)
404 return self
._method
or ('POST' if self
.data
is not None else 'GET')
407 def method(self
, method
):
410 elif isinstance(method
, str):
411 self
._method
= method
.upper()
413 raise TypeError('method must be a string')
420 def data(self
, data
: RequestData
):
421 # Try catch some common mistakes
422 if data
is not None and (
423 not isinstance(data
, (bytes, io
.IOBase
, Iterable
)) or isinstance(data
, (str, Mapping
))
425 raise TypeError('data must be bytes, iterable of bytes, or a file-like object')
427 if data
== self
._data
and self
._data
is None:
428 self
.headers
.pop('Content-Length', None)
430 # https://docs.python.org/3/library/urllib.request.html#urllib.request.Request.data
431 if data
!= self
._data
:
432 if self
._data
is not None:
433 self
.headers
.pop('Content-Length', None)
436 if self
._data
is None:
437 self
.headers
.pop('Content-Type', None)
439 if 'Content-Type' not in self
.headers
and self
._data
is not None:
440 self
.headers
['Content-Type'] = 'application/x-www-form-urlencoded'
443 def headers(self
) -> HTTPHeaderDict
:
447 def headers(self
, new_headers
: Mapping
):
448 """Replaces headers of the request. If not a CaseInsensitiveDict, it will be converted to one."""
449 if isinstance(new_headers
, HTTPHeaderDict
):
450 self
._headers
= new_headers
451 elif isinstance(new_headers
, Mapping
):
452 self
._headers
= HTTPHeaderDict(new_headers
)
454 raise TypeError('headers must be a mapping')
456 def update(self
, url
=None, data
=None, headers
=None, query
=None):
457 self
.data
= data
if data
is not None else self
.data
458 self
.headers
.update(headers
or {})
459 self
.url
= update_url_query(url
or self
.url
, query
or {})
462 return self
.__class
__(
464 headers
=copy
.deepcopy(self
.headers
),
465 proxies
=copy
.deepcopy(self
.proxies
),
467 extensions
=copy
.copy(self
.extensions
),
472 HEADRequest
= functools
.partial(Request
, method
='HEAD')
473 PUTRequest
= functools
.partial(Request
, method
='PUT')
476 class Response(io
.IOBase
):
478 Base class for HTTP response adapters.
480 By default, it provides a basic wrapper for a file-like response object.
482 Interface partially backwards-compatible with addinfourl and http.client.HTTPResponse.
484 @param fp: Original, file-like, response.
485 @param url: URL that this is a response of.
486 @param headers: response headers.
487 @param status: Response HTTP status code. Default is 200 OK.
488 @param reason: HTTP status reason. Will use built-in reasons based on status code if not provided.
495 headers
: Mapping
[str, str],
500 self
.headers
= Message()
501 for name
, value
in headers
.items():
502 self
.headers
.add_header(name
, value
)
506 self
.reason
= reason
or HTTPStatus(status
).phrase
511 return self
.fp
.readable()
513 def read(self
, amt
: int = None) -> bytes:
514 # Expected errors raised here should be of type RequestError or subclasses.
515 # Subclasses should redefine this method with more precise error handling.
517 return self
.fp
.read(amt
)
518 except Exception as e
:
519 raise TransportError(cause
=e
) from e
523 return super().close()
525 def get_header(self
, name
, default
=None):
526 """Get header for name.
527 If there are multiple matching headers, return all seperated by comma."""
528 headers
= self
.headers
.get_all(name
)
531 if name
.title() == 'Set-Cookie':
532 # Special case, only get the first one
533 # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.3-4.1
535 return ', '.join(headers
)
537 # The following methods are for compatability reasons and are deprecated
540 deprecation_warning('Response.code is deprecated, use Response.status', stacklevel
=2)
544 deprecation_warning('Response.getcode() is deprecated, use Response.status', stacklevel
=2)
548 deprecation_warning('Response.geturl() is deprecated, use Response.url', stacklevel
=2)
552 deprecation_warning('Response.info() is deprecated, use Response.headers', stacklevel
=2)
555 def getheader(self
, name
, default
=None):
556 deprecation_warning('Response.getheader() is deprecated, use Response.get_header', stacklevel
=2)
557 return self
.get_header(name
, default
)
560 if typing
.TYPE_CHECKING
:
561 RequestData
= bytes | Iterable
[bytes] | typing
.IO |
None
562 Preference
= typing
.Callable
[[RequestHandler
, Request
], int]
564 _RH_PREFERENCES
: set[Preference
] = set()