]>
Commit | Line | Data |
---|---|---|
1 | from __future__ import annotations | |
2 | ||
3 | import abc | |
4 | import copy | |
5 | import enum | |
6 | import functools | |
7 | import io | |
8 | import typing | |
9 | import urllib.parse | |
10 | import urllib.request | |
11 | import urllib.response | |
12 | from collections.abc import Iterable, Mapping | |
13 | from email.message import Message | |
14 | from http import HTTPStatus | |
15 | ||
16 | from ._helper import make_ssl_context, wrap_request_errors | |
17 | from .exceptions import ( | |
18 | NoSupportingHandlers, | |
19 | RequestError, | |
20 | TransportError, | |
21 | UnsupportedRequest, | |
22 | ) | |
23 | from ..compat.types import NoneType | |
24 | from ..cookies import YoutubeDLCookieJar | |
25 | from ..utils import ( | |
26 | bug_reports_message, | |
27 | classproperty, | |
28 | deprecation_warning, | |
29 | error_to_str, | |
30 | update_url_query, | |
31 | ) | |
32 | from ..utils.networking import HTTPHeaderDict, normalize_url | |
33 | ||
34 | ||
35 | def register_preference(*handlers: type[RequestHandler]): | |
36 | assert all(issubclass(handler, RequestHandler) for handler in handlers) | |
37 | ||
38 | def outer(preference: Preference): | |
39 | @functools.wraps(preference) | |
40 | def inner(handler, *args, **kwargs): | |
41 | if not handlers or isinstance(handler, handlers): | |
42 | return preference(handler, *args, **kwargs) | |
43 | return 0 | |
44 | _RH_PREFERENCES.add(inner) | |
45 | return inner | |
46 | return outer | |
47 | ||
48 | ||
49 | class RequestDirector: | |
50 | """RequestDirector class | |
51 | ||
52 | Helper class that, when given a request, forward it to a RequestHandler that supports it. | |
53 | ||
54 | Preference functions in the form of func(handler, request) -> int | |
55 | can be registered into the `preferences` set. These are used to sort handlers | |
56 | in order of preference. | |
57 | ||
58 | @param logger: Logger instance. | |
59 | @param verbose: Print debug request information to stdout. | |
60 | """ | |
61 | ||
62 | def __init__(self, logger, verbose=False): | |
63 | self.handlers: dict[str, RequestHandler] = {} | |
64 | self.preferences: set[Preference] = set() | |
65 | self.logger = logger # TODO(Grub4k): default logger | |
66 | self.verbose = verbose | |
67 | ||
68 | def close(self): | |
69 | for handler in self.handlers.values(): | |
70 | handler.close() | |
71 | self.handlers.clear() | |
72 | ||
73 | def add_handler(self, handler: RequestHandler): | |
74 | """Add a handler. If a handler of the same RH_KEY exists, it will overwrite it""" | |
75 | assert isinstance(handler, RequestHandler), 'handler must be a RequestHandler' | |
76 | self.handlers[handler.RH_KEY] = handler | |
77 | ||
78 | def _get_handlers(self, request: Request) -> list[RequestHandler]: | |
79 | """Sorts handlers by preference, given a request""" | |
80 | preferences = { | |
81 | rh: sum(pref(rh, request) for pref in self.preferences) | |
82 | for rh in self.handlers.values() | |
83 | } | |
84 | self._print_verbose('Handler preferences for this request: %s' % ', '.join( | |
85 | f'{rh.RH_NAME}={pref}' for rh, pref in preferences.items())) | |
86 | return sorted(self.handlers.values(), key=preferences.get, reverse=True) | |
87 | ||
88 | def _print_verbose(self, msg): | |
89 | if self.verbose: | |
90 | self.logger.stdout(f'director: {msg}') | |
91 | ||
92 | def send(self, request: Request) -> Response: | |
93 | """ | |
94 | Passes a request onto a suitable RequestHandler | |
95 | """ | |
96 | if not self.handlers: | |
97 | raise RequestError('No request handlers configured') | |
98 | ||
99 | assert isinstance(request, Request) | |
100 | ||
101 | unexpected_errors = [] | |
102 | unsupported_errors = [] | |
103 | for handler in self._get_handlers(request): | |
104 | self._print_verbose(f'Checking if "{handler.RH_NAME}" supports this request.') | |
105 | try: | |
106 | handler.validate(request) | |
107 | except UnsupportedRequest as e: | |
108 | self._print_verbose( | |
109 | f'"{handler.RH_NAME}" cannot handle this request (reason: {error_to_str(e)})') | |
110 | unsupported_errors.append(e) | |
111 | continue | |
112 | ||
113 | self._print_verbose(f'Sending request via "{handler.RH_NAME}"') | |
114 | try: | |
115 | response = handler.send(request) | |
116 | except RequestError: | |
117 | raise | |
118 | except Exception as e: | |
119 | self.logger.error( | |
120 | f'[{handler.RH_NAME}] Unexpected error: {error_to_str(e)}{bug_reports_message()}', | |
121 | is_error=False) | |
122 | unexpected_errors.append(e) | |
123 | continue | |
124 | ||
125 | assert isinstance(response, Response) | |
126 | return response | |
127 | ||
128 | raise NoSupportingHandlers(unsupported_errors, unexpected_errors) | |
129 | ||
130 | ||
131 | _REQUEST_HANDLERS = {} | |
132 | ||
133 | ||
134 | def register_rh(handler): | |
135 | """Register a RequestHandler class""" | |
136 | assert issubclass(handler, RequestHandler), f'{handler} must be a subclass of RequestHandler' | |
137 | assert handler.RH_KEY not in _REQUEST_HANDLERS, f'RequestHandler {handler.RH_KEY} already registered' | |
138 | _REQUEST_HANDLERS[handler.RH_KEY] = handler | |
139 | return handler | |
140 | ||
141 | ||
142 | class Features(enum.Enum): | |
143 | ALL_PROXY = enum.auto() | |
144 | NO_PROXY = enum.auto() | |
145 | ||
146 | ||
147 | class RequestHandler(abc.ABC): | |
148 | ||
149 | """Request Handler class | |
150 | ||
151 | Request handlers are class that, given a Request, | |
152 | process the request from start to finish and return a Response. | |
153 | ||
154 | Concrete subclasses need to redefine the _send(request) method, | |
155 | which handles the underlying request logic and returns a Response. | |
156 | ||
157 | RH_NAME class variable may contain a display name for the RequestHandler. | |
158 | By default, this is generated from the class name. | |
159 | ||
160 | The concrete request handler MUST have "RH" as the suffix in the class name. | |
161 | ||
162 | All exceptions raised by a RequestHandler should be an instance of RequestError. | |
163 | Any other exception raised will be treated as a handler issue. | |
164 | ||
165 | If a Request is not supported by the handler, an UnsupportedRequest | |
166 | should be raised with a reason. | |
167 | ||
168 | By default, some checks are done on the request in _validate() based on the following class variables: | |
169 | - `_SUPPORTED_URL_SCHEMES`: a tuple of supported url schemes. | |
170 | Any Request with an url scheme not in this list will raise an UnsupportedRequest. | |
171 | ||
172 | - `_SUPPORTED_PROXY_SCHEMES`: a tuple of support proxy url schemes. Any Request that contains | |
173 | a proxy url with an url scheme not in this list will raise an UnsupportedRequest. | |
174 | ||
175 | - `_SUPPORTED_FEATURES`: a tuple of supported features, as defined in Features enum. | |
176 | ||
177 | The above may be set to None to disable the checks. | |
178 | ||
179 | Parameters: | |
180 | @param logger: logger instance | |
181 | @param headers: HTTP Headers to include when sending requests. | |
182 | @param cookiejar: Cookiejar to use for requests. | |
183 | @param timeout: Socket timeout to use when sending requests. | |
184 | @param proxies: Proxies to use for sending requests. | |
185 | @param source_address: Client-side IP address to bind to for requests. | |
186 | @param verbose: Print debug request and traffic information to stdout. | |
187 | @param prefer_system_certs: Whether to prefer system certificates over other means (e.g. certifi). | |
188 | @param client_cert: SSL client certificate configuration. | |
189 | dict with {client_certificate, client_certificate_key, client_certificate_password} | |
190 | @param verify: Verify SSL certificates | |
191 | @param legacy_ssl_support: Enable legacy SSL options such as legacy server connect and older cipher support. | |
192 | ||
193 | Some configuration options may be available for individual Requests too. In this case, | |
194 | either the Request configuration option takes precedence or they are merged. | |
195 | ||
196 | Requests may have additional optional parameters defined as extensions. | |
197 | RequestHandler subclasses may choose to support custom extensions. | |
198 | ||
199 | If an extension is supported, subclasses should extend _check_extensions(extensions) | |
200 | to pop and validate the extension. | |
201 | - Extensions left in `extensions` are treated as unsupported and UnsupportedRequest will be raised. | |
202 | ||
203 | The following extensions are defined for RequestHandler: | |
204 | - `cookiejar`: Cookiejar to use for this request. | |
205 | - `timeout`: socket timeout to use for this request. | |
206 | To enable these, add extensions.pop('<extension>', None) to _check_extensions | |
207 | ||
208 | Apart from the url protocol, proxies dict may contain the following keys: | |
209 | - `all`: proxy to use for all protocols. Used as a fallback if no proxy is set for a specific protocol. | |
210 | - `no`: comma seperated list of hostnames (optionally with port) to not use a proxy for. | |
211 | Note: a RequestHandler may not support these, as defined in `_SUPPORTED_FEATURES`. | |
212 | ||
213 | """ | |
214 | ||
215 | _SUPPORTED_URL_SCHEMES = () | |
216 | _SUPPORTED_PROXY_SCHEMES = () | |
217 | _SUPPORTED_FEATURES = () | |
218 | ||
219 | def __init__( | |
220 | self, *, | |
221 | logger, # TODO(Grub4k): default logger | |
222 | headers: HTTPHeaderDict = None, | |
223 | cookiejar: YoutubeDLCookieJar = None, | |
224 | timeout: float | int | None = None, | |
225 | proxies: dict = None, | |
226 | source_address: str = None, | |
227 | verbose: bool = False, | |
228 | prefer_system_certs: bool = False, | |
229 | client_cert: dict[str, str | None] = None, | |
230 | verify: bool = True, | |
231 | legacy_ssl_support: bool = False, | |
232 | **_, | |
233 | ): | |
234 | ||
235 | self._logger = logger | |
236 | self.headers = headers or {} | |
237 | self.cookiejar = cookiejar if cookiejar is not None else YoutubeDLCookieJar() | |
238 | self.timeout = float(timeout or 20) | |
239 | self.proxies = proxies or {} | |
240 | self.source_address = source_address | |
241 | self.verbose = verbose | |
242 | self.prefer_system_certs = prefer_system_certs | |
243 | self._client_cert = client_cert or {} | |
244 | self.verify = verify | |
245 | self.legacy_ssl_support = legacy_ssl_support | |
246 | super().__init__() | |
247 | ||
248 | def _make_sslcontext(self): | |
249 | return make_ssl_context( | |
250 | verify=self.verify, | |
251 | legacy_support=self.legacy_ssl_support, | |
252 | use_certifi=not self.prefer_system_certs, | |
253 | **self._client_cert, | |
254 | ) | |
255 | ||
256 | def _merge_headers(self, request_headers): | |
257 | return HTTPHeaderDict(self.headers, request_headers) | |
258 | ||
259 | def _calculate_timeout(self, request): | |
260 | return float(request.extensions.get('timeout') or self.timeout) | |
261 | ||
262 | def _get_cookiejar(self, request): | |
263 | return request.extensions.get('cookiejar') or self.cookiejar | |
264 | ||
265 | def _get_proxies(self, request): | |
266 | return (request.proxies or self.proxies).copy() | |
267 | ||
268 | def _check_url_scheme(self, request: Request): | |
269 | scheme = urllib.parse.urlparse(request.url).scheme.lower() | |
270 | if self._SUPPORTED_URL_SCHEMES is not None and scheme not in self._SUPPORTED_URL_SCHEMES: | |
271 | raise UnsupportedRequest(f'Unsupported url scheme: "{scheme}"') | |
272 | return scheme # for further processing | |
273 | ||
274 | def _check_proxies(self, proxies): | |
275 | for proxy_key, proxy_url in proxies.items(): | |
276 | if proxy_url is None: | |
277 | continue | |
278 | if proxy_key == 'no': | |
279 | if self._SUPPORTED_FEATURES is not None and Features.NO_PROXY not in self._SUPPORTED_FEATURES: | |
280 | raise UnsupportedRequest('"no" proxy is not supported') | |
281 | continue | |
282 | if ( | |
283 | proxy_key == 'all' | |
284 | and self._SUPPORTED_FEATURES is not None | |
285 | and Features.ALL_PROXY not in self._SUPPORTED_FEATURES | |
286 | ): | |
287 | raise UnsupportedRequest('"all" proxy is not supported') | |
288 | ||
289 | # Unlikely this handler will use this proxy, so ignore. | |
290 | # This is to allow a case where a proxy may be set for a protocol | |
291 | # for one handler in which such protocol (and proxy) is not supported by another handler. | |
292 | if self._SUPPORTED_URL_SCHEMES is not None and proxy_key not in (*self._SUPPORTED_URL_SCHEMES, 'all'): | |
293 | continue | |
294 | ||
295 | if self._SUPPORTED_PROXY_SCHEMES is None: | |
296 | # Skip proxy scheme checks | |
297 | continue | |
298 | ||
299 | try: | |
300 | if urllib.request._parse_proxy(proxy_url)[0] is None: | |
301 | # Scheme-less proxies are not supported | |
302 | raise UnsupportedRequest(f'Proxy "{proxy_url}" missing scheme') | |
303 | except ValueError as e: | |
304 | # parse_proxy may raise on some invalid proxy urls such as "/a/b/c" | |
305 | raise UnsupportedRequest(f'Invalid proxy url "{proxy_url}": {e}') | |
306 | ||
307 | scheme = urllib.parse.urlparse(proxy_url).scheme.lower() | |
308 | if scheme not in self._SUPPORTED_PROXY_SCHEMES: | |
309 | raise UnsupportedRequest(f'Unsupported proxy type: "{scheme}"') | |
310 | ||
311 | def _check_extensions(self, extensions): | |
312 | """Check extensions for unsupported extensions. Subclasses should extend this.""" | |
313 | assert isinstance(extensions.get('cookiejar'), (YoutubeDLCookieJar, NoneType)) | |
314 | assert isinstance(extensions.get('timeout'), (float, int, NoneType)) | |
315 | ||
316 | def _validate(self, request): | |
317 | self._check_url_scheme(request) | |
318 | self._check_proxies(request.proxies or self.proxies) | |
319 | extensions = request.extensions.copy() | |
320 | self._check_extensions(extensions) | |
321 | if extensions: | |
322 | # TODO: add support for optional extensions | |
323 | raise UnsupportedRequest(f'Unsupported extensions: {", ".join(extensions.keys())}') | |
324 | ||
325 | @wrap_request_errors | |
326 | def validate(self, request: Request): | |
327 | if not isinstance(request, Request): | |
328 | raise TypeError('Expected an instance of Request') | |
329 | self._validate(request) | |
330 | ||
331 | @wrap_request_errors | |
332 | def send(self, request: Request) -> Response: | |
333 | if not isinstance(request, Request): | |
334 | raise TypeError('Expected an instance of Request') | |
335 | return self._send(request) | |
336 | ||
337 | @abc.abstractmethod | |
338 | def _send(self, request: Request): | |
339 | """Handle a request from start to finish. Redefine in subclasses.""" | |
340 | pass | |
341 | ||
342 | def close(self): | |
343 | pass | |
344 | ||
345 | @classproperty | |
346 | def RH_NAME(cls): | |
347 | return cls.__name__[:-2] | |
348 | ||
349 | @classproperty | |
350 | def RH_KEY(cls): | |
351 | assert cls.__name__.endswith('RH'), 'RequestHandler class names must end with "RH"' | |
352 | return cls.__name__[:-2] | |
353 | ||
354 | def __enter__(self): | |
355 | return self | |
356 | ||
357 | def __exit__(self, *args): | |
358 | self.close() | |
359 | ||
360 | ||
361 | class Request: | |
362 | """ | |
363 | Represents a request to be made. | |
364 | Partially backwards-compatible with urllib.request.Request. | |
365 | ||
366 | @param url: url to send. Will be sanitized. | |
367 | @param data: payload data to send. Must be bytes, iterable of bytes, a file-like object or None | |
368 | @param headers: headers to send. | |
369 | @param proxies: proxy dict mapping of proto:proxy to use for the request and any redirects. | |
370 | @param query: URL query parameters to update the url with. | |
371 | @param method: HTTP method to use. If no method specified, will use POST if payload data is present else GET | |
372 | @param extensions: Dictionary of Request extensions to add, as supported by handlers. | |
373 | """ | |
374 | ||
375 | def __init__( | |
376 | self, | |
377 | url: str, | |
378 | data: RequestData = None, | |
379 | headers: typing.Mapping = None, | |
380 | proxies: dict = None, | |
381 | query: dict = None, | |
382 | method: str = None, | |
383 | extensions: dict = None | |
384 | ): | |
385 | ||
386 | self._headers = HTTPHeaderDict() | |
387 | self._data = None | |
388 | ||
389 | if query: | |
390 | url = update_url_query(url, query) | |
391 | ||
392 | self.url = url | |
393 | self.method = method | |
394 | if headers: | |
395 | self.headers = headers | |
396 | self.data = data # note: must be done after setting headers | |
397 | self.proxies = proxies or {} | |
398 | self.extensions = extensions or {} | |
399 | ||
400 | @property | |
401 | def url(self): | |
402 | return self._url | |
403 | ||
404 | @url.setter | |
405 | def url(self, url): | |
406 | if not isinstance(url, str): | |
407 | raise TypeError('url must be a string') | |
408 | elif url.startswith('//'): | |
409 | url = 'http:' + url | |
410 | self._url = normalize_url(url) | |
411 | ||
412 | @property | |
413 | def method(self): | |
414 | return self._method or ('POST' if self.data is not None else 'GET') | |
415 | ||
416 | @method.setter | |
417 | def method(self, method): | |
418 | if method is None: | |
419 | self._method = None | |
420 | elif isinstance(method, str): | |
421 | self._method = method.upper() | |
422 | else: | |
423 | raise TypeError('method must be a string') | |
424 | ||
425 | @property | |
426 | def data(self): | |
427 | return self._data | |
428 | ||
429 | @data.setter | |
430 | def data(self, data: RequestData): | |
431 | # Try catch some common mistakes | |
432 | if data is not None and ( | |
433 | not isinstance(data, (bytes, io.IOBase, Iterable)) or isinstance(data, (str, Mapping)) | |
434 | ): | |
435 | raise TypeError('data must be bytes, iterable of bytes, or a file-like object') | |
436 | ||
437 | if data == self._data and self._data is None: | |
438 | self.headers.pop('Content-Length', None) | |
439 | ||
440 | # https://docs.python.org/3/library/urllib.request.html#urllib.request.Request.data | |
441 | if data != self._data: | |
442 | if self._data is not None: | |
443 | self.headers.pop('Content-Length', None) | |
444 | self._data = data | |
445 | ||
446 | if self._data is None: | |
447 | self.headers.pop('Content-Type', None) | |
448 | ||
449 | if 'Content-Type' not in self.headers and self._data is not None: | |
450 | self.headers['Content-Type'] = 'application/x-www-form-urlencoded' | |
451 | ||
452 | @property | |
453 | def headers(self) -> HTTPHeaderDict: | |
454 | return self._headers | |
455 | ||
456 | @headers.setter | |
457 | def headers(self, new_headers: Mapping): | |
458 | """Replaces headers of the request. If not a HTTPHeaderDict, it will be converted to one.""" | |
459 | if isinstance(new_headers, HTTPHeaderDict): | |
460 | self._headers = new_headers | |
461 | elif isinstance(new_headers, Mapping): | |
462 | self._headers = HTTPHeaderDict(new_headers) | |
463 | else: | |
464 | raise TypeError('headers must be a mapping') | |
465 | ||
466 | def update(self, url=None, data=None, headers=None, query=None, extensions=None): | |
467 | self.data = data if data is not None else self.data | |
468 | self.headers.update(headers or {}) | |
469 | self.extensions.update(extensions or {}) | |
470 | self.url = update_url_query(url or self.url, query or {}) | |
471 | ||
472 | def copy(self): | |
473 | return self.__class__( | |
474 | url=self.url, | |
475 | headers=copy.deepcopy(self.headers), | |
476 | proxies=copy.deepcopy(self.proxies), | |
477 | data=self._data, | |
478 | extensions=copy.copy(self.extensions), | |
479 | method=self._method, | |
480 | ) | |
481 | ||
482 | ||
483 | HEADRequest = functools.partial(Request, method='HEAD') | |
484 | PUTRequest = functools.partial(Request, method='PUT') | |
485 | ||
486 | ||
487 | class Response(io.IOBase): | |
488 | """ | |
489 | Base class for HTTP response adapters. | |
490 | ||
491 | By default, it provides a basic wrapper for a file-like response object. | |
492 | ||
493 | Interface partially backwards-compatible with addinfourl and http.client.HTTPResponse. | |
494 | ||
495 | @param fp: Original, file-like, response. | |
496 | @param url: URL that this is a response of. | |
497 | @param headers: response headers. | |
498 | @param status: Response HTTP status code. Default is 200 OK. | |
499 | @param reason: HTTP status reason. Will use built-in reasons based on status code if not provided. | |
500 | @param extensions: Dictionary of handler-specific response extensions. | |
501 | """ | |
502 | ||
503 | def __init__( | |
504 | self, | |
505 | fp: io.IOBase, | |
506 | url: str, | |
507 | headers: Mapping[str, str], | |
508 | status: int = 200, | |
509 | reason: str = None, | |
510 | extensions: dict = None | |
511 | ): | |
512 | ||
513 | self.fp = fp | |
514 | self.headers = Message() | |
515 | for name, value in headers.items(): | |
516 | self.headers.add_header(name, value) | |
517 | self.status = status | |
518 | self.url = url | |
519 | try: | |
520 | self.reason = reason or HTTPStatus(status).phrase | |
521 | except ValueError: | |
522 | self.reason = None | |
523 | self.extensions = extensions or {} | |
524 | ||
525 | def readable(self): | |
526 | return self.fp.readable() | |
527 | ||
528 | def read(self, amt: int = None) -> bytes: | |
529 | # Expected errors raised here should be of type RequestError or subclasses. | |
530 | # Subclasses should redefine this method with more precise error handling. | |
531 | try: | |
532 | return self.fp.read(amt) | |
533 | except Exception as e: | |
534 | raise TransportError(cause=e) from e | |
535 | ||
536 | def close(self): | |
537 | self.fp.close() | |
538 | return super().close() | |
539 | ||
540 | def get_header(self, name, default=None): | |
541 | """Get header for name. | |
542 | If there are multiple matching headers, return all seperated by comma.""" | |
543 | headers = self.headers.get_all(name) | |
544 | if not headers: | |
545 | return default | |
546 | if name.title() == 'Set-Cookie': | |
547 | # Special case, only get the first one | |
548 | # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.3-4.1 | |
549 | return headers[0] | |
550 | return ', '.join(headers) | |
551 | ||
552 | # The following methods are for compatability reasons and are deprecated | |
553 | @property | |
554 | def code(self): | |
555 | deprecation_warning('Response.code is deprecated, use Response.status', stacklevel=2) | |
556 | return self.status | |
557 | ||
558 | def getcode(self): | |
559 | deprecation_warning('Response.getcode() is deprecated, use Response.status', stacklevel=2) | |
560 | return self.status | |
561 | ||
562 | def geturl(self): | |
563 | deprecation_warning('Response.geturl() is deprecated, use Response.url', stacklevel=2) | |
564 | return self.url | |
565 | ||
566 | def info(self): | |
567 | deprecation_warning('Response.info() is deprecated, use Response.headers', stacklevel=2) | |
568 | return self.headers | |
569 | ||
570 | def getheader(self, name, default=None): | |
571 | deprecation_warning('Response.getheader() is deprecated, use Response.get_header', stacklevel=2) | |
572 | return self.get_header(name, default) | |
573 | ||
574 | ||
575 | if typing.TYPE_CHECKING: | |
576 | RequestData = bytes | Iterable[bytes] | typing.IO | None | |
577 | Preference = typing.Callable[[RequestHandler, Request], int] | |
578 | ||
579 | _RH_PREFERENCES: set[Preference] = set() |