]>
Commit | Line | Data |
---|---|---|
227bf1a3 | 1 | import collections |
c365dba8 | 2 | import random |
227bf1a3 | 3 | import urllib.parse |
4 | import urllib.request | |
5 | ||
6 | from ._utils import remove_start | |
c365dba8 | 7 | |
8 | ||
9 | def random_user_agent(): | |
10 | _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36' | |
11 | _CHROME_VERSIONS = ( | |
12 | '90.0.4430.212', | |
13 | '90.0.4430.24', | |
14 | '90.0.4430.70', | |
15 | '90.0.4430.72', | |
16 | '90.0.4430.85', | |
17 | '90.0.4430.93', | |
18 | '91.0.4472.101', | |
19 | '91.0.4472.106', | |
20 | '91.0.4472.114', | |
21 | '91.0.4472.124', | |
22 | '91.0.4472.164', | |
23 | '91.0.4472.19', | |
24 | '91.0.4472.77', | |
25 | '92.0.4515.107', | |
26 | '92.0.4515.115', | |
27 | '92.0.4515.131', | |
28 | '92.0.4515.159', | |
29 | '92.0.4515.43', | |
30 | '93.0.4556.0', | |
31 | '93.0.4577.15', | |
32 | '93.0.4577.63', | |
33 | '93.0.4577.82', | |
34 | '94.0.4606.41', | |
35 | '94.0.4606.54', | |
36 | '94.0.4606.61', | |
37 | '94.0.4606.71', | |
38 | '94.0.4606.81', | |
39 | '94.0.4606.85', | |
40 | '95.0.4638.17', | |
41 | '95.0.4638.50', | |
42 | '95.0.4638.54', | |
43 | '95.0.4638.69', | |
44 | '95.0.4638.74', | |
45 | '96.0.4664.18', | |
46 | '96.0.4664.45', | |
47 | '96.0.4664.55', | |
48 | '96.0.4664.93', | |
49 | '97.0.4692.20', | |
50 | ) | |
51 | return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS) | |
52 | ||
53 | ||
227bf1a3 | 54 | class HTTPHeaderDict(collections.UserDict, dict): |
55 | """ | |
56 | Store and access keys case-insensitively. | |
57 | The constructor can take multiple dicts, in which keys in the latter are prioritised. | |
58 | """ | |
59 | ||
60 | def __init__(self, *args, **kwargs): | |
61 | super().__init__() | |
62 | for dct in args: | |
63 | if dct is not None: | |
64 | self.update(dct) | |
65 | self.update(kwargs) | |
66 | ||
67 | def __setitem__(self, key, value): | |
68 | super().__setitem__(key.title(), str(value)) | |
69 | ||
70 | def __getitem__(self, key): | |
71 | return super().__getitem__(key.title()) | |
72 | ||
73 | def __delitem__(self, key): | |
74 | super().__delitem__(key.title()) | |
75 | ||
76 | def __contains__(self, key): | |
77 | return super().__contains__(key.title() if isinstance(key, str) else key) | |
78 | ||
79 | ||
80 | std_headers = HTTPHeaderDict({ | |
c365dba8 | 81 | 'User-Agent': random_user_agent(), |
82 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', | |
83 | 'Accept-Language': 'en-us,en;q=0.5', | |
84 | 'Sec-Fetch-Mode': 'navigate', | |
227bf1a3 | 85 | }) |
86 | ||
87 | ||
88 | def clean_proxies(proxies: dict, headers: HTTPHeaderDict): | |
89 | req_proxy = headers.pop('Ytdl-Request-Proxy', None) | |
90 | if req_proxy: | |
91 | proxies.clear() # XXX: compat: Ytdl-Request-Proxy takes preference over everything, including NO_PROXY | |
92 | proxies['all'] = req_proxy | |
93 | for proxy_key, proxy_url in proxies.items(): | |
94 | if proxy_url == '__noproxy__': | |
95 | proxies[proxy_key] = None | |
96 | continue | |
97 | if proxy_key == 'no': # special case | |
98 | continue | |
99 | if proxy_url is not None: | |
100 | # Ensure proxies without a scheme are http. | |
101 | proxy_scheme = urllib.request._parse_proxy(proxy_url)[0] | |
102 | if proxy_scheme is None: | |
103 | proxies[proxy_key] = 'http://' + remove_start(proxy_url, '//') | |
104 | ||
105 | replace_scheme = { | |
106 | 'socks5': 'socks5h', # compat: socks5 was treated as socks5h | |
107 | 'socks': 'socks4' # compat: non-standard | |
108 | } | |
109 | if proxy_scheme in replace_scheme: | |
110 | proxies[proxy_key] = urllib.parse.urlunparse( | |
111 | urllib.parse.urlparse(proxy_url)._replace(scheme=replace_scheme[proxy_scheme])) | |
c365dba8 | 112 | |
113 | ||
227bf1a3 | 114 | def clean_headers(headers: HTTPHeaderDict): |
115 | if 'Youtubedl-No-Compression' in headers: # compat | |
116 | del headers['Youtubedl-No-Compression'] | |
c365dba8 | 117 | headers['Accept-Encoding'] = 'identity' |