4 from __future__
import unicode_literals
39 import xml
.etree
.ElementTree
43 compat_HTMLParseError
,
49 compat_ctypes_WINFUNCTYPE
,
50 compat_etree_fromstring
,
53 compat_html_entities_html5
,
66 compat_urllib_parse_urlencode
,
67 compat_urllib_parse_urlparse
,
68 compat_urllib_parse_urlunparse
,
69 compat_urllib_parse_quote
,
70 compat_urllib_parse_quote_plus
,
71 compat_urllib_parse_unquote_plus
,
72 compat_urllib_request
,
83 def register_socks_protocols():
84 # "Register" SOCKS protocols
85 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
86 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
87 for scheme
in ('socks', 'socks4', 'socks4a', 'socks5'):
88 if scheme
not in compat_urlparse
.uses_netloc
:
89 compat_urlparse
.uses_netloc
.append(scheme
)
92 # This is not clearly defined otherwise
93 compiled_regex_type
= type(re
.compile(''))
96 def random_user_agent():
97 _USER_AGENT_TPL
= 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
1676 return _USER_AGENT_TPL
% random
.choice(_CHROME_VERSIONS
)
1680 'User-Agent': random_user_agent(),
1681 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
1682 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1683 'Accept-Encoding': 'gzip, deflate',
1684 'Accept-Language': 'en-us,en;q=0.5',
1689 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
1693 NO_DEFAULT
= object()
1695 ENGLISH_MONTH_NAMES
= [
1696 'January', 'February', 'March', 'April', 'May', 'June',
1697 'July', 'August', 'September', 'October', 'November', 'December']
1700 'en': ENGLISH_MONTH_NAMES
,
1702 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
1703 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
1706 KNOWN_EXTENSIONS
= (
1707 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
1708 'flv', 'f4v', 'f4a', 'f4b',
1709 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
1710 'mkv', 'mka', 'mk3d',
1713 'asf', 'wmv', 'wma',
1719 'f4f', 'f4m', 'm3u8', 'smil')
1721 # needed for sanitizing filenames in restricted mode
1722 ACCENT_CHARS
= dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
1723 itertools
.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
1724 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
1748 '%Y/%m/%d %H:%M:%S',
1752 '%Y-%m-%d %H:%M:%S',
1753 '%Y-%m-%d %H:%M:%S.%f',
1754 '%Y-%m-%d %H:%M:%S:%f',
1757 '%Y-%m-%dT%H:%M:%SZ',
1758 '%Y-%m-%dT%H:%M:%S.%fZ',
1759 '%Y-%m-%dT%H:%M:%S.%f0Z',
1760 '%Y-%m-%dT%H:%M:%S',
1761 '%Y-%m-%dT%H:%M:%S.%f',
1763 '%b %d %Y at %H:%M',
1764 '%b %d %Y at %H:%M:%S',
1765 '%B %d %Y at %H:%M',
1766 '%B %d %Y at %H:%M:%S',
1770 DATE_FORMATS_DAY_FIRST
= list(DATE_FORMATS
)
1771 DATE_FORMATS_DAY_FIRST
.extend([
1777 '%d/%m/%Y %H:%M:%S',
1780 DATE_FORMATS_MONTH_FIRST
= list(DATE_FORMATS
)
1781 DATE_FORMATS_MONTH_FIRST
.extend([
1786 '%m/%d/%Y %H:%M:%S',
1789 PACKED_CODES_RE
= r
"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
1790 JSON_LD_RE
= r
'(?is)<script[^>]+type=(["\']?
)application
/ld\
+json\
1[^
>]*>(?P
<json_ld
>.+?
)</script
>'
1793 def preferredencoding():
1794 """Get preferred encoding.
1796 Returns the best encoding scheme for the system, based on
1797 locale.getpreferredencoding() and some further tweaks.
1800 pref = locale.getpreferredencoding()
1808 def write_json_file(obj, fn):
1809 """ Encode obj as JSON and write it to fn, atomically if possible """
1811 fn = encodeFilename(fn)
1812 if sys.version_info < (3, 0) and sys.platform != 'win32
':
1813 encoding = get_filesystem_encoding()
1814 # os.path.basename returns a bytes object, but NamedTemporaryFile
1815 # will fail if the filename contains non ascii characters unless we
1816 # use a unicode object
1817 path_basename = lambda f: os.path.basename(fn).decode(encoding)
1818 # the same for os.path.dirname
1819 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
1821 path_basename = os.path.basename
1822 path_dirname = os.path.dirname
1826 'prefix
': path_basename(fn) + '.',
1827 'dir': path_dirname(fn),
1831 # In Python 2.x, json.dump expects a bytestream.
1832 # In Python 3.x, it writes to a character stream
1833 if sys.version_info < (3, 0):
1838 'encoding
': 'utf
-8',
1841 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
1846 if sys.platform == 'win32
':
1847 # Need to remove existing file on Windows, else os.rename raises
1848 # WindowsError or FileExistsError.
1856 os.chmod(tf.name, 0o666 & ~mask)
1859 os.rename(tf.name, fn)
1868 if sys.version_info >= (2, 7):
1869 def find_xpath_attr(node, xpath, key, val=None):
1870 """ Find the xpath xpath[@key=val] """
1871 assert re.match(r'^
[a
-zA
-Z_
-]+$
', key)
1872 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
1873 return node.find(expr)
1875 def find_xpath_attr(node, xpath, key, val=None):
1876 for f in node.findall(compat_xpath(xpath)):
1877 if key not in f.attrib:
1879 if val is None or f.attrib.get(key) == val:
1883 # On python2.6 the xml.etree.ElementTree.Element methods don't support
1884 # the namespace parameter
1887 def xpath_with_ns(path
, ns_map
):
1888 components
= [c
.split(':') for c
in path
.split('/')]
1890 for c
in components
:
1892 replaced
.append(c
[0])
1895 replaced
.append('{%s}%s' % (ns_map
[ns
], tag
))
1896 return '/'.join(replaced
)
1899 def xpath_element(node
, xpath
, name
=None, fatal
=False, default
=NO_DEFAULT
):
1900 def _find_xpath(xpath
):
1901 return node
.find(compat_xpath(xpath
))
1903 if isinstance(xpath
, (str, compat_str
)):
1904 n
= _find_xpath(xpath
)
1912 if default
is not NO_DEFAULT
:
1915 name
= xpath
if name
is None else name
1916 raise ExtractorError('Could not find XML element %s' % name
)
1922 def xpath_text(node
, xpath
, name
=None, fatal
=False, default
=NO_DEFAULT
):
1923 n
= xpath_element(node
, xpath
, name
, fatal
=fatal
, default
=default
)
1924 if n
is None or n
== default
:
1927 if default
is not NO_DEFAULT
:
1930 name
= xpath
if name
is None else name
1931 raise ExtractorError('Could not find XML element\'s text %s' % name
)
1937 def xpath_attr(node
, xpath
, key
, name
=None, fatal
=False, default
=NO_DEFAULT
):
1938 n
= find_xpath_attr(node
, xpath
, key
)
1940 if default
is not NO_DEFAULT
:
1943 name
= '%s[@%s]' % (xpath
, key
) if name
is None else name
1944 raise ExtractorError('Could not find XML attribute %s' % name
)
1947 return n
.attrib
[key
]
1950 def get_element_by_id(id, html
):
1951 """Return the content of the tag with the specified ID in the passed HTML document"""
1952 return get_element_by_attribute('id', id, html
)
1955 def get_element_by_class(class_name
, html
):
1956 """Return the content of the first tag with the specified class in the passed HTML document"""
1957 retval
= get_elements_by_class(class_name
, html
)
1958 return retval
[0] if retval
else None
1961 def get_element_by_attribute(attribute
, value
, html
, escape_value
=True):
1962 retval
= get_elements_by_attribute(attribute
, value
, html
, escape_value
)
1963 return retval
[0] if retval
else None
1966 def get_elements_by_class(class_name
, html
):
1967 """Return the content of all tags with the specified class in the passed HTML document as a list"""
1968 return get_elements_by_attribute(
1969 'class', r
'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
1970 html, escape_value=False)
1973 def get_elements_by_attribute(attribute, value, html, escape_value=True):
1974 """Return the content of the tag with the specified attribute in the passed HTML document"""
1976 value = re.escape(value) if escape_value else value
1979 for m in re.finditer(r'''(?xs)
1981 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^
']*'|
))*?
1983 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^
"]*"|
='[^']*'|))*?
1987 ''' % (re.escape(attribute), value), html):
1988 res = m.group('content
')
1990 if res.startswith('"') or res.startswith("'"):
1993 retlist.append(unescapeHTML(res))
1998 class HTMLAttributeParser(compat_HTMLParser):
1999 """Trivial HTML parser to gather the attributes for a single element"""
2003 compat_HTMLParser.__init__(self)
2005 def handle_starttag(self, tag, attrs):
2006 self.attrs = dict(attrs)
2009 def extract_attributes(html_element):
2010 """Given a string for an HTML element such as
2012 a="foo" B="bar" c="&98;az" d=boz
2013 empty= noval entity="&"
2016 Decode and return a dictionary of attributes.
2018 'a
': 'foo
', 'b
': 'bar
', c: 'baz
', d: 'boz
',
2019 'empty
': '', 'noval
': None, 'entity
': '&',
2020 'sq
': '"', 'dq': '\''
2022 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
2023 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
2025 parser = HTMLAttributeParser()
2027 parser.feed(html_element)
2029 # Older Python may throw HTMLParseError in case of malformed HTML
2030 except compat_HTMLParseError:
2035 def clean_html(html):
2036 """Clean an HTML snippet into a readable string"""
2038 if html is None: # Convenience for sanitizing descriptions etc.
2042 html = html.replace('\n', ' ')
2043 html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
2044 html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
2046 html = re.sub('<.*?>', '', html)
2047 # Replace html entities
2048 html = unescapeHTML(html)
2052 def sanitize_open(filename, open_mode):
2053 """Try to open the given filename, and slightly tweak it if this fails.
2055 Attempts to open the given filename. If this fails, it tries to change
2056 the filename slightly, step by step, until it's either able to open it
2057 or it fails and raises a final exception, like the standard open()
2060 It returns the tuple (stream, definitive_file_name).
2064 if sys.platform == 'win32':
2066 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
2067 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
2068 stream = open(encodeFilename(filename), open_mode)
2069 return (stream, filename)
2070 except (IOError, OSError) as err:
2071 if err.errno in (errno.EACCES,):
2074 # In case of error, try to remove win32 forbidden chars
2075 alt_filename = sanitize_path(filename)
2076 if alt_filename == filename:
2079 # An exception here should be caught in the caller
2080 stream = open(encodeFilename(alt_filename), open_mode)
2081 return (stream, alt_filename)
2084 def timeconvert(timestr):
2085 """Convert RFC 2822 defined time string into system timestamp"""
2087 timetuple = email.utils.parsedate_tz(timestr)
2088 if timetuple is not None:
2089 timestamp = email.utils.mktime_tz(timetuple)
2093 def sanitize_filename(s, restricted=False, is_id=False):
2094 """Sanitizes a string so it could be used as part of a filename.
2095 If restricted is set, use a stricter subset of allowed characters.
2096 Set is_id if this is not an arbitrary string, but an ID that should be kept
2099 def replace_insane(char):
2100 if restricted and char in ACCENT_CHARS:
2101 return ACCENT_CHARS[char]
2102 elif not restricted and char == '\n':
2104 elif char == '?' or ord(char) < 32 or ord(char) == 127:
2107 return '' if restricted else '\''
2109 return '_
-' if restricted else ' -'
2110 elif char in '\\/|
*<>':
2112 if restricted and (char in '!&\'()[]{}$
;`^
,#' or char.isspace()):
2114 if restricted
and ord(char
) > 127:
2121 s
= re
.sub(r
'[0-9]+(?::[0-9]+)+', lambda m
: m
.group(0).replace(':', '_'), s
)
2122 result
= ''.join(map(replace_insane
, s
))
2124 while '__' in result
:
2125 result
= result
.replace('__', '_')
2126 result
= result
.strip('_')
2127 # Common case of "Foreign band name - English song title"
2128 if restricted
and result
.startswith('-_'):
2130 if result
.startswith('-'):
2131 result
= '_' + result
[len('-'):]
2132 result
= result
.lstrip('.')
2138 def sanitize_path(s
, force
=False):
2139 """Sanitizes and normalizes path on Windows"""
2140 if sys
.platform
== 'win32':
2142 drive_or_unc
, _
= os
.path
.splitdrive(s
)
2143 if sys
.version_info
< (2, 7) and not drive_or_unc
:
2144 drive_or_unc
, _
= os
.path
.splitunc(s
)
2150 norm_path
= os
.path
.normpath(remove_start(s
, drive_or_unc
)).split(os
.path
.sep
)
2154 path_part
if path_part
in ['.', '..'] else re
.sub(r
'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part
)
2155 for path_part
in norm_path
]
2157 sanitized_path
.insert(0, drive_or_unc
+ os
.path
.sep
)
2158 elif force
and s
[0] == os
.path
.sep
:
2159 sanitized_path
.insert(0, os
.path
.sep
)
2160 return os
.path
.join(*sanitized_path
)
2163 def sanitize_url(url
):
2164 # Prepend protocol-less URLs with `http:` scheme in order to mitigate
2165 # the number of unwanted failures due to missing protocol
2166 if url
.startswith('//'):
2167 return 'http:%s' % url
2168 # Fix some common typos seen so far
2170 # https://github.com/ytdl-org/youtube-dl/issues/15649
2171 (r
'^httpss://', r
'https://'),
2172 # https://bx1.be/lives/direct-tv/
2173 (r
'^rmtp([es]?)://', r
'rtmp\1://'),
2175 for mistake
, fixup
in COMMON_TYPOS
:
2176 if re
.match(mistake
, url
):
2177 return re
.sub(mistake
, fixup
, url
)
2181 def extract_basic_auth(url
):
2182 parts
= compat_urlparse
.urlsplit(url
)
2183 if parts
.username
is None:
2185 url
= compat_urlparse
.urlunsplit(parts
._replace
(netloc
=(
2186 parts
.hostname
if parts
.port
is None
2187 else '%s:%d' % (parts
.hostname
, parts
.port
))))
2188 auth_payload
= base64
.b64encode(
2189 ('%s:%s' % (parts
.username
, parts
.password
or '')).encode('utf-8'))
2190 return url
, 'Basic ' + auth_payload
.decode('utf-8')
2193 def sanitized_Request(url
, *args
, **kwargs
):
2194 url
, auth_header
= extract_basic_auth(escape_url(sanitize_url(url
)))
2195 if auth_header
is not None:
2196 headers
= args
[1] if len(args
) >= 2 else kwargs
.setdefault('headers', {})
2197 headers
['Authorization'] = auth_header
2198 return compat_urllib_request
.Request(url
, *args
, **kwargs
)
2202 """Expand shell variables and ~"""
2203 return os
.path
.expandvars(compat_expanduser(s
))
2206 def orderedSet(iterable
):
2207 """ Remove all duplicates from the input iterable """
2215 def _htmlentity_transform(entity_with_semicolon
):
2216 """Transforms an HTML entity to a character."""
2217 entity
= entity_with_semicolon
[:-1]
2219 # Known non-numeric HTML entity
2220 if entity
in compat_html_entities
.name2codepoint
:
2221 return compat_chr(compat_html_entities
.name2codepoint
[entity
])
2223 # TODO: HTML5 allows entities without a semicolon. For example,
2224 # 'Éric' should be decoded as 'Éric'.
2225 if entity_with_semicolon
in compat_html_entities_html5
:
2226 return compat_html_entities_html5
[entity_with_semicolon
]
2228 mobj
= re
.match(r
'#(x[0-9a-fA-F]+|[0-9]+)', entity
)
2229 if mobj
is not None:
2230 numstr
= mobj
.group(1)
2231 if numstr
.startswith('x'):
2233 numstr
= '0%s' % numstr
2236 # See https://github.com/ytdl-org/youtube-dl/issues/7518
2238 return compat_chr(int(numstr
, base
))
2242 # Unknown entity in name, return its literal representation
2243 return '&%s;' % entity
2246 def unescapeHTML(s
):
2249 assert type(s
) == compat_str
2252 r
'&([^&;]+;)', lambda m
: _htmlentity_transform(m
.group(1)), s
)
2255 def escapeHTML(text
):
2258 .replace('&', '&')
2259 .replace('<', '<')
2260 .replace('>', '>')
2261 .replace('"', '"')
2262 .replace("'", ''')
2266 def process_communicate_or_kill(p
, *args
, **kwargs
):
2268 return p
.communicate(*args
, **kwargs
)
2269 except BaseException
: # Including KeyboardInterrupt
2275 class Popen(subprocess
.Popen
):
2276 if sys
.platform
== 'win32':
2277 _startupinfo
= subprocess
.STARTUPINFO()
2278 _startupinfo
.dwFlags |
= subprocess
.STARTF_USESHOWWINDOW
2282 def __init__(self
, *args
, **kwargs
):
2283 super(Popen
, self
).__init
__(*args
, **kwargs
, startupinfo
=self
._startupinfo
)
2285 def communicate_or_kill(self
, *args
, **kwargs
):
2286 return process_communicate_or_kill(self
, *args
, **kwargs
)
2289 def get_subprocess_encoding():
2290 if sys
.platform
== 'win32' and sys
.getwindowsversion()[0] >= 5:
2291 # For subprocess calls, encode with locale encoding
2292 # Refer to http://stackoverflow.com/a/9951851/35070
2293 encoding
= preferredencoding()
2295 encoding
= sys
.getfilesystemencoding()
2296 if encoding
is None:
2301 def encodeFilename(s
, for_subprocess
=False):
2303 @param s The name of the file
2306 assert type(s
) == compat_str
2308 # Python 3 has a Unicode API
2309 if sys
.version_info
>= (3, 0):
2312 # Pass '' directly to use Unicode APIs on Windows 2000 and up
2313 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
2314 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
2315 if not for_subprocess
and sys
.platform
== 'win32' and sys
.getwindowsversion()[0] >= 5:
2318 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
2319 if sys
.platform
.startswith('java'):
2322 return s
.encode(get_subprocess_encoding(), 'ignore')
2325 def decodeFilename(b
, for_subprocess
=False):
2327 if sys
.version_info
>= (3, 0):
2330 if not isinstance(b
, bytes):
2333 return b
.decode(get_subprocess_encoding(), 'ignore')
2336 def encodeArgument(s
):
2337 if not isinstance(s
, compat_str
):
2338 # Legacy code that uses byte strings
2339 # Uncomment the following line after fixing all post processors
2340 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
2341 s
= s
.decode('ascii')
2342 return encodeFilename(s
, True)
2345 def decodeArgument(b
):
2346 return decodeFilename(b
, True)
2349 def decodeOption(optval
):
2352 if isinstance(optval
, bytes):
2353 optval
= optval
.decode(preferredencoding())
2355 assert isinstance(optval
, compat_str
)
2359 _timetuple
= collections
.namedtuple('Time', ('hours', 'minutes', 'seconds', 'milliseconds'))
2362 def timetuple_from_msec(msec
):
2363 secs
, msec
= divmod(msec
, 1000)
2364 mins
, secs
= divmod(secs
, 60)
2365 hrs
, mins
= divmod(mins
, 60)
2366 return _timetuple(hrs
, mins
, secs
, msec
)
2369 def formatSeconds(secs
, delim
=':', msec
=False):
2370 time
= timetuple_from_msec(secs
* 1000)
2372 ret
= '%d%s%02d%s%02d' % (time
.hours
, delim
, time
.minutes
, delim
, time
.seconds
)
2374 ret
= '%d%s%02d' % (time
.minutes
, delim
, time
.seconds
)
2376 ret
= '%d' % time
.seconds
2377 return '%s.%03d' % (ret
, time
.milliseconds
) if msec
else ret
2380 def _ssl_load_windows_store_certs(ssl_context
, storename
):
2381 # Code adapted from _load_windows_store_certs in https://github.com/python/cpython/blob/main/Lib/ssl.py
2383 certs
= [cert
for cert
, encoding
, trust
in ssl
.enum_certificates(storename
)
2384 if encoding
== 'x509_asn' and (
2385 trust
is True or ssl
.Purpose
.SERVER_AUTH
.oid
in trust
)]
2386 except PermissionError
:
2390 ssl_context
.load_verify_locations(cadata
=cert
)
2391 except ssl
.SSLError
:
2395 def make_HTTPS_handler(params
, **kwargs
):
2396 opts_check_certificate
= not params
.get('nocheckcertificate')
2397 context
= ssl
.SSLContext(ssl
.PROTOCOL_TLS_CLIENT
)
2398 context
.check_hostname
= opts_check_certificate
2399 context
.verify_mode
= ssl
.CERT_REQUIRED
if opts_check_certificate
else ssl
.CERT_NONE
2400 if opts_check_certificate
:
2402 context
.load_default_certs()
2403 # Work around the issue in load_default_certs when there are bad certificates. See:
2404 # https://github.com/yt-dlp/yt-dlp/issues/1060,
2405 # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
2406 except ssl
.SSLError
:
2407 # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
2408 if sys
.platform
== 'win32' and hasattr(ssl
, 'enum_certificates'):
2409 # Create a new context to discard any certificates that were already loaded
2410 context
= ssl
.SSLContext(ssl
.PROTOCOL_TLS_CLIENT
)
2411 context
.check_hostname
, context
.verify_mode
= True, ssl
.CERT_REQUIRED
2412 for storename
in ('CA', 'ROOT'):
2413 _ssl_load_windows_store_certs(context
, storename
)
2414 context
.set_default_verify_paths()
2415 return YoutubeDLHTTPSHandler(params
, context
=context
, **kwargs
)
2418 def bug_reports_message(before
=';'):
2419 if ytdl_is_updateable():
2420 update_cmd
= 'type yt-dlp -U to update'
2422 update_cmd
= 'see https://github.com/yt-dlp/yt-dlp on how to update'
2423 msg
= 'please report this issue on https://github.com/yt-dlp/yt-dlp .'
2424 msg
+= ' Make sure you are using the latest version; %s.' % update_cmd
2425 msg
+= ' Be sure to call yt-dlp with the --verbose flag and include its complete output.'
2427 before
= before
.rstrip()
2428 if not before
or before
.endswith(('.', '!', '?')):
2429 msg
= msg
[0].title() + msg
[1:]
2431 return (before
+ ' ' if before
else '') + msg
2434 class YoutubeDLError(Exception):
2435 """Base exception for YoutubeDL errors."""
2439 network_exceptions
= [compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
]
2440 if hasattr(ssl
, 'CertificateError'):
2441 network_exceptions
.append(ssl
.CertificateError
)
2442 network_exceptions
= tuple(network_exceptions
)
2445 class ExtractorError(YoutubeDLError
):
2446 """Error during info extraction."""
2448 def __init__(self
, msg
, tb
=None, expected
=False, cause
=None, video_id
=None, ie
=None):
2449 """ tb, if given, is the original traceback (so that it can be printed out).
2450 If expected is set, this is a normal error message and most likely not a bug in yt-dlp.
2452 if sys
.exc_info()[0] in network_exceptions
:
2457 self
.expected
= expected
2459 self
.video_id
= video_id
2461 self
.exc_info
= sys
.exc_info() # preserve original exception
2463 super(ExtractorError
, self
).__init
__(''.join((
2464 format_field(ie
, template
='[%s] '),
2465 format_field(video_id
, template
='%s: '),
2467 format_field(cause
, template
=' (caused by %r)'),
2468 '' if expected
else bug_reports_message())))
2470 def format_traceback(self
):
2471 if self
.traceback
is None:
2473 return ''.join(traceback
.format_tb(self
.traceback
))
2476 class UnsupportedError(ExtractorError
):
2477 def __init__(self
, url
):
2478 super(UnsupportedError
, self
).__init
__(
2479 'Unsupported URL: %s' % url
, expected
=True)
2483 class RegexNotFoundError(ExtractorError
):
2484 """Error when a regex didn't match"""
2488 class GeoRestrictedError(ExtractorError
):
2489 """Geographic restriction Error exception.
2491 This exception may be thrown when a video is not available from your
2492 geographic location due to geographic restrictions imposed by a website.
2495 def __init__(self
, msg
, countries
=None):
2496 super(GeoRestrictedError
, self
).__init
__(msg
, expected
=True)
2498 self
.countries
= countries
2501 class DownloadError(YoutubeDLError
):
2502 """Download Error exception.
2504 This exception may be thrown by FileDownloader objects if they are not
2505 configured to continue on errors. They will contain the appropriate
2509 def __init__(self
, msg
, exc_info
=None):
2510 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
2511 super(DownloadError
, self
).__init
__(msg
)
2512 self
.exc_info
= exc_info
2515 class EntryNotInPlaylist(YoutubeDLError
):
2516 """Entry not in playlist exception.
2518 This exception will be thrown by YoutubeDL when a requested entry
2519 is not found in the playlist info_dict
2524 class SameFileError(YoutubeDLError
):
2525 """Same File exception.
2527 This exception will be thrown by FileDownloader objects if they detect
2528 multiple files would have to be downloaded to the same file on disk.
2533 class PostProcessingError(YoutubeDLError
):
2534 """Post Processing exception.
2536 This exception may be raised by PostProcessor's .run() method to
2537 indicate an error in the postprocessing task.
2540 def __init__(self
, msg
):
2541 super(PostProcessingError
, self
).__init
__(msg
)
2545 class ExistingVideoReached(YoutubeDLError
):
2546 """ --max-downloads limit has been reached. """
2550 class RejectedVideoReached(YoutubeDLError
):
2551 """ --max-downloads limit has been reached. """
2555 class ThrottledDownload(YoutubeDLError
):
2556 """ Download speed below --throttled-rate. """
2560 class MaxDownloadsReached(YoutubeDLError
):
2561 """ --max-downloads limit has been reached. """
2565 class UnavailableVideoError(YoutubeDLError
):
2566 """Unavailable Format exception.
2568 This exception will be thrown when a video is requested
2569 in a format that is not available for that video.
2574 class ContentTooShortError(YoutubeDLError
):
2575 """Content Too Short exception.
2577 This exception may be raised by FileDownloader objects when a file they
2578 download is too small for what the server announced first, indicating
2579 the connection was probably interrupted.
2582 def __init__(self
, downloaded
, expected
):
2583 super(ContentTooShortError
, self
).__init
__(
2584 'Downloaded {0} bytes, expected {1} bytes'.format(downloaded
, expected
)
2587 self
.downloaded
= downloaded
2588 self
.expected
= expected
2591 class XAttrMetadataError(YoutubeDLError
):
2592 def __init__(self
, code
=None, msg
='Unknown error'):
2593 super(XAttrMetadataError
, self
).__init
__(msg
)
2597 # Parsing code and msg
2598 if (self
.code
in (errno
.ENOSPC
, errno
.EDQUOT
)
2599 or 'No space left' in self
.msg
or 'Disk quota exceeded' in self
.msg
):
2600 self
.reason
= 'NO_SPACE'
2601 elif self
.code
== errno
.E2BIG
or 'Argument list too long' in self
.msg
:
2602 self
.reason
= 'VALUE_TOO_LONG'
2604 self
.reason
= 'NOT_SUPPORTED'
2607 class XAttrUnavailableError(YoutubeDLError
):
2611 def _create_http_connection(ydl_handler
, http_class
, is_https
, *args
, **kwargs
):
2612 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
2613 # expected HTTP responses to meet HTTP/1.0 or later (see also
2614 # https://github.com/ytdl-org/youtube-dl/issues/6727)
2615 if sys
.version_info
< (3, 0):
2616 kwargs
['strict'] = True
2617 hc
= http_class(*args
, **compat_kwargs(kwargs
))
2618 source_address
= ydl_handler
._params
.get('source_address')
2620 if source_address
is not None:
2621 # This is to workaround _create_connection() from socket where it will try all
2622 # address data from getaddrinfo() including IPv6. This filters the result from
2623 # getaddrinfo() based on the source_address value.
2624 # This is based on the cpython socket.create_connection() function.
2625 # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
2626 def _create_connection(address
, timeout
=socket
._GLOBAL
_DEFAULT
_TIMEOUT
, source_address
=None):
2627 host
, port
= address
2629 addrs
= socket
.getaddrinfo(host
, port
, 0, socket
.SOCK_STREAM
)
2630 af
= socket
.AF_INET
if '.' in source_address
[0] else socket
.AF_INET6
2631 ip_addrs
= [addr
for addr
in addrs
if addr
[0] == af
]
2632 if addrs
and not ip_addrs
:
2633 ip_version
= 'v4' if af
== socket
.AF_INET
else 'v6'
2635 "No remote IP%s addresses available for connect, can't use '%s' as source address"
2636 % (ip_version
, source_address
[0]))
2637 for res
in ip_addrs
:
2638 af
, socktype
, proto
, canonname
, sa
= res
2641 sock
= socket
.socket(af
, socktype
, proto
)
2642 if timeout
is not socket
._GLOBAL
_DEFAULT
_TIMEOUT
:
2643 sock
.settimeout(timeout
)
2644 sock
.bind(source_address
)
2646 err
= None # Explicitly break reference cycle
2648 except socket
.error
as _
:
2650 if sock
is not None:
2655 raise socket
.error('getaddrinfo returns an empty list')
2656 if hasattr(hc
, '_create_connection'):
2657 hc
._create
_connection
= _create_connection
2658 sa
= (source_address
, 0)
2659 if hasattr(hc
, 'source_address'): # Python 2.7+
2660 hc
.source_address
= sa
2662 def _hc_connect(self
, *args
, **kwargs
):
2663 sock
= _create_connection(
2664 (self
.host
, self
.port
), self
.timeout
, sa
)
2666 self
.sock
= ssl
.wrap_socket(
2667 sock
, self
.key_file
, self
.cert_file
,
2668 ssl_version
=ssl
.PROTOCOL_TLSv1
)
2671 hc
.connect
= functools
.partial(_hc_connect
, hc
)
2676 def handle_youtubedl_headers(headers
):
2677 filtered_headers
= headers
2679 if 'Youtubedl-no-compression' in filtered_headers
:
2680 filtered_headers
= dict((k
, v
) for k
, v
in filtered_headers
.items() if k
.lower() != 'accept-encoding')
2681 del filtered_headers
['Youtubedl-no-compression']
2683 return filtered_headers
2686 class YoutubeDLHandler(compat_urllib_request
.HTTPHandler
):
2687 """Handler for HTTP requests and responses.
2689 This class, when installed with an OpenerDirector, automatically adds
2690 the standard headers to every HTTP request and handles gzipped and
2691 deflated responses from web servers. If compression is to be avoided in
2692 a particular request, the original request in the program code only has
2693 to include the HTTP header "Youtubedl-no-compression", which will be
2694 removed before making the real request.
2696 Part of this code was copied from:
2698 http://techknack.net/python-urllib2-handlers/
2700 Andrew Rowls, the author of that code, agreed to release it to the
2704 def __init__(self
, params
, *args
, **kwargs
):
2705 compat_urllib_request
.HTTPHandler
.__init
__(self
, *args
, **kwargs
)
2706 self
._params
= params
2708 def http_open(self
, req
):
2709 conn_class
= compat_http_client
.HTTPConnection
2711 socks_proxy
= req
.headers
.get('Ytdl-socks-proxy')
2713 conn_class
= make_socks_conn_class(conn_class
, socks_proxy
)
2714 del req
.headers
['Ytdl-socks-proxy']
2716 return self
.do_open(functools
.partial(
2717 _create_http_connection
, self
, conn_class
, False),
2725 return zlib
.decompress(data
, -zlib
.MAX_WBITS
)
2727 return zlib
.decompress(data
)
2729 def http_request(self
, req
):
2730 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
2731 # always respected by websites, some tend to give out URLs with non percent-encoded
2732 # non-ASCII characters (see telemb.py, ard.py [#3412])
2733 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
2734 # To work around aforementioned issue we will replace request's original URL with
2735 # percent-encoded one
2736 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
2737 # the code of this workaround has been moved here from YoutubeDL.urlopen()
2738 url
= req
.get_full_url()
2739 url_escaped
= escape_url(url
)
2741 # Substitute URL if any change after escaping
2742 if url
!= url_escaped
:
2743 req
= update_Request(req
, url
=url_escaped
)
2745 for h
, v
in std_headers
.items():
2746 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
2747 # The dict keys are capitalized because of this bug by urllib
2748 if h
.capitalize() not in req
.headers
:
2749 req
.add_header(h
, v
)
2751 req
.headers
= handle_youtubedl_headers(req
.headers
)
2753 if sys
.version_info
< (2, 7) and '#' in req
.get_full_url():
2754 # Python 2.6 is brain-dead when it comes to fragments
2755 req
._Request
__original
= req
._Request
__original
.partition('#')[0]
2756 req
._Request
__r
_type
= req
._Request
__r
_type
.partition('#')[0]
2760 def http_response(self
, req
, resp
):
2763 if resp
.headers
.get('Content-encoding', '') == 'gzip':
2764 content
= resp
.read()
2765 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
), mode
='rb')
2767 uncompressed
= io
.BytesIO(gz
.read())
2768 except IOError as original_ioerror
:
2769 # There may be junk add the end of the file
2770 # See http://stackoverflow.com/q/4928560/35070 for details
2771 for i
in range(1, 1024):
2773 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
[:-i
]), mode
='rb')
2774 uncompressed
= io
.BytesIO(gz
.read())
2779 raise original_ioerror
2780 resp
= compat_urllib_request
.addinfourl(uncompressed
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
2781 resp
.msg
= old_resp
.msg
2782 del resp
.headers
['Content-encoding']
2784 if resp
.headers
.get('Content-encoding', '') == 'deflate':
2785 gz
= io
.BytesIO(self
.deflate(resp
.read()))
2786 resp
= compat_urllib_request
.addinfourl(gz
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
2787 resp
.msg
= old_resp
.msg
2788 del resp
.headers
['Content-encoding']
2789 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
2790 # https://github.com/ytdl-org/youtube-dl/issues/6457).
2791 if 300 <= resp
.code
< 400:
2792 location
= resp
.headers
.get('Location')
2794 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
2795 if sys
.version_info
>= (3, 0):
2796 location
= location
.encode('iso-8859-1').decode('utf-8')
2798 location
= location
.decode('utf-8')
2799 location_escaped
= escape_url(location
)
2800 if location
!= location_escaped
:
2801 del resp
.headers
['Location']
2802 if sys
.version_info
< (3, 0):
2803 location_escaped
= location_escaped
.encode('utf-8')
2804 resp
.headers
['Location'] = location_escaped
2807 https_request
= http_request
2808 https_response
= http_response
2811 def make_socks_conn_class(base_class
, socks_proxy
):
2812 assert issubclass(base_class
, (
2813 compat_http_client
.HTTPConnection
, compat_http_client
.HTTPSConnection
))
2815 url_components
= compat_urlparse
.urlparse(socks_proxy
)
2816 if url_components
.scheme
.lower() == 'socks5':
2817 socks_type
= ProxyType
.SOCKS5
2818 elif url_components
.scheme
.lower() in ('socks', 'socks4'):
2819 socks_type
= ProxyType
.SOCKS4
2820 elif url_components
.scheme
.lower() == 'socks4a':
2821 socks_type
= ProxyType
.SOCKS4A
2823 def unquote_if_non_empty(s
):
2826 return compat_urllib_parse_unquote_plus(s
)
2830 url_components
.hostname
, url_components
.port
or 1080,
2832 unquote_if_non_empty(url_components
.username
),
2833 unquote_if_non_empty(url_components
.password
),
2836 class SocksConnection(base_class
):
2838 self
.sock
= sockssocket()
2839 self
.sock
.setproxy(*proxy_args
)
2840 if type(self
.timeout
) in (int, float):
2841 self
.sock
.settimeout(self
.timeout
)
2842 self
.sock
.connect((self
.host
, self
.port
))
2844 if isinstance(self
, compat_http_client
.HTTPSConnection
):
2845 if hasattr(self
, '_context'): # Python > 2.6
2846 self
.sock
= self
._context
.wrap_socket(
2847 self
.sock
, server_hostname
=self
.host
)
2849 self
.sock
= ssl
.wrap_socket(self
.sock
)
2851 return SocksConnection
2854 class YoutubeDLHTTPSHandler(compat_urllib_request
.HTTPSHandler
):
2855 def __init__(self
, params
, https_conn_class
=None, *args
, **kwargs
):
2856 compat_urllib_request
.HTTPSHandler
.__init
__(self
, *args
, **kwargs
)
2857 self
._https
_conn
_class
= https_conn_class
or compat_http_client
.HTTPSConnection
2858 self
._params
= params
2860 def https_open(self
, req
):
2862 conn_class
= self
._https
_conn
_class
2864 if hasattr(self
, '_context'): # python > 2.6
2865 kwargs
['context'] = self
._context
2866 if hasattr(self
, '_check_hostname'): # python 3.x
2867 kwargs
['check_hostname'] = self
._check
_hostname
2869 socks_proxy
= req
.headers
.get('Ytdl-socks-proxy')
2871 conn_class
= make_socks_conn_class(conn_class
, socks_proxy
)
2872 del req
.headers
['Ytdl-socks-proxy']
2874 return self
.do_open(functools
.partial(
2875 _create_http_connection
, self
, conn_class
, True),
2879 class YoutubeDLCookieJar(compat_cookiejar
.MozillaCookieJar
):
2881 See [1] for cookie file format.
2883 1. https://curl.haxx.se/docs/http-cookies.html
2885 _HTTPONLY_PREFIX
= '#HttpOnly_'
2887 _HEADER
= '''# Netscape HTTP Cookie File
2888 # This file is generated by yt-dlp. Do not edit.
2891 _CookieFileEntry
= collections
.namedtuple(
2893 ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))
2895 def save(self
, filename
=None, ignore_discard
=False, ignore_expires
=False):
2897 Save cookies to a file.
2899 Most of the code is taken from CPython 3.8 and slightly adapted
2900 to support cookie files with UTF-8 in both python 2 and 3.
2902 if filename
is None:
2903 if self
.filename
is not None:
2904 filename
= self
.filename
2906 raise ValueError(compat_cookiejar
.MISSING_FILENAME_TEXT
)
2908 # Store session cookies with `expires` set to 0 instead of an empty
2911 if cookie
.expires
is None:
2914 with io
.open(filename
, 'w', encoding
='utf-8') as f
:
2915 f
.write(self
._HEADER
)
2918 if not ignore_discard
and cookie
.discard
:
2920 if not ignore_expires
and cookie
.is_expired(now
):
2926 if cookie
.domain
.startswith('.'):
2927 initial_dot
= 'TRUE'
2929 initial_dot
= 'FALSE'
2930 if cookie
.expires
is not None:
2931 expires
= compat_str(cookie
.expires
)
2934 if cookie
.value
is None:
2935 # cookies.txt regards 'Set-Cookie: foo' as a cookie
2936 # with no name, whereas http.cookiejar regards it as a
2937 # cookie with no value.
2942 value
= cookie
.value
2944 '\t'.join([cookie
.domain
, initial_dot
, cookie
.path
,
2945 secure
, expires
, name
, value
]) + '\n')
2947 def load(self
, filename
=None, ignore_discard
=False, ignore_expires
=False):
2948 """Load cookies from a file."""
2949 if filename
is None:
2950 if self
.filename
is not None:
2951 filename
= self
.filename
2953 raise ValueError(compat_cookiejar
.MISSING_FILENAME_TEXT
)
2955 def prepare_line(line
):
2956 if line
.startswith(self
._HTTPONLY
_PREFIX
):
2957 line
= line
[len(self
._HTTPONLY
_PREFIX
):]
2958 # comments and empty lines are fine
2959 if line
.startswith('#') or not line
.strip():
2961 cookie_list
= line
.split('\t')
2962 if len(cookie_list
) != self
._ENTRY
_LEN
:
2963 raise compat_cookiejar
.LoadError('invalid length %d' % len(cookie_list
))
2964 cookie
= self
._CookieFileEntry
(*cookie_list
)
2965 if cookie
.expires_at
and not cookie
.expires_at
.isdigit():
2966 raise compat_cookiejar
.LoadError('invalid expires at %s' % cookie
.expires_at
)
2970 with io
.open(filename
, encoding
='utf-8') as f
:
2973 cf
.write(prepare_line(line
))
2974 except compat_cookiejar
.LoadError
as e
:
2976 'WARNING: skipping cookie file entry due to %s: %r\n'
2977 % (e
, line
), sys
.stderr
)
2980 self
._really
_load
(cf
, filename
, ignore_discard
, ignore_expires
)
2981 # Session cookies are denoted by either `expires` field set to
2982 # an empty string or 0. MozillaCookieJar only recognizes the former
2983 # (see [1]). So we need force the latter to be recognized as session
2984 # cookies on our own.
2985 # Session cookies may be important for cookies-based authentication,
2986 # e.g. usually, when user does not check 'Remember me' check box while
2987 # logging in on a site, some important cookies are stored as session
2988 # cookies so that not recognizing them will result in failed login.
2989 # 1. https://bugs.python.org/issue17164
2991 # Treat `expires=0` cookies as session cookies
2992 if cookie
.expires
== 0:
2993 cookie
.expires
= None
2994 cookie
.discard
= True
2997 class YoutubeDLCookieProcessor(compat_urllib_request
.HTTPCookieProcessor
):
2998 def __init__(self
, cookiejar
=None):
2999 compat_urllib_request
.HTTPCookieProcessor
.__init
__(self
, cookiejar
)
3001 def http_response(self
, request
, response
):
3002 # Python 2 will choke on next HTTP request in row if there are non-ASCII
3003 # characters in Set-Cookie HTTP header of last response (see
3004 # https://github.com/ytdl-org/youtube-dl/issues/6769).
3005 # In order to at least prevent crashing we will percent encode Set-Cookie
3006 # header before HTTPCookieProcessor starts processing it.
3007 # if sys.version_info < (3, 0) and response.headers:
3008 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
3009 # set_cookie = response.headers.get(set_cookie_header)
3011 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
3012 # if set_cookie != set_cookie_escaped:
3013 # del response.headers[set_cookie_header]
3014 # response.headers[set_cookie_header] = set_cookie_escaped
3015 return compat_urllib_request
.HTTPCookieProcessor
.http_response(self
, request
, response
)
3017 https_request
= compat_urllib_request
.HTTPCookieProcessor
.http_request
3018 https_response
= http_response
3021 class YoutubeDLRedirectHandler(compat_urllib_request
.HTTPRedirectHandler
):
3022 """YoutubeDL redirect handler
3024 The code is based on HTTPRedirectHandler implementation from CPython [1].
3026 This redirect handler solves two issues:
3027 - ensures redirect URL is always unicode under python 2
3028 - introduces support for experimental HTTP response status code
3029 308 Permanent Redirect [2] used by some sites [3]
3031 1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
3032 2. https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308
3033 3. https://github.com/ytdl-org/youtube-dl/issues/28768
3036 http_error_301
= http_error_303
= http_error_307
= http_error_308
= compat_urllib_request
.HTTPRedirectHandler
.http_error_302
3038 def redirect_request(self
, req
, fp
, code
, msg
, headers
, newurl
):
3039 """Return a Request or None in response to a redirect.
3041 This is called by the http_error_30x methods when a
3042 redirection response is received. If a redirection should
3043 take place, return a new Request to allow http_error_30x to
3044 perform the redirect. Otherwise, raise HTTPError if no-one
3045 else should try to handle this url. Return None if you can't
3046 but another Handler might.
3048 m
= req
.get_method()
3049 if (not (code
in (301, 302, 303, 307, 308) and m
in ("GET", "HEAD")
3050 or code
in (301, 302, 303) and m
== "POST")):
3051 raise compat_HTTPError(req
.full_url
, code
, msg
, headers
, fp
)
3052 # Strictly (according to RFC 2616), 301 or 302 in response to
3053 # a POST MUST NOT cause a redirection without confirmation
3054 # from the user (of urllib.request, in this case). In practice,
3055 # essentially all clients do redirect in this case, so we do
3058 # On python 2 urlh.geturl() may sometimes return redirect URL
3059 # as byte string instead of unicode. This workaround allows
3060 # to force it always return unicode.
3061 if sys
.version_info
[0] < 3:
3062 newurl
= compat_str(newurl
)
3064 # Be conciliant with URIs containing a space. This is mainly
3065 # redundant with the more complete encoding done in http_error_302(),
3066 # but it is kept for compatibility with other callers.
3067 newurl
= newurl
.replace(' ', '%20')
3069 CONTENT_HEADERS
= ("content-length", "content-type")
3070 # NB: don't use dict comprehension for python 2.6 compatibility
3071 newheaders
= dict((k
, v
) for k
, v
in req
.headers
.items()
3072 if k
.lower() not in CONTENT_HEADERS
)
3073 return compat_urllib_request
.Request(
3074 newurl
, headers
=newheaders
, origin_req_host
=req
.origin_req_host
,
3078 def extract_timezone(date_str
):
3081 ^.{8,}? # >=8 char non-TZ prefix, if present
3082 (?P<tz>Z| # just the UTC Z, or
3083 (?:(?<=.\b\d{4}|\b\d{2}:\d\d)| # preceded by 4 digits or hh:mm or
3084 (?<!.\b[a-zA-Z]{3}|[a-zA-Z]{4}|..\b\d\d)) # not preceded by 3 alpha word or >= 4 alpha or 2 digits
3085 [ ]? # optional space
3086 (?P<sign>\+|-) # +/-
3087 (?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2}) # hh[:]mm
3091 timezone
= datetime
.timedelta()
3093 date_str
= date_str
[:-len(m
.group('tz'))]
3094 if not m
.group('sign'):
3095 timezone
= datetime
.timedelta()
3097 sign
= 1 if m
.group('sign') == '+' else -1
3098 timezone
= datetime
.timedelta(
3099 hours
=sign
* int(m
.group('hours')),
3100 minutes
=sign
* int(m
.group('minutes')))
3101 return timezone
, date_str
3104 def parse_iso8601(date_str
, delimiter
='T', timezone
=None):
3105 """ Return a UNIX timestamp from the given date """
3107 if date_str
is None:
3110 date_str
= re
.sub(r
'\.[0-9]+', '', date_str
)
3112 if timezone
is None:
3113 timezone
, date_str
= extract_timezone(date_str
)
3116 date_format
= '%Y-%m-%d{0}%H:%M:%S'.format(delimiter
)
3117 dt
= datetime
.datetime
.strptime(date_str
, date_format
) - timezone
3118 return calendar
.timegm(dt
.timetuple())
3123 def date_formats(day_first
=True):
3124 return DATE_FORMATS_DAY_FIRST
if day_first
else DATE_FORMATS_MONTH_FIRST
3127 def unified_strdate(date_str
, day_first
=True):
3128 """Return a string with the date in the format YYYYMMDD"""
3130 if date_str
is None:
3134 date_str
= date_str
.replace(',', ' ')
3135 # Remove AM/PM + timezone
3136 date_str
= re
.sub(r
'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str
)
3137 _
, date_str
= extract_timezone(date_str
)
3139 for expression
in date_formats(day_first
):
3141 upload_date
= datetime
.datetime
.strptime(date_str
, expression
).strftime('%Y%m%d')
3144 if upload_date
is None:
3145 timetuple
= email
.utils
.parsedate_tz(date_str
)
3148 upload_date
= datetime
.datetime(*timetuple
[:6]).strftime('%Y%m%d')
3151 if upload_date
is not None:
3152 return compat_str(upload_date
)
3155 def unified_timestamp(date_str
, day_first
=True):
3156 if date_str
is None:
3159 date_str
= re
.sub(r
'[,|]', '', date_str
)
3161 pm_delta
= 12 if re
.search(r
'(?i)PM', date_str
) else 0
3162 timezone
, date_str
= extract_timezone(date_str
)
3164 # Remove AM/PM + timezone
3165 date_str
= re
.sub(r
'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str
)
3167 # Remove unrecognized timezones from ISO 8601 alike timestamps
3168 m
= re
.search(r
'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str
)
3170 date_str
= date_str
[:-len(m
.group('tz'))]
3172 # Python only supports microseconds, so remove nanoseconds
3173 m
= re
.search(r
'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str
)
3175 date_str
= m
.group(1)
3177 for expression
in date_formats(day_first
):
3179 dt
= datetime
.datetime
.strptime(date_str
, expression
) - timezone
+ datetime
.timedelta(hours
=pm_delta
)
3180 return calendar
.timegm(dt
.timetuple())
3183 timetuple
= email
.utils
.parsedate_tz(date_str
)
3185 return calendar
.timegm(timetuple
) + pm_delta
* 3600
3188 def determine_ext(url
, default_ext
='unknown_video'):
3189 if url
is None or '.' not in url
:
3191 guess
= url
.partition('?')[0].rpartition('.')[2]
3192 if re
.match(r
'^[A-Za-z0-9]+$', guess
):
3194 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
3195 elif guess
.rstrip('/') in KNOWN_EXTENSIONS
:
3196 return guess
.rstrip('/')
3201 def subtitles_filename(filename
, sub_lang
, sub_format
, expected_real_ext
=None):
3202 return replace_extension(filename
, sub_lang
+ '.' + sub_format
, expected_real_ext
)
3205 def datetime_from_str(date_str
, precision
='auto', format
='%Y%m%d'):
3207 Return a datetime object from a string in the format YYYYMMDD or
3208 (now|today|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)?
3210 format: string date format used to return datetime object from
3211 precision: round the time portion of a datetime object.
3212 auto|microsecond|second|minute|hour|day.
3213 auto: round to the unit provided in date_str (if applicable).
3215 auto_precision
= False
3216 if precision
== 'auto':
3217 auto_precision
= True
3218 precision
= 'microsecond'
3219 today
= datetime_round(datetime
.datetime
.now(), precision
)
3220 if date_str
in ('now', 'today'):
3222 if date_str
== 'yesterday':
3223 return today
- datetime
.timedelta(days
=1)
3225 r
'(?P<start>.+)(?P<sign>[+-])(?P<time>\d+)(?P<unit>microsecond|second|minute|hour|day|week|month|year)(s)?',
3227 if match
is not None:
3228 start_time
= datetime_from_str(match
.group('start'), precision
, format
)
3229 time
= int(match
.group('time')) * (-1 if match
.group('sign') == '-' else 1)
3230 unit
= match
.group('unit')
3231 if unit
== 'month' or unit
== 'year':
3232 new_date
= datetime_add_months(start_time
, time
* 12 if unit
== 'year' else time
)
3238 delta
= datetime
.timedelta(**{unit + 's': time}
)
3239 new_date
= start_time
+ delta
3241 return datetime_round(new_date
, unit
)
3244 return datetime_round(datetime
.datetime
.strptime(date_str
, format
), precision
)
3247 def date_from_str(date_str
, format
='%Y%m%d'):
3249 Return a datetime object from a string in the format YYYYMMDD or
3250 (now|today|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)?
3252 format: string date format used to return datetime object from
3254 return datetime_from_str(date_str
, precision
='microsecond', format
=format
).date()
3257 def datetime_add_months(dt
, months
):
3258 """Increment/Decrement a datetime object by months."""
3259 month
= dt
.month
+ months
- 1
3260 year
= dt
.year
+ month
// 12
3261 month
= month
% 12 + 1
3262 day
= min(dt
.day
, calendar
.monthrange(year
, month
)[1])
3263 return dt
.replace(year
, month
, day
)
3266 def datetime_round(dt
, precision
='day'):
3268 Round a datetime object's time to a specific precision
3270 if precision
== 'microsecond':
3279 roundto
= lambda x
, n
: ((x
+ n
/ 2) // n
) * n
3280 timestamp
= calendar
.timegm(dt
.timetuple())
3281 return datetime
.datetime
.utcfromtimestamp(roundto(timestamp
, unit_seconds
[precision
]))
3284 def hyphenate_date(date_str
):
3286 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
3287 match
= re
.match(r
'^(\d\d\d\d)(\d\d)(\d\d)$', date_str
)
3288 if match
is not None:
3289 return '-'.join(match
.groups())
3294 class DateRange(object):
3295 """Represents a time interval between two dates"""
3297 def __init__(self
, start
=None, end
=None):
3298 """start and end must be strings in the format accepted by date"""
3299 if start
is not None:
3300 self
.start
= date_from_str(start
)
3302 self
.start
= datetime
.datetime
.min.date()
3304 self
.end
= date_from_str(end
)
3306 self
.end
= datetime
.datetime
.max.date()
3307 if self
.start
> self
.end
:
3308 raise ValueError('Date range: "%s" , the start date must be before the end date' % self
)
3312 """Returns a range that only contains the given day"""
3313 return cls(day
, day
)
3315 def __contains__(self
, date
):
3316 """Check if the date is in the range"""
3317 if not isinstance(date
, datetime
.date
):
3318 date
= date_from_str(date
)
3319 return self
.start
<= date
<= self
.end
3322 return '%s - %s' % (self
.start
.isoformat(), self
.end
.isoformat())
3325 def platform_name():
3326 """ Returns the platform name as a compat_str """
3327 res
= platform
.platform()
3328 if isinstance(res
, bytes):
3329 res
= res
.decode(preferredencoding())
3331 assert isinstance(res
, compat_str
)
3335 def get_windows_version():
3336 ''' Get Windows version. None if it's not running on Windows '''
3337 if compat_os_name
== 'nt':
3338 return version_tuple(platform
.win32_ver()[1])
3343 def _windows_write_string(s
, out
):
3344 """ Returns True if the string was written using special methods,
3345 False if it has yet to be written out."""
3346 # Adapted from http://stackoverflow.com/a/3259271/35070
3349 import ctypes
.wintypes
3357 fileno
= out
.fileno()
3358 except AttributeError:
3359 # If the output stream doesn't have a fileno, it's virtual
3361 except io
.UnsupportedOperation
:
3362 # Some strange Windows pseudo files?
3364 if fileno
not in WIN_OUTPUT_IDS
:
3367 GetStdHandle
= compat_ctypes_WINFUNCTYPE(
3368 ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.DWORD
)(
3369 ('GetStdHandle', ctypes
.windll
.kernel32
))
3370 h
= GetStdHandle(WIN_OUTPUT_IDS
[fileno
])
3372 WriteConsoleW
= compat_ctypes_WINFUNCTYPE(
3373 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.LPWSTR
,
3374 ctypes
.wintypes
.DWORD
, ctypes
.POINTER(ctypes
.wintypes
.DWORD
),
3375 ctypes
.wintypes
.LPVOID
)(('WriteConsoleW', ctypes
.windll
.kernel32
))
3376 written
= ctypes
.wintypes
.DWORD(0)
3378 GetFileType
= compat_ctypes_WINFUNCTYPE(ctypes
.wintypes
.DWORD
, ctypes
.wintypes
.DWORD
)(('GetFileType', ctypes
.windll
.kernel32
))
3379 FILE_TYPE_CHAR
= 0x0002
3380 FILE_TYPE_REMOTE
= 0x8000
3381 GetConsoleMode
= compat_ctypes_WINFUNCTYPE(
3382 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
,
3383 ctypes
.POINTER(ctypes
.wintypes
.DWORD
))(
3384 ('GetConsoleMode', ctypes
.windll
.kernel32
))
3385 INVALID_HANDLE_VALUE
= ctypes
.wintypes
.DWORD(-1).value
3387 def not_a_console(handle
):
3388 if handle
== INVALID_HANDLE_VALUE
or handle
is None:
3390 return ((GetFileType(handle
) & ~FILE_TYPE_REMOTE
) != FILE_TYPE_CHAR
3391 or GetConsoleMode(handle
, ctypes
.byref(ctypes
.wintypes
.DWORD())) == 0)
3393 if not_a_console(h
):
3396 def next_nonbmp_pos(s
):
3398 return next(i
for i
, c
in enumerate(s
) if ord(c
) > 0xffff)
3399 except StopIteration:
3403 count
= min(next_nonbmp_pos(s
), 1024)
3405 ret
= WriteConsoleW(
3406 h
, s
, count
if count
else 2, ctypes
.byref(written
), None)
3408 raise OSError('Failed to write string')
3409 if not count
: # We just wrote a non-BMP character
3410 assert written
.value
== 2
3413 assert written
.value
> 0
3414 s
= s
[written
.value
:]
3418 def write_string(s
, out
=None, encoding
=None):
3421 assert type(s
) == compat_str
3423 if sys
.platform
== 'win32' and encoding
is None and hasattr(out
, 'fileno'):
3424 if _windows_write_string(s
, out
):
3427 if ('b' in getattr(out
, 'mode', '')
3428 or sys
.version_info
[0] < 3): # Python 2 lies about mode of sys.stderr
3429 byt
= s
.encode(encoding
or preferredencoding(), 'ignore')
3431 elif hasattr(out
, 'buffer'):
3432 enc
= encoding
or getattr(out
, 'encoding', None) or preferredencoding()
3433 byt
= s
.encode(enc
, 'ignore')
3434 out
.buffer.write(byt
)
3440 def bytes_to_intlist(bs
):
3443 if isinstance(bs
[0], int): # Python 3
3446 return [ord(c
) for c
in bs
]
3449 def intlist_to_bytes(xs
):
3452 return compat_struct_pack('%dB' % len(xs
), *xs
)
3455 # Cross-platform file locking
3456 if sys
.platform
== 'win32':
3457 import ctypes
.wintypes
3460 class OVERLAPPED(ctypes
.Structure
):
3462 ('Internal', ctypes
.wintypes
.LPVOID
),
3463 ('InternalHigh', ctypes
.wintypes
.LPVOID
),
3464 ('Offset', ctypes
.wintypes
.DWORD
),
3465 ('OffsetHigh', ctypes
.wintypes
.DWORD
),
3466 ('hEvent', ctypes
.wintypes
.HANDLE
),
3469 kernel32
= ctypes
.windll
.kernel32
3470 LockFileEx
= kernel32
.LockFileEx
3471 LockFileEx
.argtypes
= [
3472 ctypes
.wintypes
.HANDLE
, # hFile
3473 ctypes
.wintypes
.DWORD
, # dwFlags
3474 ctypes
.wintypes
.DWORD
, # dwReserved
3475 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
3476 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
3477 ctypes
.POINTER(OVERLAPPED
) # Overlapped
3479 LockFileEx
.restype
= ctypes
.wintypes
.BOOL
3480 UnlockFileEx
= kernel32
.UnlockFileEx
3481 UnlockFileEx
.argtypes
= [
3482 ctypes
.wintypes
.HANDLE
, # hFile
3483 ctypes
.wintypes
.DWORD
, # dwReserved
3484 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
3485 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
3486 ctypes
.POINTER(OVERLAPPED
) # Overlapped
3488 UnlockFileEx
.restype
= ctypes
.wintypes
.BOOL
3489 whole_low
= 0xffffffff
3490 whole_high
= 0x7fffffff
3492 def _lock_file(f
, exclusive
):
3493 overlapped
= OVERLAPPED()
3494 overlapped
.Offset
= 0
3495 overlapped
.OffsetHigh
= 0
3496 overlapped
.hEvent
= 0
3497 f
._lock
_file
_overlapped
_p
= ctypes
.pointer(overlapped
)
3498 handle
= msvcrt
.get_osfhandle(f
.fileno())
3499 if not LockFileEx(handle
, 0x2 if exclusive
else 0x0, 0,
3500 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
3501 raise OSError('Locking file failed: %r' % ctypes
.FormatError())
3503 def _unlock_file(f
):
3504 assert f
._lock
_file
_overlapped
_p
3505 handle
= msvcrt
.get_osfhandle(f
.fileno())
3506 if not UnlockFileEx(handle
, 0,
3507 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
3508 raise OSError('Unlocking file failed: %r' % ctypes
.FormatError())
3511 # Some platforms, such as Jython, is missing fcntl
3515 def _lock_file(f
, exclusive
):
3516 fcntl
.flock(f
, fcntl
.LOCK_EX
if exclusive
else fcntl
.LOCK_SH
)
3518 def _unlock_file(f
):
3519 fcntl
.flock(f
, fcntl
.LOCK_UN
)
3521 UNSUPPORTED_MSG
= 'file locking is not supported on this platform'
3523 def _lock_file(f
, exclusive
):
3524 raise IOError(UNSUPPORTED_MSG
)
3526 def _unlock_file(f
):
3527 raise IOError(UNSUPPORTED_MSG
)
3530 class locked_file(object):
3531 def __init__(self
, filename
, mode
, encoding
=None):
3532 assert mode
in ['r', 'a', 'w']
3533 self
.f
= io
.open(filename
, mode
, encoding
=encoding
)
3536 def __enter__(self
):
3537 exclusive
= self
.mode
!= 'r'
3539 _lock_file(self
.f
, exclusive
)
3545 def __exit__(self
, etype
, value
, traceback
):
3547 _unlock_file(self
.f
)
3554 def write(self
, *args
):
3555 return self
.f
.write(*args
)
3557 def read(self
, *args
):
3558 return self
.f
.read(*args
)
3561 def get_filesystem_encoding():
3562 encoding
= sys
.getfilesystemencoding()
3563 return encoding
if encoding
is not None else 'utf-8'
3566 def shell_quote(args
):
3568 encoding
= get_filesystem_encoding()
3570 if isinstance(a
, bytes):
3571 # We may get a filename encoded with 'encodeFilename'
3572 a
= a
.decode(encoding
)
3573 quoted_args
.append(compat_shlex_quote(a
))
3574 return ' '.join(quoted_args
)
3577 def smuggle_url(url
, data
):
3578 """ Pass additional data in a URL for internal use. """
3580 url
, idata
= unsmuggle_url(url
, {})
3582 sdata
= compat_urllib_parse_urlencode(
3583 {'__youtubedl_smuggle': json.dumps(data)}
)
3584 return url
+ '#' + sdata
3587 def unsmuggle_url(smug_url
, default
=None):
3588 if '#__youtubedl_smuggle' not in smug_url
:
3589 return smug_url
, default
3590 url
, _
, sdata
= smug_url
.rpartition('#')
3591 jsond
= compat_parse_qs(sdata
)['__youtubedl_smuggle'][0]
3592 data
= json
.loads(jsond
)
3596 def format_bytes(bytes):
3599 if type(bytes) is str:
3600 bytes = float(bytes)
3604 exponent
= int(math
.log(bytes, 1024.0))
3605 suffix
= ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent
]
3606 converted
= float(bytes) / float(1024 ** exponent
)
3607 return '%.2f%s' % (converted
, suffix
)
3610 def lookup_unit_table(unit_table
, s
):
3611 units_re
= '|'.join(re
.escape(u
) for u
in unit_table
)
3613 r
'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re
, s
)
3616 num_str
= m
.group('num').replace(',', '.')
3617 mult
= unit_table
[m
.group('unit')]
3618 return int(float(num_str
) * mult
)
3621 def parse_filesize(s
):
3625 # The lower-case forms are of course incorrect and unofficial,
3626 # but we support those too
3643 'megabytes': 1000 ** 2,
3644 'mebibytes': 1024 ** 2,
3650 'gigabytes': 1000 ** 3,
3651 'gibibytes': 1024 ** 3,
3657 'terabytes': 1000 ** 4,
3658 'tebibytes': 1024 ** 4,
3664 'petabytes': 1000 ** 5,
3665 'pebibytes': 1024 ** 5,
3671 'exabytes': 1000 ** 6,
3672 'exbibytes': 1024 ** 6,
3678 'zettabytes': 1000 ** 7,
3679 'zebibytes': 1024 ** 7,
3685 'yottabytes': 1000 ** 8,
3686 'yobibytes': 1024 ** 8,
3689 return lookup_unit_table(_UNIT_TABLE
, s
)
3698 if re
.match(r
'^[\d,.]+$', s
):
3699 return str_to_int(s
)
3710 return lookup_unit_table(_UNIT_TABLE
, s
)
3713 def parse_resolution(s
):
3717 mobj
= re
.search(r
'(?<![a-zA-Z0-9])(?P<w>\d+)\s*[xX×,]\s*(?P<h>\d+)(?![a-zA-Z0-9])', s
)
3720 'width': int(mobj
.group('w')),
3721 'height': int(mobj
.group('h')),
3724 mobj
= re
.search(r
'(?<![a-zA-Z0-9])(\d+)[pPiI](?![a-zA-Z0-9])', s
)
3726 return {'height': int(mobj.group(1))}
3728 mobj
= re
.search(r
'\b([48])[kK]\b', s
)
3730 return {'height': int(mobj.group(1)) * 540}
3735 def parse_bitrate(s
):
3736 if not isinstance(s
, compat_str
):
3738 mobj
= re
.search(r
'\b(\d+)\s*kbps', s
)
3740 return int(mobj
.group(1))
3743 def month_by_name(name
, lang
='en'):
3744 """ Return the number of a month by (locale-independently) English name """
3746 month_names
= MONTH_NAMES
.get(lang
, MONTH_NAMES
['en'])
3749 return month_names
.index(name
) + 1
3754 def month_by_abbreviation(abbrev
):
3755 """ Return the number of a month by (locale-independently) English
3759 return [s
[:3] for s
in ENGLISH_MONTH_NAMES
].index(abbrev
) + 1
3764 def fix_xml_ampersands(xml_str
):
3765 """Replace all the '&' by '&' in XML"""
3767 r
'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
3772 def setproctitle(title
):
3773 assert isinstance(title
, compat_str
)
3775 # ctypes in Jython is not complete
3776 # http://bugs.jython.org/issue2148
3777 if sys
.platform
.startswith('java'):
3781 libc
= ctypes
.cdll
.LoadLibrary('libc.so.6')
3785 # LoadLibrary in Windows Python 2.7.13 only expects
3786 # a bytestring, but since unicode_literals turns
3787 # every string into a unicode string, it fails.
3789 title_bytes
= title
.encode('utf-8')
3790 buf
= ctypes
.create_string_buffer(len(title_bytes
))
3791 buf
.value
= title_bytes
3793 libc
.prctl(15, buf
, 0, 0, 0)
3794 except AttributeError:
3795 return # Strange libc, just skip this
3798 def remove_start(s
, start
):
3799 return s
[len(start
):] if s
is not None and s
.startswith(start
) else s
3802 def remove_end(s
, end
):
3803 return s
[:-len(end
)] if s
is not None and s
.endswith(end
) else s
3806 def remove_quotes(s
):
3807 if s
is None or len(s
) < 2:
3809 for quote
in ('"', "'", ):
3810 if s
[0] == quote
and s
[-1] == quote
:
3815 def get_domain(url
):
3816 domain
= re
.match(r
'(?:https?:\/\/)?(?:www\.)?(?P<domain>[^\n\/]+\.[^\n\/]+)(?:\/(.*))?', url
)
3817 return domain
.group('domain') if domain
else None
3820 def url_basename(url
):
3821 path
= compat_urlparse
.urlparse(url
).path
3822 return path
.strip('/').split('/')[-1]
3826 return re
.match(r
'https?://[^?#&]+/', url
).group()
3829 def urljoin(base
, path
):
3830 if isinstance(path
, bytes):
3831 path
= path
.decode('utf-8')
3832 if not isinstance(path
, compat_str
) or not path
:
3834 if re
.match(r
'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path
):
3836 if isinstance(base
, bytes):
3837 base
= base
.decode('utf-8')
3838 if not isinstance(base
, compat_str
) or not re
.match(
3839 r
'^(?:https?:)?//', base
):
3841 return compat_urlparse
.urljoin(base
, path
)
3844 class HEADRequest(compat_urllib_request
.Request
):
3845 def get_method(self
):
3849 class PUTRequest(compat_urllib_request
.Request
):
3850 def get_method(self
):
3854 def int_or_none(v
, scale
=1, default
=None, get_attr
=None, invscale
=1):
3857 v
= getattr(v
, get_attr
, None)
3863 return int(v
) * invscale
// scale
3864 except (ValueError, TypeError):
3868 def str_or_none(v
, default
=None):
3869 return default
if v
is None else compat_str(v
)
3872 def str_to_int(int_str
):
3873 """ A more relaxed version of int_or_none """
3874 if isinstance(int_str
, compat_integer_types
):
3876 elif isinstance(int_str
, compat_str
):
3877 int_str
= re
.sub(r
'[,\.\+]', '', int_str
)
3878 return int_or_none(int_str
)
3881 def float_or_none(v
, scale
=1, invscale
=1, default
=None):
3885 return float(v
) * invscale
/ scale
3886 except (ValueError, TypeError):
3890 def bool_or_none(v
, default
=None):
3891 return v
if isinstance(v
, bool) else default
3894 def strip_or_none(v
, default
=None):
3895 return v
.strip() if isinstance(v
, compat_str
) else default
3898 def url_or_none(url
):
3899 if not url
or not isinstance(url
, compat_str
):
3902 return url
if re
.match(r
'^(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url
) else None
3905 def strftime_or_none(timestamp
, date_format
, default
=None):
3906 datetime_object
= None
3908 if isinstance(timestamp
, compat_numeric_types
): # unix timestamp
3909 datetime_object
= datetime
.datetime
.utcfromtimestamp(timestamp
)
3910 elif isinstance(timestamp
, compat_str
): # assume YYYYMMDD
3911 datetime_object
= datetime
.datetime
.strptime(timestamp
, '%Y%m%d')
3912 return datetime_object
.strftime(date_format
)
3913 except (ValueError, TypeError, AttributeError):
3917 def parse_duration(s
):
3918 if not isinstance(s
, compat_basestring
):
3923 days
, hours
, mins
, secs
, ms
= [None] * 5
3924 m
= re
.match(r
'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s
)
3926 days
, hours
, mins
, secs
, ms
= m
.groups()
3931 [0-9]+\s*y(?:ears?)?\s*
3934 [0-9]+\s*m(?:onths?)?\s*
3937 [0-9]+\s*w(?:eeks?)?\s*
3940 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
3944 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
3947 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
3950 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
3953 days
, hours
, mins
, secs
, ms
= m
.groups()
3955 m
= re
.match(r
'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s
)
3957 hours
, mins
= m
.groups()
3963 duration
+= float(secs
)
3965 duration
+= float(mins
) * 60
3967 duration
+= float(hours
) * 60 * 60
3969 duration
+= float(days
) * 24 * 60 * 60
3971 duration
+= float(ms
)
3975 def prepend_extension(filename
, ext
, expected_real_ext
=None):
3976 name
, real_ext
= os
.path
.splitext(filename
)
3978 '{0}.{1}{2}'.format(name
, ext
, real_ext
)
3979 if not expected_real_ext
or real_ext
[1:] == expected_real_ext
3980 else '{0}.{1}'.format(filename
, ext
))
3983 def replace_extension(filename
, ext
, expected_real_ext
=None):
3984 name
, real_ext
= os
.path
.splitext(filename
)
3985 return '{0}.{1}'.format(
3986 name
if not expected_real_ext
or real_ext
[1:] == expected_real_ext
else filename
,
3990 def check_executable(exe
, args
=[]):
3991 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
3992 args can be a list of arguments for a short output (like -version) """
3994 Popen([exe
] + args
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
).communicate_or_kill()
4000 def get_exe_version(exe
, args
=['--version'],
4001 version_re
=None, unrecognized
='present'):
4002 """ Returns the version of the specified executable,
4003 or False if the executable is not present """
4005 # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
4006 # SIGTTOU if yt-dlp is run in the background.
4007 # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
4009 [encodeArgument(exe
)] + args
, stdin
=subprocess
.PIPE
,
4010 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
).communicate_or_kill()
4013 if isinstance(out
, bytes): # Python 2.x
4014 out
= out
.decode('ascii', 'ignore')
4015 return detect_exe_version(out
, version_re
, unrecognized
)
4018 def detect_exe_version(output
, version_re
=None, unrecognized
='present'):
4019 assert isinstance(output
, compat_str
)
4020 if version_re
is None:
4021 version_re
= r
'version\s+([-0-9._a-zA-Z]+)'
4022 m
= re
.search(version_re
, output
)
4029 class LazyList(collections
.abc
.Sequence
):
4030 ''' Lazy immutable list from an iterable
4031 Note that slices of a LazyList are lists and not LazyList'''
4033 class IndexError(IndexError):
4036 def __init__(self
, iterable
):
4037 self
.__iterable
= iter(iterable
)
4039 self
.__reversed
= False
4043 # We need to consume the entire iterable to iterate in reverse
4044 yield from self
.exhaust()
4046 yield from self
.__cache
4047 for item
in self
.__iterable
:
4048 self
.__cache
.append(item
)
4051 def __exhaust(self
):
4052 self
.__cache
.extend(self
.__iterable
)
4056 ''' Evaluate the entire iterable '''
4057 return self
.__exhaust
()[::-1 if self
.__reversed
else 1]
4060 def __reverse_index(x
):
4061 return None if x
is None else -(x
+ 1)
4063 def __getitem__(self
, idx
):
4064 if isinstance(idx
, slice):
4066 idx
= slice(self
.__reverse
_index
(idx
.start
), self
.__reverse
_index
(idx
.stop
), -(idx
.step
or 1))
4067 start
, stop
, step
= idx
.start
, idx
.stop
, idx
.step
or 1
4068 elif isinstance(idx
, int):
4070 idx
= self
.__reverse
_index
(idx
)
4071 start
, stop
, step
= idx
, idx
, 0
4073 raise TypeError('indices must be integers or slices')
4074 if ((start
or 0) < 0 or (stop
or 0) < 0
4075 or (start
is None and step
< 0)
4076 or (stop
is None and step
> 0)):
4077 # We need to consume the entire iterable to be able to slice from the end
4078 # Obviously, never use this with infinite iterables
4081 return self
.__cache
[idx
]
4082 except IndexError as e
:
4083 raise self
.IndexError(e
) from e
4084 n
= max(start
or 0, stop
or 0) - len(self
.__cache
) + 1
4086 self
.__cache
.extend(itertools
.islice(self
.__iterable
, n
))
4088 return self
.__cache
[idx
]
4089 except IndexError as e
:
4090 raise self
.IndexError(e
) from e
4094 self
[-1] if self
.__reversed
else self
[0]
4095 except self
.IndexError:
4101 return len(self
.__cache
)
4104 self
.__reversed
= not self
.__reversed
4108 # repr and str should mimic a list. So we exhaust the iterable
4109 return repr(self
.exhaust())
4112 return repr(self
.exhaust())
4117 # This is only useful for tests
4118 return len(self
.getslice())
4120 def __init__(self
, pagefunc
, pagesize
, use_cache
=True):
4121 self
._pagefunc
= pagefunc
4122 self
._pagesize
= pagesize
4123 self
._use
_cache
= use_cache
4126 def getpage(self
, pagenum
):
4127 page_results
= self
._cache
.get(pagenum
) or list(self
._pagefunc
(pagenum
))
4129 self
._cache
[pagenum
] = page_results
4132 def getslice(self
, start
=0, end
=None):
4133 return list(self
._getslice
(start
, end
))
4135 def _getslice(self
, start
, end
):
4136 raise NotImplementedError('This method must be implemented by subclasses')
4138 def __getitem__(self
, idx
):
4139 # NOTE: cache must be enabled if this is used
4140 if not isinstance(idx
, int) or idx
< 0:
4141 raise TypeError('indices must be non-negative integers')
4142 entries
= self
.getslice(idx
, idx
+ 1)
4143 return entries
[0] if entries
else None
4146 class OnDemandPagedList(PagedList
):
4147 def _getslice(self
, start
, end
):
4148 for pagenum
in itertools
.count(start
// self
._pagesize
):
4149 firstid
= pagenum
* self
._pagesize
4150 nextfirstid
= pagenum
* self
._pagesize
+ self
._pagesize
4151 if start
>= nextfirstid
:
4155 start
% self
._pagesize
4156 if firstid
<= start
< nextfirstid
4159 ((end
- 1) % self
._pagesize
) + 1
4160 if (end
is not None and firstid
<= end
<= nextfirstid
)
4163 page_results
= self
.getpage(pagenum
)
4164 if startv
!= 0 or endv
is not None:
4165 page_results
= page_results
[startv
:endv
]
4166 yield from page_results
4168 # A little optimization - if current page is not "full", ie. does
4169 # not contain page_size videos then we can assume that this page
4170 # is the last one - there are no more ids on further pages -
4171 # i.e. no need to query again.
4172 if len(page_results
) + startv
< self
._pagesize
:
4175 # If we got the whole page, but the next page is not interesting,
4176 # break out early as well
4177 if end
== nextfirstid
:
4181 class InAdvancePagedList(PagedList
):
4182 def __init__(self
, pagefunc
, pagecount
, pagesize
):
4183 self
._pagecount
= pagecount
4184 PagedList
.__init
__(self
, pagefunc
, pagesize
, True)
4186 def _getslice(self
, start
, end
):
4187 start_page
= start
// self
._pagesize
4189 self
._pagecount
if end
is None else (end
// self
._pagesize
+ 1))
4190 skip_elems
= start
- start_page
* self
._pagesize
4191 only_more
= None if end
is None else end
- start
4192 for pagenum
in range(start_page
, end_page
):
4193 page_results
= self
.getpage(pagenum
)
4195 page_results
= page_results
[skip_elems
:]
4197 if only_more
is not None:
4198 if len(page_results
) < only_more
:
4199 only_more
-= len(page_results
)
4201 yield from page_results
[:only_more
]
4203 yield from page_results
4206 def uppercase_escape(s
):
4207 unicode_escape
= codecs
.getdecoder('unicode_escape')
4209 r
'\\U[0-9a-fA-F]{8}',
4210 lambda m
: unicode_escape(m
.group(0))[0],
4214 def lowercase_escape(s
):
4215 unicode_escape
= codecs
.getdecoder('unicode_escape')
4217 r
'\\u[0-9a-fA-F]{4}',
4218 lambda m
: unicode_escape(m
.group(0))[0],
4222 def escape_rfc3986(s
):
4223 """Escape non-ASCII characters as suggested by RFC 3986"""
4224 if sys
.version_info
< (3, 0) and isinstance(s
, compat_str
):
4225 s
= s
.encode('utf-8')
4226 return compat_urllib_parse
.quote(s
, b
"%/;:@&=+$,!~*'()?#[]")
4229 def escape_url(url
):
4230 """Escape URL as suggested by RFC 3986"""
4231 url_parsed
= compat_urllib_parse_urlparse(url
)
4232 return url_parsed
._replace
(
4233 netloc
=url_parsed
.netloc
.encode('idna').decode('ascii'),
4234 path
=escape_rfc3986(url_parsed
.path
),
4235 params
=escape_rfc3986(url_parsed
.params
),
4236 query
=escape_rfc3986(url_parsed
.query
),
4237 fragment
=escape_rfc3986(url_parsed
.fragment
)
4242 return compat_parse_qs(compat_urllib_parse_urlparse(url
).query
)
4245 def read_batch_urls(batch_fd
):
4247 if not isinstance(url
, compat_str
):
4248 url
= url
.decode('utf-8', 'replace')
4249 BOM_UTF8
= ('\xef\xbb\xbf', '\ufeff')
4250 for bom
in BOM_UTF8
:
4251 if url
.startswith(bom
):
4252 url
= url
[len(bom
):]
4254 if not url
or url
.startswith(('#', ';', ']')):
4256 # "#" cannot be stripped out since it is part of the URI
4257 # However, it can be safely stipped out if follwing a whitespace
4258 return re
.split(r
'\s#', url
, 1)[0].rstrip()
4260 with contextlib
.closing(batch_fd
) as fd
:
4261 return [url
for url
in map(fixup
, fd
) if url
]
4264 def urlencode_postdata(*args
, **kargs
):
4265 return compat_urllib_parse_urlencode(*args
, **kargs
).encode('ascii')
4268 def update_url_query(url
, query
):
4271 parsed_url
= compat_urlparse
.urlparse(url
)
4272 qs
= compat_parse_qs(parsed_url
.query
)
4274 return compat_urlparse
.urlunparse(parsed_url
._replace
(
4275 query
=compat_urllib_parse_urlencode(qs
, True)))
4278 def update_Request(req
, url
=None, data
=None, headers
={}, query={}
):
4279 req_headers
= req
.headers
.copy()
4280 req_headers
.update(headers
)
4281 req_data
= data
or req
.data
4282 req_url
= update_url_query(url
or req
.get_full_url(), query
)
4283 req_get_method
= req
.get_method()
4284 if req_get_method
== 'HEAD':
4285 req_type
= HEADRequest
4286 elif req_get_method
== 'PUT':
4287 req_type
= PUTRequest
4289 req_type
= compat_urllib_request
.Request
4291 req_url
, data
=req_data
, headers
=req_headers
,
4292 origin_req_host
=req
.origin_req_host
, unverifiable
=req
.unverifiable
)
4293 if hasattr(req
, 'timeout'):
4294 new_req
.timeout
= req
.timeout
4298 def _multipart_encode_impl(data
, boundary
):
4299 content_type
= 'multipart/form-data; boundary=%s' % boundary
4302 for k
, v
in data
.items():
4303 out
+= b
'--' + boundary
.encode('ascii') + b
'\r\n'
4304 if isinstance(k
, compat_str
):
4305 k
= k
.encode('utf-8')
4306 if isinstance(v
, compat_str
):
4307 v
= v
.encode('utf-8')
4308 # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
4309 # suggests sending UTF-8 directly. Firefox sends UTF-8, too
4310 content
= b
'Content-Disposition: form-data; name="' + k
+ b
'"\r\n\r\n' + v
+ b
'\r\n'
4311 if boundary
.encode('ascii') in content
:
4312 raise ValueError('Boundary overlaps with data')
4315 out
+= b
'--' + boundary
.encode('ascii') + b
'--\r\n'
4317 return out
, content_type
4320 def multipart_encode(data
, boundary
=None):
4322 Encode a dict to RFC 7578-compliant form-data
4325 A dict where keys and values can be either Unicode or bytes-like
4328 If specified a Unicode object, it's used as the boundary. Otherwise
4329 a random boundary is generated.
4331 Reference: https://tools.ietf.org/html/rfc7578
4333 has_specified_boundary
= boundary
is not None
4336 if boundary
is None:
4337 boundary
= '---------------' + str(random
.randrange(0x0fffffff, 0xffffffff))
4340 out
, content_type
= _multipart_encode_impl(data
, boundary
)
4343 if has_specified_boundary
:
4347 return out
, content_type
4350 def dict_get(d
, key_or_keys
, default
=None, skip_false_values
=True):
4351 if isinstance(key_or_keys
, (list, tuple)):
4352 for key
in key_or_keys
:
4353 if key
not in d
or d
[key
] is None or skip_false_values
and not d
[key
]:
4357 return d
.get(key_or_keys
, default
)
4360 def try_get(src
, getter
, expected_type
=None):
4361 for get
in variadic(getter
):
4364 except (AttributeError, KeyError, TypeError, IndexError):
4367 if expected_type
is None or isinstance(v
, expected_type
):
4371 def merge_dicts(*dicts
):
4373 for a_dict
in dicts
:
4374 for k
, v
in a_dict
.items():
4378 or (isinstance(v
, compat_str
) and v
4379 and isinstance(merged
[k
], compat_str
)
4380 and not merged
[k
])):
4385 def encode_compat_str(string
, encoding
=preferredencoding(), errors
='strict'):
4386 return string
if isinstance(string
, compat_str
) else compat_str(string
, encoding
, errors
)
4398 TV_PARENTAL_GUIDELINES
= {
4408 def parse_age_limit(s
):
4410 return s
if 0 <= s
<= 21 else None
4411 if not isinstance(s
, compat_basestring
):
4413 m
= re
.match(r
'^(?P<age>\d{1,2})\+?$', s
)
4415 return int(m
.group('age'))
4418 return US_RATINGS
[s
]
4419 m
= re
.match(r
'^TV[_-]?(%s)$' % '|'.join(k
[3:] for k
in TV_PARENTAL_GUIDELINES
), s
)
4421 return TV_PARENTAL_GUIDELINES
['TV-' + m
.group(1)]
4425 def strip_jsonp(code
):
4428 (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
4429 (?:\s*&&\s*(?P=func_name))?
4430 \s*\(\s*(?P<callback_data>.*)\);?
4431 \s*?(?://[^\n]*)*$''',
4432 r
'\g<callback_data>', code
)
4435 def js_to_json(code
, vars={}):
4436 # vars is a dict of var, val pairs to substitute
4437 COMMENT_RE
= r
'/\*(?:(?!\*/).)*?\*/|//[^\n]*\n'
4438 SKIP_RE
= r
'\s*(?:{comment})?\s*'.format(comment
=COMMENT_RE
)
4440 (r
'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip
=SKIP_RE
), 16),
4441 (r
'(?s)^(0+[0-7]+){skip}:?$'.format(skip
=SKIP_RE
), 8),
4446 if v
in ('true', 'false', 'null'):
4448 elif v
in ('undefined', 'void 0'):
4450 elif v
.startswith('/*') or v
.startswith('//') or v
.startswith('!') or v
== ',':
4453 if v
[0] in ("'", '"'):
4454 v
= re
.sub(r
'(?s)\\.|"', lambda m
: {
4459 }.get(m
.group(0), m
.group(0)), v
[1:-1])
4461 for regex
, base
in INTEGER_TABLE
:
4462 im
= re
.match(regex
, v
)
4464 i
= int(im
.group(1), base
)
4465 return '"%d":' % i
if v
.endswith(':') else '%d' % i
4472 return re
.sub(r
'''(?sx)
4473 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
4474 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
4475 {comment}|,(?={skip}[\]}}])|
4476 void\s0|(?:(?<![0-9])[eE]|[a-df-zA-DF-Z_$])[.a-zA-Z_$0-9]*|
4477 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
4480 '''.format(comment
=COMMENT_RE
, skip
=SKIP_RE
), fix_kv
, code
)
4483 def qualities(quality_ids
):
4484 """ Get a numeric quality value out of a list of possible values """
4487 return quality_ids
.index(qid
)
4494 'default': '%(title)s [%(id)s].%(ext)s',
4495 'chapter': '%(title)s - %(section_number)03d %(section_title)s [%(id)s].%(ext)s',
4501 'description': 'description',
4502 'annotation': 'annotations.xml',
4503 'infojson': 'info.json',
4504 'pl_thumbnail': None,
4505 'pl_description': 'description',
4506 'pl_infojson': 'info.json',
4509 # As of [1] format syntax is:
4510 # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
4511 # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
4512 STR_FORMAT_RE_TMPL
= r
'''(?x)
4513 (?<!%)(?P<prefix>(?:%%)*)
4515 (?P<has_key>\((?P<key>{0})\))?
4517 (?P<conversion>[#0\-+ ]+)?
4519 (?P<precision>\.\d+)?
4520 (?P<len_mod>[hlL])? # unused in python
4521 {1} # conversion type
4526 STR_FORMAT_TYPES
= 'diouxXeEfFgGcrs'
4529 def limit_length(s
, length
):
4530 """ Add ellipses to overly long strings """
4535 return s
[:length
- len(ELLIPSES
)] + ELLIPSES
4539 def version_tuple(v
):
4540 return tuple(int(e
) for e
in re
.split(r
'[-.]', v
))
4543 def is_outdated_version(version
, limit
, assume_new
=True):
4545 return not assume_new
4547 return version_tuple(version
) < version_tuple(limit
)
4549 return not assume_new
4552 def ytdl_is_updateable():
4553 """ Returns if yt-dlp can be updated with -U """
4555 from .update
import is_non_updateable
4557 return not is_non_updateable()
4560 def args_to_str(args
):
4561 # Get a short string representation for a subprocess command
4562 return ' '.join(compat_shlex_quote(a
) for a
in args
)
4565 def error_to_compat_str(err
):
4567 # On python 2 error byte string must be decoded with proper
4568 # encoding rather than ascii
4569 if sys
.version_info
[0] < 3:
4570 err_str
= err_str
.decode(preferredencoding())
4574 def mimetype2ext(mt
):
4578 mt
, _
, params
= mt
.partition(';')
4583 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
4584 # it's the most popular one
4585 'audio/mpeg': 'mp3',
4586 'audio/x-wav': 'wav',
4588 'audio/wave': 'wav',
4591 ext
= FULL_MAP
.get(mt
)
4597 'smptett+xml': 'tt',
4601 'x-mp4-fragmented': 'mp4',
4602 'x-ms-sami': 'sami',
4605 'x-mpegurl': 'm3u8',
4606 'vnd.apple.mpegurl': 'm3u8',
4610 'vnd.ms-sstr+xml': 'ism',
4614 'filmstrip+json': 'fs',
4618 _
, _
, subtype
= mt
.rpartition('/')
4619 ext
= SUBTYPE_MAP
.get(subtype
.lower())
4630 _
, _
, suffix
= subtype
.partition('+')
4631 ext
= SUFFIX_MAP
.get(suffix
)
4635 return subtype
.replace('+', '.')
4638 def parse_codecs(codecs_str
):
4639 # http://tools.ietf.org/html/rfc6381
4642 split_codecs
= list(filter(None, map(
4643 str.strip
, codecs_str
.strip().strip(',').split(','))))
4644 vcodec
, acodec
, hdr
= None, None, None
4645 for full_codec
in split_codecs
:
4646 codec
= full_codec
.split('.')[0]
4647 if codec
in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora', 'dvh1', 'dvhe'):
4650 if codec
in ('dvh1', 'dvhe'):
4652 elif codec
== 'vp9' and vcodec
.startswith('vp9.2'):
4654 elif codec
== 'av01':
4655 parts
= full_codec
.split('.')
4656 if len(parts
) > 3 and parts
[3] == '10':
4658 vcodec
= '.'.join(parts
[:4])
4659 elif codec
in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
4663 write_string('WARNING: Unknown codec %s\n' % full_codec
, sys
.stderr
)
4664 if not vcodec
and not acodec
:
4665 if len(split_codecs
) == 2:
4667 'vcodec': split_codecs
[0],
4668 'acodec': split_codecs
[1],
4672 'vcodec': vcodec
or 'none',
4673 'acodec': acodec
or 'none',
4674 'dynamic_range': hdr
,
4679 def urlhandle_detect_ext(url_handle
):
4680 getheader
= url_handle
.headers
.get
4682 cd
= getheader('Content-Disposition')
4684 m
= re
.match(r
'attachment;\s*filename="(?P<filename>[^"]+)"', cd
)
4686 e
= determine_ext(m
.group('filename'), default_ext
=None)
4690 return mimetype2ext(getheader('Content-Type'))
4693 def encode_data_uri(data
, mime_type
):
4694 return 'data:%s;base64,%s' % (mime_type
, base64
.b64encode(data
).decode('ascii'))
4697 def age_restricted(content_limit
, age_limit
):
4698 """ Returns True iff the content should be blocked """
4700 if age_limit
is None: # No limit set
4702 if content_limit
is None:
4703 return False # Content available for everyone
4704 return age_limit
< content_limit
4707 def is_html(first_bytes
):
4708 """ Detect whether a file contains HTML by examining its first bytes. """
4711 (b
'\xef\xbb\xbf', 'utf-8'),
4712 (b
'\x00\x00\xfe\xff', 'utf-32-be'),
4713 (b
'\xff\xfe\x00\x00', 'utf-32-le'),
4714 (b
'\xff\xfe', 'utf-16-le'),
4715 (b
'\xfe\xff', 'utf-16-be'),
4717 for bom
, enc
in BOMS
:
4718 if first_bytes
.startswith(bom
):
4719 s
= first_bytes
[len(bom
):].decode(enc
, 'replace')
4722 s
= first_bytes
.decode('utf-8', 'replace')
4724 return re
.match(r
'^\s*<', s
)
4727 def determine_protocol(info_dict
):
4728 protocol
= info_dict
.get('protocol')
4729 if protocol
is not None:
4732 url
= info_dict
['url']
4733 if url
.startswith('rtmp'):
4735 elif url
.startswith('mms'):
4737 elif url
.startswith('rtsp'):
4740 ext
= determine_ext(url
)
4746 return compat_urllib_parse_urlparse(url
).scheme
4749 def render_table(header_row
, data
, delim
=False, extraGap
=0, hideEmpty
=False):
4750 """ Render a list of rows, each as a list of values """
4752 return len(remove_terminal_sequences(string
))
4754 def get_max_lens(table
):
4755 return [max(width(str(v
)) for v
in col
) for col
in zip(*table
)]
4757 def filter_using_list(row
, filterArray
):
4758 return [col
for (take
, col
) in zip(filterArray
, row
) if take
]
4761 max_lens
= get_max_lens(data
)
4762 header_row
= filter_using_list(header_row
, max_lens
)
4763 data
= [filter_using_list(row
, max_lens
) for row
in data
]
4765 table
= [header_row
] + data
4766 max_lens
= get_max_lens(table
)
4769 table
= [header_row
] + [[delim
* (ml
+ extraGap
) for ml
in max_lens
]] + data
4772 for pos
, text
in enumerate(map(str, row
)):
4773 row
[pos
] = text
+ (' ' * (max_lens
[pos
] - width(text
) + extraGap
))
4774 ret
= '\n'.join(''.join(row
) for row
in table
)
4778 def _match_one(filter_part
, dct
, incomplete
):
4779 # TODO: Generalize code with YoutubeDL._build_format_filter
4780 STRING_OPERATORS
= {
4781 '*=': operator
.contains
,
4782 '^=': lambda attr
, value
: attr
.startswith(value
),
4783 '$=': lambda attr
, value
: attr
.endswith(value
),
4784 '~=': lambda attr
, value
: re
.search(value
, attr
),
4786 COMPARISON_OPERATORS
= {
4788 '<=': operator
.le
, # "<=" must be defined above "<"
4795 operator_rex
= re
.compile(r
'''(?x)\s*
4797 \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
4799 (?P<quote>["\'])(?P
<quotedstrval
>.+?
)(?P
=quote
)|
4803 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
4804 m = operator_rex.search(filter_part)
4807 unnegated_op = COMPARISON_OPERATORS[m['op']]
4809 op = lambda attr, value: not unnegated_op(attr, value)
4812 comparison_value = m['quotedstrval'] or m['strval'] or m['intval']
4814 comparison_value = comparison_value.replace(r'\%s' % m['quote'], m['quote'])
4815 actual_value = dct.get(m['key'])
4816 numeric_comparison = None
4817 if isinstance(actual_value, compat_numeric_types):
4818 # If the original field is a string and matching comparisonvalue is
4819 # a number we should respect the origin of the original field
4820 # and process comparison value as a string (see
4821 # https://github.com/ytdl-org/youtube-dl/issues/11082)
4823 numeric_comparison = int(comparison_value)
4825 numeric_comparison = parse_filesize(comparison_value)
4826 if numeric_comparison is None:
4827 numeric_comparison = parse_filesize(f'{comparison_value}B')
4828 if numeric_comparison is None:
4829 numeric_comparison = parse_duration(comparison_value)
4830 if numeric_comparison is not None and m['op'] in STRING_OPERATORS:
4831 raise ValueError('Operator %s only supports string values!' % m['op'])
4832 if actual_value is None:
4833 return incomplete or m['none_inclusive']
4834 return op(actual_value, comparison_value if numeric_comparison is None else numeric_comparison)
4837 '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
4838 '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
4840 operator_rex = re.compile(r'''(?x
)\s
*
4841 (?P
<op
>%s)\s
*(?P
<key
>[a
-z_
]+)
4843 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
4844 m = operator_rex.search(filter_part)
4846 op = UNARY_OPERATORS[m.group('op')]
4847 actual_value = dct.get(m.group('key'))
4848 if incomplete and actual_value is None:
4850 return op(actual_value)
4852 raise ValueError('Invalid filter part %r' % filter_part)
4855 def match_str(filter_str, dct, incomplete=False):
4856 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false
4857 When incomplete, all conditions passes on missing fields
4860 _match_one(filter_part.replace(r'\&', '&'), dct, incomplete)
4861 for filter_part in re.split(r'(?<!\\)&', filter_str))
4864 def match_filter_func(filter_str):
4865 def _match_func(info_dict, *args, **kwargs):
4866 if match_str(filter_str, info_dict, *args, **kwargs):
4869 video_title = info_dict.get('title', info_dict.get('id', 'video'))
4870 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
4874 def parse_dfxp_time_expr(time_expr):
4878 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
4880 return float(mobj.group('time_offset'))
4882 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
4884 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
4887 def srt_subtitles_timecode(seconds):
4888 return '%02d:%02d:%02d,%03d' % timetuple_from_msec(seconds * 1000)
4891 def ass_subtitles_timecode(seconds):
4892 time = timetuple_from_msec(seconds * 1000)
4893 return '%01d:%02d:%02d.%02d' % (*time[:-1], time.milliseconds / 10)
4896 def dfxp2srt(dfxp_data):
4898 @param dfxp_data A
bytes-like
object containing DFXP data
4899 @returns A
unicode object containing converted SRT data
4901 LEGACY_NAMESPACES = (
4902 (b'http://www.w3.org/ns/ttml', [
4903 b'http://www.w3.org/2004/11/ttaf1',
4904 b'http://www.w3.org/2006/04/ttaf1',
4905 b'http://www.w3.org/2006/10/ttaf1',
4907 (b'http://www.w3.org/ns/ttml#styling', [
4908 b'http://www.w3.org/ns/ttml#style',
4912 SUPPORTED_STYLING = [
4921 _x = functools.partial(xpath_with_ns, ns_map={
4922 'xml': 'http://www.w3.org/XML/1998/namespace',
4923 'ttml': 'http://www.w3.org/ns/ttml',
4924 'tts': 'http://www.w3.org/ns/ttml#styling',
4930 class TTMLPElementParser(object):
4932 _unclosed_elements = []
4933 _applied_styles = []
4935 def start(self, tag, attrib):
4936 if tag in (_x('ttml:br'), 'br'):
4939 unclosed_elements = []
4941 element_style_id = attrib.get('style')
4943 style.update(default_style)
4944 if element_style_id:
4945 style.update(styles.get(element_style_id, {}))
4946 for prop in SUPPORTED_STYLING:
4947 prop_val = attrib.get(_x('tts:' + prop))
4949 style[prop] = prop_val
4952 for k, v in sorted(style.items()):
4953 if self._applied_styles and self._applied_styles[-1].get(k) == v:
4956 font += ' color="%s"' % v
4957 elif k == 'fontSize':
4958 font += ' size="%s"' % v
4959 elif k == 'fontFamily':
4960 font += ' face="%s"' % v
4961 elif k == 'fontWeight' and v == 'bold':
4963 unclosed_elements.append('b')
4964 elif k == 'fontStyle' and v == 'italic':
4966 unclosed_elements.append('i')
4967 elif k == 'textDecoration' and v == 'underline':
4969 unclosed_elements.append('u')
4971 self._out += '<font' + font + '>'
4972 unclosed_elements.append('font')
4974 if self._applied_styles:
4975 applied_style.update(self._applied_styles[-1])
4976 applied_style.update(style)
4977 self._applied_styles.append(applied_style)
4978 self._unclosed_elements.append(unclosed_elements)
4981 if tag not in (_x('ttml:br'), 'br'):
4982 unclosed_elements = self._unclosed_elements.pop()
4983 for element in reversed(unclosed_elements):
4984 self._out += '</%s>' % element
4985 if unclosed_elements and self._applied_styles:
4986 self._applied_styles.pop()
4988 def data(self, data):
4992 return self._out.strip()
4994 def parse_node(node):
4995 target = TTMLPElementParser()
4996 parser = xml.etree.ElementTree.XMLParser(target=target)
4997 parser.feed(xml.etree.ElementTree.tostring(node))
4998 return parser.close()
5000 for k, v in LEGACY_NAMESPACES:
5002 dfxp_data = dfxp_data.replace(ns, k)
5004 dfxp = compat_etree_fromstring(dfxp_data)
5006 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
5009 raise ValueError('Invalid dfxp/TTML subtitle')
5013 for style in dfxp.findall(_x('.//ttml:style')):
5014 style_id = style.get('id') or style.get(_x('xml:id'))
5017 parent_style_id = style.get('style')
5019 if parent_style_id not in styles:
5022 styles[style_id] = styles[parent_style_id].copy()
5023 for prop in SUPPORTED_STYLING:
5024 prop_val = style.get(_x('tts:' + prop))
5026 styles.setdefault(style_id, {})[prop] = prop_val
5032 for p in ('body', 'div'):
5033 ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
5036 style = styles.get(ele.get('style'))
5039 default_style.update(style)
5041 for para, index in zip(paras, itertools.count(1)):
5042 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
5043 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
5044 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
5045 if begin_time is None:
5050 end_time = begin_time + dur
5051 out.append('%d\n%s --> %s\n%s\n\n' % (
5053 srt_subtitles_timecode(begin_time),
5054 srt_subtitles_timecode(end_time),
5060 def cli_option(params, command_option, param):
5061 param = params.get(param)
5063 param = compat_str(param)
5064 return [command_option, param] if param is not None else []
5067 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
5068 param = params.get(param)
5071 assert isinstance(param, bool)
5073 return [command_option + separator + (true_value if param else false_value)]
5074 return [command_option, true_value if param else false_value]
5077 def cli_valueless_option(params, command_option, param, expected_value=True):
5078 param = params.get(param)
5079 return [command_option] if param == expected_value else []
5082 def cli_configuration_args(argdict, keys, default=[], use_compat=True):
5083 if isinstance(argdict, (list, tuple)): # for backward compatibility
5090 assert isinstance(argdict, dict)
5092 assert isinstance(keys, (list, tuple))
5093 for key_list in keys:
5094 arg_list = list(filter(
5095 lambda x: x is not None,
5096 [argdict.get(key.lower()) for key in variadic(key_list)]))
5098 return [arg for args in arg_list for arg in args]
5102 def _configuration_args(main_key, argdict, exe, keys=None, default=[], use_compat=True):
5103 main_key, exe = main_key.lower(), exe.lower()
5104 root_key = exe if main_key == exe else f'{main_key}+{exe}'
5105 keys = [f'{root_key}{k}' for k in (keys or [''])]
5106 if root_key in keys:
5108 keys.append((main_key, exe))
5109 keys.append('default')
5112 return cli_configuration_args(argdict, keys, default, use_compat)
5115 class ISO639Utils(object):
5116 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
5175 'iw': 'heb', # Replaced by he in 1989 revision
5185 'in': 'ind', # Replaced by id in 1989 revision
5300 'ji': 'yid', # Replaced by yi in 1989 revision
5308 def short2long(cls, code):
5309 """Convert language code from ISO 639-1 to ISO 639-2/T"""
5310 return cls._lang_map.get(code[:2])
5313 def long2short(cls, code):
5314 """Convert language code from ISO 639-2/T to ISO 639-1"""
5315 for short_name, long_name in cls._lang_map.items():
5316 if long_name == code:
5320 class ISO3166Utils(object):
5321 # From http://data.okfn.org/data/core/country-list
5323 'AF': 'Afghanistan',
5324 'AX': 'Åland Islands',
5327 'AS': 'American Samoa',
5332 'AG': 'Antigua and Barbuda',
5349 'BO': 'Bolivia, Plurinational State of',
5350 'BQ': 'Bonaire, Sint Eustatius and Saba',
5351 'BA': 'Bosnia and Herzegovina',
5353 'BV': 'Bouvet Island',
5355 'IO': 'British Indian Ocean Territory',
5356 'BN': 'Brunei Darussalam',
5358 'BF': 'Burkina Faso',
5364 'KY': 'Cayman Islands',
5365 'CF': 'Central African Republic',
5369 'CX': 'Christmas Island',
5370 'CC': 'Cocos (Keeling) Islands',
5374 'CD': 'Congo, the Democratic Republic of the',
5375 'CK': 'Cook Islands',
5377 'CI': 'Côte d\'Ivoire',
5382 'CZ': 'Czech Republic',
5386 'DO': 'Dominican Republic',
5389 'SV': 'El Salvador',
5390 'GQ': 'Equatorial Guinea',
5394 'FK': 'Falkland Islands (Malvinas)',
5395 'FO': 'Faroe Islands',
5399 'GF': 'French Guiana',
5400 'PF': 'French Polynesia',
5401 'TF': 'French Southern Territories',
5416 'GW': 'Guinea-Bissau',
5419 'HM': 'Heard Island and McDonald Islands',
5420 'VA': 'Holy See (Vatican City State)',
5427 'IR': 'Iran, Islamic Republic of',
5430 'IM': 'Isle of Man',
5440 'KP': 'Korea, Democratic People\'s Republic of',
5441 'KR': 'Korea, Republic of',
5444 'LA': 'Lao People\'s Democratic Republic',
5450 'LI': 'Liechtenstein',
5454 'MK': 'Macedonia, the Former Yugoslav Republic of',
5461 'MH': 'Marshall Islands',
5467 'FM': 'Micronesia, Federated States of',
5468 'MD': 'Moldova, Republic of',
5479 'NL': 'Netherlands',
5480 'NC': 'New Caledonia',
5481 'NZ': 'New Zealand',
5486 'NF': 'Norfolk Island',
5487 'MP': 'Northern Mariana Islands',
5492 'PS': 'Palestine, State of',
5494 'PG': 'Papua New Guinea',
5497 'PH': 'Philippines',
5501 'PR': 'Puerto Rico',
5505 'RU': 'Russian Federation',
5507 'BL': 'Saint Barthélemy',
5508 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
5509 'KN': 'Saint Kitts and Nevis',
5510 'LC': 'Saint Lucia',
5511 'MF': 'Saint Martin (French part)',
5512 'PM': 'Saint Pierre and Miquelon',
5513 'VC': 'Saint Vincent and the Grenadines',
5516 'ST': 'Sao Tome and Principe',
5517 'SA': 'Saudi Arabia',
5521 'SL': 'Sierra Leone',
5523 'SX': 'Sint Maarten (Dutch part)',
5526 'SB': 'Solomon Islands',
5528 'ZA': 'South Africa',
5529 'GS': 'South Georgia and the South Sandwich Islands',
5530 'SS': 'South Sudan',
5535 'SJ': 'Svalbard and Jan Mayen',
5538 'CH': 'Switzerland',
5539 'SY': 'Syrian Arab Republic',
5540 'TW': 'Taiwan, Province of China',
5542 'TZ': 'Tanzania, United Republic of',
5544 'TL': 'Timor-Leste',
5548 'TT': 'Trinidad and Tobago',
5551 'TM': 'Turkmenistan',
5552 'TC': 'Turks and Caicos Islands',
5556 'AE': 'United Arab Emirates',
5557 'GB': 'United Kingdom',
5558 'US': 'United States',
5559 'UM': 'United States Minor Outlying Islands',
5563 'VE': 'Venezuela, Bolivarian Republic of',
5565 'VG': 'Virgin Islands, British',
5566 'VI': 'Virgin Islands, U.S.',
5567 'WF': 'Wallis and Futuna',
5568 'EH': 'Western Sahara',
5575 def short2full(cls, code):
5576 """Convert an ISO 3166-2 country code to the corresponding full name"""
5577 return cls._country_map.get(code.upper())
5580 class GeoUtils(object):
5581 # Major IPv4 address blocks per country
5583 'AD': '46.172.224.0/19',
5584 'AE': '94.200.0.0/13',
5585 'AF': '149.54.0.0/17',
5586 'AG': '209.59.64.0/18',
5587 'AI': '204.14.248.0/21',
5588 'AL': '46.99.0.0/16',
5589 'AM': '46.70.0.0/15',
5590 'AO': '105.168.0.0/13',
5591 'AP': '182.50.184.0/21',
5592 'AQ': '23.154.160.0/24',
5593 'AR': '181.0.0.0/12',
5594 'AS': '202.70.112.0/20',
5595 'AT': '77.116.0.0/14',
5596 'AU': '1.128.0.0/11',
5597 'AW': '181.41.0.0/18',
5598 'AX': '185.217.4.0/22',
5599 'AZ': '5.197.0.0/16',
5600 'BA': '31.176.128.0/17',
5601 'BB': '65.48.128.0/17',
5602 'BD': '114.130.0.0/16',
5604 'BF': '102.178.0.0/15',
5605 'BG': '95.42.0.0/15',
5606 'BH': '37.131.0.0/17',
5607 'BI': '154.117.192.0/18',
5608 'BJ': '137.255.0.0/16',
5609 'BL': '185.212.72.0/23',
5610 'BM': '196.12.64.0/18',
5611 'BN': '156.31.0.0/16',
5612 'BO': '161.56.0.0/16',
5613 'BQ': '161.0.80.0/20',
5614 'BR': '191.128.0.0/12',
5615 'BS': '24.51.64.0/18',
5616 'BT': '119.2.96.0/19',
5617 'BW': '168.167.0.0/16',
5618 'BY': '178.120.0.0/13',
5619 'BZ': '179.42.192.0/18',
5620 'CA': '99.224.0.0/11',
5621 'CD': '41.243.0.0/16',
5622 'CF': '197.242.176.0/21',
5623 'CG': '160.113.0.0/16',
5624 'CH': '85.0.0.0/13',
5625 'CI': '102.136.0.0/14',
5626 'CK': '202.65.32.0/19',
5627 'CL': '152.172.0.0/14',
5628 'CM': '102.244.0.0/14',
5629 'CN': '36.128.0.0/10',
5630 'CO': '181.240.0.0/12',
5631 'CR': '201.192.0.0/12',
5632 'CU': '152.206.0.0/15',
5633 'CV': '165.90.96.0/19',
5634 'CW': '190.88.128.0/17',
5635 'CY': '31.153.0.0/16',
5636 'CZ': '88.100.0.0/14',
5638 'DJ': '197.241.0.0/17',
5639 'DK': '87.48.0.0/12',
5640 'DM': '192.243.48.0/20',
5641 'DO': '152.166.0.0/15',
5642 'DZ': '41.96.0.0/12',
5643 'EC': '186.68.0.0/15',
5644 'EE': '90.190.0.0/15',
5645 'EG': '156.160.0.0/11',
5646 'ER': '196.200.96.0/20',
5647 'ES': '88.0.0.0/11',
5648 'ET': '196.188.0.0/14',
5649 'EU': '2.16.0.0/13',
5650 'FI': '91.152.0.0/13',
5651 'FJ': '144.120.0.0/16',
5652 'FK': '80.73.208.0/21',
5653 'FM': '119.252.112.0/20',
5654 'FO': '88.85.32.0/19',
5656 'GA': '41.158.0.0/15',
5658 'GD': '74.122.88.0/21',
5659 'GE': '31.146.0.0/16',
5660 'GF': '161.22.64.0/18',
5661 'GG': '62.68.160.0/19',
5662 'GH': '154.160.0.0/12',
5663 'GI': '95.164.0.0/16',
5664 'GL': '88.83.0.0/19',
5665 'GM': '160.182.0.0/15',
5666 'GN': '197.149.192.0/18',
5667 'GP': '104.250.0.0/19',
5668 'GQ': '105.235.224.0/20',
5669 'GR': '94.64.0.0/13',
5670 'GT': '168.234.0.0/16',
5671 'GU': '168.123.0.0/16',
5672 'GW': '197.214.80.0/20',
5673 'GY': '181.41.64.0/18',
5674 'HK': '113.252.0.0/14',
5675 'HN': '181.210.0.0/16',
5676 'HR': '93.136.0.0/13',
5677 'HT': '148.102.128.0/17',
5678 'HU': '84.0.0.0/14',
5679 'ID': '39.192.0.0/10',
5680 'IE': '87.32.0.0/12',
5681 'IL': '79.176.0.0/13',
5682 'IM': '5.62.80.0/20',
5683 'IN': '117.192.0.0/10',
5684 'IO': '203.83.48.0/21',
5685 'IQ': '37.236.0.0/14',
5686 'IR': '2.176.0.0/12',
5687 'IS': '82.221.0.0/16',
5688 'IT': '79.0.0.0/10',
5689 'JE': '87.244.64.0/18',
5690 'JM': '72.27.0.0/17',
5691 'JO': '176.29.0.0/16',
5692 'JP': '133.0.0.0/8',
5693 'KE': '105.48.0.0/12',
5694 'KG': '158.181.128.0/17',
5695 'KH': '36.37.128.0/17',
5696 'KI': '103.25.140.0/22',
5697 'KM': '197.255.224.0/20',
5698 'KN': '198.167.192.0/19',
5699 'KP': '175.45.176.0/22',
5700 'KR': '175.192.0.0/10',
5701 'KW': '37.36.0.0/14',
5702 'KY': '64.96.0.0/15',
5703 'KZ': '2.72.0.0/13',
5704 'LA': '115.84.64.0/18',
5705 'LB': '178.135.0.0/16',
5706 'LC': '24.92.144.0/20',
5707 'LI': '82.117.0.0/19',
5708 'LK': '112.134.0.0/15',
5709 'LR': '102.183.0.0/16',
5710 'LS': '129.232.0.0/17',
5711 'LT': '78.56.0.0/13',
5712 'LU': '188.42.0.0/16',
5713 'LV': '46.109.0.0/16',
5714 'LY': '41.252.0.0/14',
5715 'MA': '105.128.0.0/11',
5716 'MC': '88.209.64.0/18',
5717 'MD': '37.246.0.0/16',
5718 'ME': '178.175.0.0/17',
5719 'MF': '74.112.232.0/21',
5720 'MG': '154.126.0.0/17',
5721 'MH': '117.103.88.0/21',
5722 'MK': '77.28.0.0/15',
5723 'ML': '154.118.128.0/18',
5724 'MM': '37.111.0.0/17',
5725 'MN': '49.0.128.0/17',
5726 'MO': '60.246.0.0/16',
5727 'MP': '202.88.64.0/20',
5728 'MQ': '109.203.224.0/19',
5729 'MR': '41.188.64.0/18',
5730 'MS': '208.90.112.0/22',
5731 'MT': '46.11.0.0/16',
5732 'MU': '105.16.0.0/12',
5733 'MV': '27.114.128.0/18',
5734 'MW': '102.70.0.0/15',
5735 'MX': '187.192.0.0/11',
5736 'MY': '175.136.0.0/13',
5737 'MZ': '197.218.0.0/15',
5738 'NA': '41.182.0.0/16',
5739 'NC': '101.101.0.0/18',
5740 'NE': '197.214.0.0/18',
5741 'NF': '203.17.240.0/22',
5742 'NG': '105.112.0.0/12',
5743 'NI': '186.76.0.0/15',
5744 'NL': '145.96.0.0/11',
5745 'NO': '84.208.0.0/13',
5746 'NP': '36.252.0.0/15',
5747 'NR': '203.98.224.0/19',
5748 'NU': '49.156.48.0/22',
5749 'NZ': '49.224.0.0/14',
5750 'OM': '5.36.0.0/15',
5751 'PA': '186.72.0.0/15',
5752 'PE': '186.160.0.0/14',
5753 'PF': '123.50.64.0/18',
5754 'PG': '124.240.192.0/19',
5755 'PH': '49.144.0.0/13',
5756 'PK': '39.32.0.0/11',
5757 'PL': '83.0.0.0/11',
5758 'PM': '70.36.0.0/20',
5759 'PR': '66.50.0.0/16',
5760 'PS': '188.161.0.0/16',
5761 'PT': '85.240.0.0/13',
5762 'PW': '202.124.224.0/20',
5763 'PY': '181.120.0.0/14',
5764 'QA': '37.210.0.0/15',
5765 'RE': '102.35.0.0/16',
5766 'RO': '79.112.0.0/13',
5767 'RS': '93.86.0.0/15',
5768 'RU': '5.136.0.0/13',
5769 'RW': '41.186.0.0/16',
5770 'SA': '188.48.0.0/13',
5771 'SB': '202.1.160.0/19',
5772 'SC': '154.192.0.0/11',
5773 'SD': '102.120.0.0/13',
5774 'SE': '78.64.0.0/12',
5775 'SG': '8.128.0.0/10',
5776 'SI': '188.196.0.0/14',
5777 'SK': '78.98.0.0/15',
5778 'SL': '102.143.0.0/17',
5779 'SM': '89.186.32.0/19',
5780 'SN': '41.82.0.0/15',
5781 'SO': '154.115.192.0/18',
5782 'SR': '186.179.128.0/17',
5783 'SS': '105.235.208.0/21',
5784 'ST': '197.159.160.0/19',
5785 'SV': '168.243.0.0/16',
5786 'SX': '190.102.0.0/20',
5788 'SZ': '41.84.224.0/19',
5789 'TC': '65.255.48.0/20',
5790 'TD': '154.68.128.0/19',
5791 'TG': '196.168.0.0/14',
5792 'TH': '171.96.0.0/13',
5793 'TJ': '85.9.128.0/18',
5794 'TK': '27.96.24.0/21',
5795 'TL': '180.189.160.0/20',
5796 'TM': '95.85.96.0/19',
5797 'TN': '197.0.0.0/11',
5798 'TO': '175.176.144.0/21',
5799 'TR': '78.160.0.0/11',
5800 'TT': '186.44.0.0/15',
5801 'TV': '202.2.96.0/19',
5802 'TW': '120.96.0.0/11',
5803 'TZ': '156.156.0.0/14',
5804 'UA': '37.52.0.0/14',
5805 'UG': '102.80.0.0/13',
5807 'UY': '167.56.0.0/13',
5808 'UZ': '84.54.64.0/18',
5809 'VA': '212.77.0.0/19',
5810 'VC': '207.191.240.0/21',
5811 'VE': '186.88.0.0/13',
5812 'VG': '66.81.192.0/20',
5813 'VI': '146.226.0.0/16',
5814 'VN': '14.160.0.0/11',
5815 'VU': '202.80.32.0/20',
5816 'WF': '117.20.32.0/21',
5817 'WS': '202.4.32.0/19',
5818 'YE': '134.35.0.0/16',
5819 'YT': '41.242.116.0/22',
5820 'ZA': '41.0.0.0/11',
5821 'ZM': '102.144.0.0/13',
5822 'ZW': '102.177.192.0/18',
5826 def random_ipv4(cls, code_or_block):
5827 if len(code_or_block) == 2:
5828 block = cls._country_ip_map.get(code_or_block.upper())
5832 block = code_or_block
5833 addr, preflen = block.split('/')
5834 addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
5835 addr_max = addr_min | (0xffffffff >> int(preflen))
5836 return compat_str(socket.inet_ntoa(
5837 compat_struct_pack('!L', random.randint(addr_min, addr_max))))
5840 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
5841 def __init__(self, proxies=None):
5842 # Set default handlers
5843 for type in ('http', 'https'):
5844 setattr(self, '%s_open' % type,
5845 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
5846 meth(r, proxy, type))
5847 compat_urllib_request.ProxyHandler.__init__(self, proxies)
5849 def proxy_open(self, req, proxy, type):
5850 req_proxy = req.headers.get('Ytdl-request-proxy')
5851 if req_proxy is not None:
5853 del req.headers['Ytdl-request-proxy']
5855 if proxy == '__noproxy__':
5856 return None # No Proxy
5857 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
5858 req.add_header('Ytdl-socks-proxy', proxy)
5859 # yt-dlp's http/https handlers do wrapping the socket with socks
5861 return compat_urllib_request.ProxyHandler.proxy_open(
5862 self, req, proxy, type)
5865 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
5866 # released into Public Domain
5867 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
5869 def long_to_bytes(n, blocksize=0):
5870 """long_to_bytes(n:long, blocksize:int) : string
5871 Convert a long integer to a byte string.
5873 If optional blocksize is given and greater than zero, pad the front of the
5874 byte string with binary zeros so that the length is a multiple of
5877 # after much testing, this algorithm was deemed to be the fastest
5881 s = compat_struct_pack('>I', n & 0xffffffff) + s
5883 # strip off leading zeros
5884 for i in range(len(s)):
5885 if s[i] != b'\000'[0]:
5888 # only happens when n == 0
5892 # add back some pad bytes. this could be done more efficiently w.r.t. the
5893 # de-padding being done above, but sigh...
5894 if blocksize > 0 and len(s) % blocksize:
5895 s = (blocksize - len(s) % blocksize) * b'\000' + s
5899 def bytes_to_long(s):
5900 """bytes_to_long(string) : long
5901 Convert a byte string to a long integer.
5903 This is (essentially) the inverse of long_to_bytes().
5908 extra = (4 - length % 4)
5909 s = b'\000' * extra + s
5910 length = length + extra
5911 for i in range(0, length, 4):
5912 acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
5916 def ohdave_rsa_encrypt(data, exponent, modulus):
5918 Implement OHDave
's RSA algorithm. See http://www.ohdave.com/rsa/
5921 data: data to encrypt, bytes-like object
5922 exponent, modulus: parameter e and N of RSA algorithm, both integer
5923 Output: hex string of encrypted data
5925 Limitation: supports one block encryption only
5928 payload = int(binascii.hexlify(data[::-1]), 16)
5929 encrypted = pow(payload, exponent, modulus)
5930 return '%x' % encrypted
5933 def pkcs1pad(data, length):
5935 Padding input data with PKCS#1 scheme
5937 @param {int[]} data input data
5938 @param {int} length target length
5939 @returns {int[]} padded data
5941 if len(data) > length - 11:
5942 raise ValueError('Input data too
long for PKCS
#1 padding')
5944 pseudo_random
= [random
.randint(0, 254) for _
in range(length
- len(data
) - 3)]
5945 return [0, 2] + pseudo_random
+ [0] + data
5948 def encode_base_n(num
, n
, table
=None):
5949 FULL_TABLE
= '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
5951 table
= FULL_TABLE
[:n
]
5954 raise ValueError('base %d exceeds table length %d' % (n
, len(table
)))
5961 ret
= table
[num
% n
] + ret
5966 def decode_packed_codes(code
):
5967 mobj
= re
.search(PACKED_CODES_RE
, code
)
5968 obfuscated_code
, base
, count
, symbols
= mobj
.groups()
5971 symbols
= symbols
.split('|')
5976 base_n_count
= encode_base_n(count
, base
)
5977 symbol_table
[base_n_count
] = symbols
[count
] or base_n_count
5980 r
'\b(\w+)\b', lambda mobj
: symbol_table
[mobj
.group(0)],
5984 def caesar(s
, alphabet
, shift
):
5989 alphabet
[(alphabet
.index(c
) + shift
) % l
] if c
in alphabet
else c
5994 return caesar(s
, r
'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''', 47)
5997 def parse_m3u8_attributes(attrib
):
5999 for (key
, val
) in re
.findall(r
'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib
):
6000 if val
.startswith('"'):
6006 def urshift(val
, n
):
6007 return val
>> n
if val
>= 0 else (val
+ 0x100000000) >> n
6010 # Based on png2str() written by @gdkchan and improved by @yokrysty
6011 # Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
6012 def decode_png(png_data
):
6013 # Reference: https://www.w3.org/TR/PNG/
6014 header
= png_data
[8:]
6016 if png_data
[:8] != b
'\x89PNG\x0d\x0a\x1a\x0a' or header
[4:8] != b
'IHDR':
6017 raise IOError('Not a valid PNG file.')
6019 int_map
= {1: '>B', 2: '>H', 4: '>I'}
6020 unpack_integer
= lambda x
: compat_struct_unpack(int_map
[len(x
)], x
)[0]
6025 length
= unpack_integer(header
[:4])
6028 chunk_type
= header
[:4]
6031 chunk_data
= header
[:length
]
6032 header
= header
[length
:]
6034 header
= header
[4:] # Skip CRC
6042 ihdr
= chunks
[0]['data']
6044 width
= unpack_integer(ihdr
[:4])
6045 height
= unpack_integer(ihdr
[4:8])
6049 for chunk
in chunks
:
6050 if chunk
['type'] == b
'IDAT':
6051 idat
+= chunk
['data']
6054 raise IOError('Unable to read PNG data.')
6056 decompressed_data
= bytearray(zlib
.decompress(idat
))
6061 def _get_pixel(idx
):
6066 for y
in range(height
):
6067 basePos
= y
* (1 + stride
)
6068 filter_type
= decompressed_data
[basePos
]
6072 pixels
.append(current_row
)
6074 for x
in range(stride
):
6075 color
= decompressed_data
[1 + basePos
+ x
]
6076 basex
= y
* stride
+ x
6081 left
= _get_pixel(basex
- 3)
6083 up
= _get_pixel(basex
- stride
)
6085 if filter_type
== 1: # Sub
6086 color
= (color
+ left
) & 0xff
6087 elif filter_type
== 2: # Up
6088 color
= (color
+ up
) & 0xff
6089 elif filter_type
== 3: # Average
6090 color
= (color
+ ((left
+ up
) >> 1)) & 0xff
6091 elif filter_type
== 4: # Paeth
6097 c
= _get_pixel(basex
- stride
- 3)
6105 if pa
<= pb
and pa
<= pc
:
6106 color
= (color
+ a
) & 0xff
6108 color
= (color
+ b
) & 0xff
6110 color
= (color
+ c
) & 0xff
6112 current_row
.append(color
)
6114 return width
, height
, pixels
6117 def write_xattr(path
, key
, value
):
6118 # This mess below finds the best xattr tool for the job
6120 # try the pyxattr module...
6123 if hasattr(xattr
, 'set'): # pyxattr
6124 # Unicode arguments are not supported in python-pyxattr until
6126 # See https://github.com/ytdl-org/youtube-dl/issues/5498
6127 pyxattr_required_version
= '0.5.0'
6128 if version_tuple(xattr
.__version
__) < version_tuple(pyxattr_required_version
):
6129 # TODO: fallback to CLI tools
6130 raise XAttrUnavailableError(
6131 'python-pyxattr is detected but is too old. '
6132 'yt-dlp requires %s or above while your version is %s. '
6133 'Falling back to other xattr implementations' % (
6134 pyxattr_required_version
, xattr
.__version
__))
6136 setxattr
= xattr
.set
6138 setxattr
= xattr
.setxattr
6141 setxattr(path
, key
, value
)
6142 except EnvironmentError as e
:
6143 raise XAttrMetadataError(e
.errno
, e
.strerror
)
6146 if compat_os_name
== 'nt':
6147 # Write xattrs to NTFS Alternate Data Streams:
6148 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
6149 assert ':' not in key
6150 assert os
.path
.exists(path
)
6152 ads_fn
= path
+ ':' + key
6154 with open(ads_fn
, 'wb') as f
:
6156 except EnvironmentError as e
:
6157 raise XAttrMetadataError(e
.errno
, e
.strerror
)
6159 user_has_setfattr
= check_executable('setfattr', ['--version'])
6160 user_has_xattr
= check_executable('xattr', ['-h'])
6162 if user_has_setfattr
or user_has_xattr
:
6164 value
= value
.decode('utf-8')
6165 if user_has_setfattr
:
6166 executable
= 'setfattr'
6167 opts
= ['-n', key
, '-v', value
]
6168 elif user_has_xattr
:
6169 executable
= 'xattr'
6170 opts
= ['-w', key
, value
]
6172 cmd
= ([encodeFilename(executable
, True)]
6173 + [encodeArgument(o
) for o
in opts
]
6174 + [encodeFilename(path
, True)])
6178 cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, stdin
=subprocess
.PIPE
)
6179 except EnvironmentError as e
:
6180 raise XAttrMetadataError(e
.errno
, e
.strerror
)
6181 stdout
, stderr
= p
.communicate_or_kill()
6182 stderr
= stderr
.decode('utf-8', 'replace')
6183 if p
.returncode
!= 0:
6184 raise XAttrMetadataError(p
.returncode
, stderr
)
6187 # On Unix, and can't find pyxattr, setfattr, or xattr.
6188 if sys
.platform
.startswith('linux'):
6189 raise XAttrUnavailableError(
6190 "Couldn't find a tool to set the xattrs. "
6191 "Install either the python 'pyxattr' or 'xattr' "
6192 "modules, or the GNU 'attr' package "
6193 "(which contains the 'setfattr' tool).")
6195 raise XAttrUnavailableError(
6196 "Couldn't find a tool to set the xattrs. "
6197 "Install either the python 'xattr' module, "
6198 "or the 'xattr' binary.")
6201 def random_birthday(year_field
, month_field
, day_field
):
6202 start_date
= datetime
.date(1950, 1, 1)
6203 end_date
= datetime
.date(1995, 12, 31)
6204 offset
= random
.randint(0, (end_date
- start_date
).days
)
6205 random_date
= start_date
+ datetime
.timedelta(offset
)
6207 year_field
: str(random_date
.year
),
6208 month_field
: str(random_date
.month
),
6209 day_field
: str(random_date
.day
),
6213 # Templates for internet shortcut files, which are plain text files.
6214 DOT_URL_LINK_TEMPLATE
= '''
6219 DOT_WEBLOC_LINK_TEMPLATE
= '''
6220 <?xml version="1.0" encoding="UTF-8"?>
6221 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
6222 <plist version="1.0">
6225 \t<string>%(url)s</string>
6230 DOT_DESKTOP_LINK_TEMPLATE
= '''
6240 def iri_to_uri(iri
):
6242 Converts an IRI (Internationalized Resource Identifier, allowing Unicode characters) to a URI (Uniform Resource Identifier, ASCII-only).
6244 The function doesn't add an additional layer of escaping; e.g., it doesn't escape `%3C` as `%253C`. Instead, it percent-escapes characters with an underlying UTF-8 encoding *besides* those already escaped, leaving the URI intact.
6247 iri_parts
= compat_urllib_parse_urlparse(iri
)
6249 if '[' in iri_parts
.netloc
:
6250 raise ValueError('IPv6 URIs are not, yet, supported.')
6251 # Querying `.netloc`, when there's only one bracket, also raises a ValueError.
6253 # The `safe` argument values, that the following code uses, contain the characters that should not be percent-encoded. Everything else but letters, digits and '_.-' will be percent-encoded with an underlying UTF-8 encoding. Everything already percent-encoded will be left as is.
6256 if iri_parts
.username
:
6257 net_location
+= compat_urllib_parse_quote(iri_parts
.username
, safe
=r
"!$%&'()*+,~")
6258 if iri_parts
.password
is not None:
6259 net_location
+= ':' + compat_urllib_parse_quote(iri_parts
.password
, safe
=r
"!$%&'()*+,~")
6262 net_location
+= iri_parts
.hostname
.encode('idna').decode('utf-8') # Punycode for Unicode hostnames.
6263 # The 'idna' encoding produces ASCII text.
6264 if iri_parts
.port
is not None and iri_parts
.port
!= 80:
6265 net_location
+= ':' + str(iri_parts
.port
)
6267 return compat_urllib_parse_urlunparse(
6271 compat_urllib_parse_quote_plus(iri_parts
.path
, safe
=r
"!$%&'()*+,/:;=@|~"),
6273 # Unsure about the `safe` argument, since this is a legacy way of handling parameters.
6274 compat_urllib_parse_quote_plus(iri_parts
.params
, safe
=r
"!$%&'()*+,/:;=@|~"),
6276 # Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component.
6277 compat_urllib_parse_quote_plus(iri_parts
.query
, safe
=r
"!$%&'()*+,/:;=?@{|}~"),
6279 compat_urllib_parse_quote_plus(iri_parts
.fragment
, safe
=r
"!#$%&'()*+,/:;=?@{|}~")))
6281 # Source for `safe` arguments: https://url.spec.whatwg.org/#percent-encoded-bytes.
6284 def to_high_limit_path(path
):
6285 if sys
.platform
in ['win32', 'cygwin']:
6286 # Work around MAX_PATH limitation on Windows. The maximum allowed length for the individual path segments may still be quite limited.
6287 return r
'\\?\ '.rstrip() + os
.path
.abspath(path
)
6292 def format_field(obj
, field
=None, template
='%s', ignore
=(None, ''), default
='', func
=None):
6294 val
= obj
if obj
is not None else default
6296 val
= obj
.get(field
, default
)
6297 if func
and val
not in ignore
:
6299 return template
% val
if val
not in ignore
else default
6302 def clean_podcast_url(url
):
6303 return re
.sub(r
'''(?x)
6307 media\.blubrry\.com| # https://create.blubrry.com/resources/podcast-media-download-statistics/getting-started/
6310 (?:dts|www)\.podtrac\.com/(?:pts/)?redirect\.[0-9a-z]{3,4}| # http://analytics.podtrac.com/how-to-measure
6313 cn\.co| # https://podcorn.com/analytics-prefix/
6314 st\.fm # https://podsights.com/docs/
6319 _HEX_TABLE
= '0123456789abcdef'
6322 def random_uuidv4():
6323 return re
.sub(r
'[xy]', lambda x
: _HEX_TABLE
[random
.randint(0, 15)], 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx')
6326 def make_dir(path
, to_screen
=None):
6328 dn
= os
.path
.dirname(path
)
6329 if dn
and not os
.path
.exists(dn
):
6332 except (OSError, IOError) as err
:
6333 if callable(to_screen
) is not None:
6334 to_screen('unable to create directory ' + error_to_compat_str(err
))
6338 def get_executable_path():
6339 from zipimport
import zipimporter
6340 if hasattr(sys
, 'frozen'): # Running from PyInstaller
6341 path
= os
.path
.dirname(sys
.executable
)
6342 elif isinstance(globals().get('__loader__'), zipimporter
): # Running from ZIP
6343 path
= os
.path
.join(os
.path
.dirname(__file__
), '../..')
6345 path
= os
.path
.join(os
.path
.dirname(__file__
), '..')
6346 return os
.path
.abspath(path
)
6349 def load_plugins(name
, suffix
, namespace
):
6352 plugins_spec
= importlib
.util
.spec_from_file_location(
6353 name
, os
.path
.join(get_executable_path(), 'ytdlp_plugins', name
, '__init__.py'))
6354 plugins
= importlib
.util
.module_from_spec(plugins_spec
)
6355 sys
.modules
[plugins_spec
.name
] = plugins
6356 plugins_spec
.loader
.exec_module(plugins
)
6357 for name
in dir(plugins
):
6358 if name
in namespace
:
6360 if not name
.endswith(suffix
):
6362 klass
= getattr(plugins
, name
)
6363 classes
[name
] = namespace
[name
] = klass
6364 except FileNotFoundError
:
6370 obj
, *path_list
, default
=None, expected_type
=None, get_all
=True,
6371 casesense
=True, is_user_input
=False, traverse_string
=False):
6372 ''' Traverse nested list/dict/tuple
6373 @param path_list A list of paths which are checked one by one.
6374 Each path is a list of keys where each key is a string,
6375 a function, a tuple of strings or "...".
6376 When a fuction is given, it takes the key as argument and
6377 returns whether the key matches or not. When a tuple is given,
6378 all the keys given in the tuple are traversed, and
6379 "..." traverses all the keys in the object
6380 @param default Default value to return
6381 @param expected_type Only accept final value of this type (Can also be any callable)
6382 @param get_all Return all the values obtained from a path or only the first one
6383 @param casesense Whether to consider dictionary keys as case sensitive
6384 @param is_user_input Whether the keys are generated from user input. If True,
6385 strings are converted to int/slice if necessary
6386 @param traverse_string Whether to traverse inside strings. If True, any
6387 non-compatible object will also be converted into a string
6391 _lower
= lambda k
: (k
.lower() if isinstance(k
, str) else k
)
6392 path_list
= (map(_lower
, variadic(path
)) for path
in path_list
)
6394 def _traverse_obj(obj
, path
, _current_depth
=0):
6398 path
= tuple(variadic(path
))
6399 for i
, key
in enumerate(path
):
6400 if isinstance(key
, (list, tuple)):
6401 obj
= [_traverse_obj(obj
, sub_key
, _current_depth
) for sub_key
in key
]
6404 obj
= (obj
.values() if isinstance(obj
, dict)
6405 else obj
if isinstance(obj
, (list, tuple, LazyList
))
6406 else str(obj
) if traverse_string
else [])
6408 depth
= max(depth
, _current_depth
)
6409 return [_traverse_obj(inner_obj
, path
[i
+ 1:], _current_depth
) for inner_obj
in obj
]
6411 if isinstance(obj
, (list, tuple, LazyList
)):
6412 obj
= enumerate(obj
)
6413 elif isinstance(obj
, dict):
6416 if not traverse_string
:
6420 depth
= max(depth
, _current_depth
)
6421 return [_traverse_obj(v
, path
[i
+ 1:], _current_depth
) for k
, v
in obj
if key(k
)]
6422 elif isinstance(obj
, dict) and not (is_user_input
and key
== ':'):
6423 obj
= (obj
.get(key
) if casesense
or (key
in obj
)
6424 else next((v
for k
, v
in obj
.items() if _lower(k
) == key
), None))
6427 key
= (int_or_none(key
) if ':' not in key
6428 else slice(*map(int_or_none
, key
.split(':'))))
6429 if key
== slice(None):
6430 return _traverse_obj(obj
, (..., *path
[i
+ 1:]), _current_depth
)
6431 if not isinstance(key
, (int, slice)):
6433 if not isinstance(obj
, (list, tuple, LazyList
)):
6434 if not traverse_string
:
6443 if isinstance(expected_type
, type):
6444 type_test
= lambda val
: val
if isinstance(val
, expected_type
) else None
6445 elif expected_type
is not None:
6446 type_test
= expected_type
6448 type_test
= lambda val
: val
6450 for path
in path_list
:
6452 val
= _traverse_obj(obj
, path
)
6455 for _
in range(depth
- 1):
6456 val
= itertools
.chain
.from_iterable(v
for v
in val
if v
is not None)
6457 val
= [v
for v
in map(type_test
, val
) if v
is not None]
6459 return val
if get_all
else val
[0]
6461 val
= type_test(val
)
6467 def traverse_dict(dictn
, keys
, casesense
=True):
6468 ''' For backward compatibility. Do not use '''
6469 return traverse_obj(dictn
, keys
, casesense
=casesense
,
6470 is_user_input
=True, traverse_string
=True)
6473 def variadic(x
, allowed_types
=(str, bytes)):
6474 return x
if isinstance(x
, collections
.abc
.Iterable
) and not isinstance(x
, allowed_types
) else (x
,)
6477 # create a JSON Web Signature (jws) with HS256 algorithm
6478 # the resulting format is in JWS Compact Serialization
6479 # implemented following JWT https://www.rfc-editor.org/rfc/rfc7519.html
6480 # implemented following JWS https://www.rfc-editor.org/rfc/rfc7515.html
6481 def jwt_encode_hs256(payload_data
, key
, headers
={}):
6487 header_data
.update(headers
)
6488 header_b64
= base64
.b64encode(json
.dumps(header_data
).encode('utf-8'))
6489 payload_b64
= base64
.b64encode(json
.dumps(payload_data
).encode('utf-8'))
6490 h
= hmac
.new(key
.encode('utf-8'), header_b64
+ b
'.' + payload_b64
, hashlib
.sha256
)
6491 signature_b64
= base64
.b64encode(h
.digest())
6492 token
= header_b64
+ b
'.' + payload_b64
+ b
'.' + signature_b64
6496 def supports_terminal_sequences(stream
):
6497 if compat_os_name
== 'nt':
6498 if get_windows_version() < (10, 0, 10586):
6500 elif not os
.getenv('TERM'):
6503 return stream
.isatty()
6504 except BaseException
:
6508 _terminal_sequences_re
= re
.compile('\033\\[[^m]+m')
6511 def remove_terminal_sequences(string
):
6512 return _terminal_sequences_re
.sub('', string
)
6515 def number_of_digits(number
):
6516 return len('%d' % number
)
projects / yt-dlp.git / blob