4 from __future__
import unicode_literals
36 import xml
.etree
.ElementTree
40 compat_HTMLParseError
,
45 compat_ctypes_WINFUNCTYPE
,
46 compat_etree_fromstring
,
49 compat_html_entities_html5
,
61 compat_urllib_parse_urlencode
,
62 compat_urllib_parse_urlparse
,
63 compat_urllib_parse_urlunparse
,
64 compat_urllib_parse_quote
,
65 compat_urllib_parse_quote_plus
,
66 compat_urllib_parse_unquote_plus
,
67 compat_urllib_request
,
78 def register_socks_protocols():
79 # "Register" SOCKS protocols
80 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
81 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
82 for scheme
in ('socks', 'socks4', 'socks4a', 'socks5'):
83 if scheme
not in compat_urlparse
.uses_netloc
:
84 compat_urlparse
.uses_netloc
.append(scheme
)
87 # This is not clearly defined otherwise
88 compiled_regex_type
= type(re
.compile(''))
91 def random_user_agent():
92 _USER_AGENT_TPL
= 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
1671 return _USER_AGENT_TPL
% random
.choice(_CHROME_VERSIONS
)
1675 'User-Agent': random_user_agent(),
1676 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
1677 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1678 'Accept-Encoding': 'gzip, deflate',
1679 'Accept-Language': 'en-us,en;q=0.5',
1684 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
1688 NO_DEFAULT
= object()
1690 ENGLISH_MONTH_NAMES
= [
1691 'January', 'February', 'March', 'April', 'May', 'June',
1692 'July', 'August', 'September', 'October', 'November', 'December']
1695 'en': ENGLISH_MONTH_NAMES
,
1697 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
1698 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
1701 KNOWN_EXTENSIONS
= (
1702 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
1703 'flv', 'f4v', 'f4a', 'f4b',
1704 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
1705 'mkv', 'mka', 'mk3d',
1708 'asf', 'wmv', 'wma',
1714 'f4f', 'f4m', 'm3u8', 'smil')
1716 # needed for sanitizing filenames in restricted mode
1717 ACCENT_CHARS
= dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
1718 itertools
.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
1719 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
1742 '%Y/%m/%d %H:%M:%S',
1744 '%Y-%m-%d %H:%M:%S',
1745 '%Y-%m-%d %H:%M:%S.%f',
1748 '%Y-%m-%dT%H:%M:%SZ',
1749 '%Y-%m-%dT%H:%M:%S.%fZ',
1750 '%Y-%m-%dT%H:%M:%S.%f0Z',
1751 '%Y-%m-%dT%H:%M:%S',
1752 '%Y-%m-%dT%H:%M:%S.%f',
1754 '%b %d %Y at %H:%M',
1755 '%b %d %Y at %H:%M:%S',
1756 '%B %d %Y at %H:%M',
1757 '%B %d %Y at %H:%M:%S',
1760 DATE_FORMATS_DAY_FIRST
= list(DATE_FORMATS
)
1761 DATE_FORMATS_DAY_FIRST
.extend([
1767 '%d/%m/%Y %H:%M:%S',
1770 DATE_FORMATS_MONTH_FIRST
= list(DATE_FORMATS
)
1771 DATE_FORMATS_MONTH_FIRST
.extend([
1776 '%m/%d/%Y %H:%M:%S',
1779 PACKED_CODES_RE
= r
"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
1780 JSON_LD_RE
= r
'(?is)<script[^>]+type=(["\']?
)application
/ld\
+json\
1[^
>]*>(?P
<json_ld
>.+?
)</script
>'
1783 def preferredencoding():
1784 """Get preferred encoding.
1786 Returns the best encoding scheme for the system, based on
1787 locale.getpreferredencoding() and some further tweaks.
1790 pref = locale.getpreferredencoding()
1798 def write_json_file(obj, fn):
1799 """ Encode obj as JSON and write it to fn, atomically if possible """
1801 fn = encodeFilename(fn)
1802 if sys.version_info < (3, 0) and sys.platform != 'win32
':
1803 encoding = get_filesystem_encoding()
1804 # os.path.basename returns a bytes object, but NamedTemporaryFile
1805 # will fail if the filename contains non ascii characters unless we
1806 # use a unicode object
1807 path_basename = lambda f: os.path.basename(fn).decode(encoding)
1808 # the same for os.path.dirname
1809 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
1811 path_basename = os.path.basename
1812 path_dirname = os.path.dirname
1816 'prefix
': path_basename(fn) + '.',
1817 'dir': path_dirname(fn),
1821 # In Python 2.x, json.dump expects a bytestream.
1822 # In Python 3.x, it writes to a character stream
1823 if sys.version_info < (3, 0):
1828 'encoding
': 'utf
-8',
1831 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
1836 if sys.platform == 'win32
':
1837 # Need to remove existing file on Windows, else os.rename raises
1838 # WindowsError or FileExistsError.
1846 os.chmod(tf.name, 0o666 & ~mask)
1849 os.rename(tf.name, fn)
1858 if sys.version_info >= (2, 7):
1859 def find_xpath_attr(node, xpath, key, val=None):
1860 """ Find the xpath xpath[@key=val] """
1861 assert re.match(r'^
[a
-zA
-Z_
-]+$
', key)
1862 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
1863 return node.find(expr)
1865 def find_xpath_attr(node, xpath, key, val=None):
1866 for f in node.findall(compat_xpath(xpath)):
1867 if key not in f.attrib:
1869 if val is None or f.attrib.get(key) == val:
1873 # On python2.6 the xml.etree.ElementTree.Element methods don't support
1874 # the namespace parameter
1877 def xpath_with_ns(path
, ns_map
):
1878 components
= [c
.split(':') for c
in path
.split('/')]
1880 for c
in components
:
1882 replaced
.append(c
[0])
1885 replaced
.append('{%s}%s' % (ns_map
[ns
], tag
))
1886 return '/'.join(replaced
)
1889 def xpath_element(node
, xpath
, name
=None, fatal
=False, default
=NO_DEFAULT
):
1890 def _find_xpath(xpath
):
1891 return node
.find(compat_xpath(xpath
))
1893 if isinstance(xpath
, (str, compat_str
)):
1894 n
= _find_xpath(xpath
)
1902 if default
is not NO_DEFAULT
:
1905 name
= xpath
if name
is None else name
1906 raise ExtractorError('Could not find XML element %s' % name
)
1912 def xpath_text(node
, xpath
, name
=None, fatal
=False, default
=NO_DEFAULT
):
1913 n
= xpath_element(node
, xpath
, name
, fatal
=fatal
, default
=default
)
1914 if n
is None or n
== default
:
1917 if default
is not NO_DEFAULT
:
1920 name
= xpath
if name
is None else name
1921 raise ExtractorError('Could not find XML element\'s text %s' % name
)
1927 def xpath_attr(node
, xpath
, key
, name
=None, fatal
=False, default
=NO_DEFAULT
):
1928 n
= find_xpath_attr(node
, xpath
, key
)
1930 if default
is not NO_DEFAULT
:
1933 name
= '%s[@%s]' % (xpath
, key
) if name
is None else name
1934 raise ExtractorError('Could not find XML attribute %s' % name
)
1937 return n
.attrib
[key
]
1940 def get_element_by_id(id, html
):
1941 """Return the content of the tag with the specified ID in the passed HTML document"""
1942 return get_element_by_attribute('id', id, html
)
1945 def get_element_by_class(class_name
, html
):
1946 """Return the content of the first tag with the specified class in the passed HTML document"""
1947 retval
= get_elements_by_class(class_name
, html
)
1948 return retval
[0] if retval
else None
1951 def get_element_by_attribute(attribute
, value
, html
, escape_value
=True):
1952 retval
= get_elements_by_attribute(attribute
, value
, html
, escape_value
)
1953 return retval
[0] if retval
else None
1956 def get_elements_by_class(class_name
, html
):
1957 """Return the content of all tags with the specified class in the passed HTML document as a list"""
1958 return get_elements_by_attribute(
1959 'class', r
'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
1960 html, escape_value=False)
1963 def get_elements_by_attribute(attribute, value, html, escape_value=True):
1964 """Return the content of the tag with the specified attribute in the passed HTML document"""
1966 value = re.escape(value) if escape_value else value
1969 for m in re.finditer(r'''(?xs)
1971 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^
']*'|
))*?
1973 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^
"]*"|
='[^']*'|))*?
1977 ''' % (re.escape(attribute), value), html):
1978 res = m.group('content
')
1980 if res.startswith('"') or res.startswith("'"):
1983 retlist.append(unescapeHTML(res))
1988 class HTMLAttributeParser(compat_HTMLParser):
1989 """Trivial HTML parser to gather the attributes for a single element"""
1993 compat_HTMLParser.__init__(self)
1995 def handle_starttag(self, tag, attrs):
1996 self.attrs = dict(attrs)
1999 def extract_attributes(html_element):
2000 """Given a string for an HTML element such as
2002 a="foo" B="bar" c="&98;az" d=boz
2003 empty= noval entity="&"
2006 Decode and return a dictionary of attributes.
2008 'a
': 'foo
', 'b
': 'bar
', c: 'baz
', d: 'boz
',
2009 'empty
': '', 'noval
': None, 'entity
': '&',
2010 'sq
': '"', 'dq': '\''
2012 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
2013 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
2015 parser = HTMLAttributeParser()
2017 parser.feed(html_element)
2019 # Older Python may throw HTMLParseError in case of malformed HTML
2020 except compat_HTMLParseError:
2025 def clean_html(html):
2026 """Clean an HTML snippet into a readable string"""
2028 if html is None: # Convenience for sanitizing descriptions etc.
2032 html = html.replace('\n', ' ')
2033 html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
2034 html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
2036 html = re.sub('<.*?>', '', html)
2037 # Replace html entities
2038 html = unescapeHTML(html)
2042 def sanitize_open(filename, open_mode):
2043 """Try to open the given filename, and slightly tweak it if this fails.
2045 Attempts to open the given filename. If this fails, it tries to change
2046 the filename slightly, step by step, until it's either able to open it
2047 or it fails and raises a final exception, like the standard open()
2050 It returns the tuple (stream, definitive_file_name).
2054 if sys.platform == 'win32':
2056 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
2057 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
2058 stream = open(encodeFilename(filename), open_mode)
2059 return (stream, filename)
2060 except (IOError, OSError) as err:
2061 if err.errno in (errno.EACCES,):
2064 # In case of error, try to remove win32 forbidden chars
2065 alt_filename = sanitize_path(filename)
2066 if alt_filename == filename:
2069 # An exception here should be caught in the caller
2070 stream = open(encodeFilename(alt_filename), open_mode)
2071 return (stream, alt_filename)
2074 def timeconvert(timestr):
2075 """Convert RFC 2822 defined time string into system timestamp"""
2077 timetuple = email.utils.parsedate_tz(timestr)
2078 if timetuple is not None:
2079 timestamp = email.utils.mktime_tz(timetuple)
2083 def sanitize_filename(s, restricted=False, is_id=False):
2084 """Sanitizes a string so it could be used as part of a filename.
2085 If restricted is set, use a stricter subset of allowed characters.
2086 Set is_id if this is not an arbitrary string, but an ID that should be kept
2089 def replace_insane(char):
2090 if restricted and char in ACCENT_CHARS:
2091 return ACCENT_CHARS[char]
2092 if char == '?' or ord(char) < 32 or ord(char) == 127:
2095 return '' if restricted else '\''
2097 return '_
-' if restricted else ' -'
2098 elif char in '\\/|
*<>':
2100 if restricted and (char in '!&\'()[]{}$
;`^
,#' or char.isspace()):
2102 if restricted
and ord(char
) > 127:
2107 s
= re
.sub(r
'[0-9]+(?::[0-9]+)+', lambda m
: m
.group(0).replace(':', '_'), s
)
2108 result
= ''.join(map(replace_insane
, s
))
2110 while '__' in result
:
2111 result
= result
.replace('__', '_')
2112 result
= result
.strip('_')
2113 # Common case of "Foreign band name - English song title"
2114 if restricted
and result
.startswith('-_'):
2116 if result
.startswith('-'):
2117 result
= '_' + result
[len('-'):]
2118 result
= result
.lstrip('.')
2124 def sanitize_path(s
):
2125 """Sanitizes and normalizes path on Windows"""
2126 if sys
.platform
!= 'win32':
2128 drive_or_unc
, _
= os
.path
.splitdrive(s
)
2129 if sys
.version_info
< (2, 7) and not drive_or_unc
:
2130 drive_or_unc
, _
= os
.path
.splitunc(s
)
2131 norm_path
= os
.path
.normpath(remove_start(s
, drive_or_unc
)).split(os
.path
.sep
)
2135 path_part
if path_part
in ['.', '..'] else re
.sub(r
'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part
)
2136 for path_part
in norm_path
]
2138 sanitized_path
.insert(0, drive_or_unc
+ os
.path
.sep
)
2139 return os
.path
.join(*sanitized_path
)
2142 def sanitize_url(url
):
2143 # Prepend protocol-less URLs with `http:` scheme in order to mitigate
2144 # the number of unwanted failures due to missing protocol
2145 if url
.startswith('//'):
2146 return 'http:%s' % url
2147 # Fix some common typos seen so far
2149 # https://github.com/ytdl-org/youtube-dl/issues/15649
2150 (r
'^httpss://', r
'https://'),
2151 # https://bx1.be/lives/direct-tv/
2152 (r
'^rmtp([es]?)://', r
'rtmp\1://'),
2154 for mistake
, fixup
in COMMON_TYPOS
:
2155 if re
.match(mistake
, url
):
2156 return re
.sub(mistake
, fixup
, url
)
2160 def sanitized_Request(url
, *args
, **kwargs
):
2161 return compat_urllib_request
.Request(sanitize_url(url
), *args
, **kwargs
)
2165 """Expand shell variables and ~"""
2166 return os
.path
.expandvars(compat_expanduser(s
))
2169 def orderedSet(iterable
):
2170 """ Remove all duplicates from the input iterable """
2178 def _htmlentity_transform(entity_with_semicolon
):
2179 """Transforms an HTML entity to a character."""
2180 entity
= entity_with_semicolon
[:-1]
2182 # Known non-numeric HTML entity
2183 if entity
in compat_html_entities
.name2codepoint
:
2184 return compat_chr(compat_html_entities
.name2codepoint
[entity
])
2186 # TODO: HTML5 allows entities without a semicolon. For example,
2187 # 'Éric' should be decoded as 'Éric'.
2188 if entity_with_semicolon
in compat_html_entities_html5
:
2189 return compat_html_entities_html5
[entity_with_semicolon
]
2191 mobj
= re
.match(r
'#(x[0-9a-fA-F]+|[0-9]+)', entity
)
2192 if mobj
is not None:
2193 numstr
= mobj
.group(1)
2194 if numstr
.startswith('x'):
2196 numstr
= '0%s' % numstr
2199 # See https://github.com/ytdl-org/youtube-dl/issues/7518
2201 return compat_chr(int(numstr
, base
))
2205 # Unknown entity in name, return its literal representation
2206 return '&%s;' % entity
2209 def unescapeHTML(s
):
2212 assert type(s
) == compat_str
2215 r
'&([^&;]+;)', lambda m
: _htmlentity_transform(m
.group(1)), s
)
2218 def process_communicate_or_kill(p
, *args
, **kwargs
):
2220 return p
.communicate(*args
, **kwargs
)
2221 except BaseException
: # Including KeyboardInterrupt
2227 def get_subprocess_encoding():
2228 if sys
.platform
== 'win32' and sys
.getwindowsversion()[0] >= 5:
2229 # For subprocess calls, encode with locale encoding
2230 # Refer to http://stackoverflow.com/a/9951851/35070
2231 encoding
= preferredencoding()
2233 encoding
= sys
.getfilesystemencoding()
2234 if encoding
is None:
2239 def encodeFilename(s
, for_subprocess
=False):
2241 @param s The name of the file
2244 assert type(s
) == compat_str
2246 # Python 3 has a Unicode API
2247 if sys
.version_info
>= (3, 0):
2250 # Pass '' directly to use Unicode APIs on Windows 2000 and up
2251 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
2252 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
2253 if not for_subprocess
and sys
.platform
== 'win32' and sys
.getwindowsversion()[0] >= 5:
2256 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
2257 if sys
.platform
.startswith('java'):
2260 return s
.encode(get_subprocess_encoding(), 'ignore')
2263 def decodeFilename(b
, for_subprocess
=False):
2265 if sys
.version_info
>= (3, 0):
2268 if not isinstance(b
, bytes):
2271 return b
.decode(get_subprocess_encoding(), 'ignore')
2274 def encodeArgument(s
):
2275 if not isinstance(s
, compat_str
):
2276 # Legacy code that uses byte strings
2277 # Uncomment the following line after fixing all post processors
2278 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
2279 s
= s
.decode('ascii')
2280 return encodeFilename(s
, True)
2283 def decodeArgument(b
):
2284 return decodeFilename(b
, True)
2287 def decodeOption(optval
):
2290 if isinstance(optval
, bytes):
2291 optval
= optval
.decode(preferredencoding())
2293 assert isinstance(optval
, compat_str
)
2297 def formatSeconds(secs
, delim
=':'):
2299 return '%d%s%02d%s%02d' % (secs
// 3600, delim
, (secs
% 3600) // 60, delim
, secs
% 60)
2301 return '%d%s%02d' % (secs
// 60, delim
, secs
% 60)
2306 def make_HTTPS_handler(params
, **kwargs
):
2307 opts_no_check_certificate
= params
.get('nocheckcertificate', False)
2308 if hasattr(ssl
, 'create_default_context'): # Python >= 3.4 or 2.7.9
2309 context
= ssl
.create_default_context(ssl
.Purpose
.SERVER_AUTH
)
2310 if opts_no_check_certificate
:
2311 context
.check_hostname
= False
2312 context
.verify_mode
= ssl
.CERT_NONE
2314 return YoutubeDLHTTPSHandler(params
, context
=context
, **kwargs
)
2317 # (create_default_context present but HTTPSHandler has no context=)
2320 if sys
.version_info
< (3, 2):
2321 return YoutubeDLHTTPSHandler(params
, **kwargs
)
2322 else: # Python < 3.4
2323 context
= ssl
.SSLContext(ssl
.PROTOCOL_TLSv1
)
2324 context
.verify_mode
= (ssl
.CERT_NONE
2325 if opts_no_check_certificate
2326 else ssl
.CERT_REQUIRED
)
2327 context
.set_default_verify_paths()
2328 return YoutubeDLHTTPSHandler(params
, context
=context
, **kwargs
)
2331 def bug_reports_message():
2332 if ytdl_is_updateable():
2333 update_cmd
= 'type youtube-dlc -U to update'
2335 update_cmd
= 'see https://github.com/pukkandan/yt-dlp on how to update'
2336 msg
= '; please report this issue on https://github.com/pukkandan/yt-dlp .'
2337 msg
+= ' Make sure you are using the latest version; %s.' % update_cmd
2338 msg
+= ' Be sure to call youtube-dlc with the --verbose flag and include its complete output.'
2342 class YoutubeDLError(Exception):
2343 """Base exception for YoutubeDL errors."""
2347 class ExtractorError(YoutubeDLError
):
2348 """Error during info extraction."""
2350 def __init__(self
, msg
, tb
=None, expected
=False, cause
=None, video_id
=None):
2351 """ tb, if given, is the original traceback (so that it can be printed out).
2352 If expected is set, this is a normal error message and most likely not a bug in youtube-dlc.
2355 if sys
.exc_info()[0] in (compat_urllib_error
.URLError
, socket
.timeout
, UnavailableVideoError
):
2357 if video_id
is not None:
2358 msg
= video_id
+ ': ' + msg
2360 msg
+= ' (caused by %r)' % cause
2362 msg
+= bug_reports_message()
2363 super(ExtractorError
, self
).__init
__(msg
)
2366 self
.exc_info
= sys
.exc_info() # preserve original exception
2368 self
.video_id
= video_id
2370 def format_traceback(self
):
2371 if self
.traceback
is None:
2373 return ''.join(traceback
.format_tb(self
.traceback
))
2376 class UnsupportedError(ExtractorError
):
2377 def __init__(self
, url
):
2378 super(UnsupportedError
, self
).__init
__(
2379 'Unsupported URL: %s' % url
, expected
=True)
2383 class RegexNotFoundError(ExtractorError
):
2384 """Error when a regex didn't match"""
2388 class GeoRestrictedError(ExtractorError
):
2389 """Geographic restriction Error exception.
2391 This exception may be thrown when a video is not available from your
2392 geographic location due to geographic restrictions imposed by a website.
2395 def __init__(self
, msg
, countries
=None):
2396 super(GeoRestrictedError
, self
).__init
__(msg
, expected
=True)
2398 self
.countries
= countries
2401 class DownloadError(YoutubeDLError
):
2402 """Download Error exception.
2404 This exception may be thrown by FileDownloader objects if they are not
2405 configured to continue on errors. They will contain the appropriate
2409 def __init__(self
, msg
, exc_info
=None):
2410 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
2411 super(DownloadError
, self
).__init
__(msg
)
2412 self
.exc_info
= exc_info
2415 class SameFileError(YoutubeDLError
):
2416 """Same File exception.
2418 This exception will be thrown by FileDownloader objects if they detect
2419 multiple files would have to be downloaded to the same file on disk.
2424 class PostProcessingError(YoutubeDLError
):
2425 """Post Processing exception.
2427 This exception may be raised by PostProcessor's .run() method to
2428 indicate an error in the postprocessing task.
2431 def __init__(self
, msg
):
2432 super(PostProcessingError
, self
).__init
__(msg
)
2436 class ExistingVideoReached(YoutubeDLError
):
2437 """ --max-downloads limit has been reached. """
2441 class RejectedVideoReached(YoutubeDLError
):
2442 """ --max-downloads limit has been reached. """
2446 class MaxDownloadsReached(YoutubeDLError
):
2447 """ --max-downloads limit has been reached. """
2451 class UnavailableVideoError(YoutubeDLError
):
2452 """Unavailable Format exception.
2454 This exception will be thrown when a video is requested
2455 in a format that is not available for that video.
2460 class ContentTooShortError(YoutubeDLError
):
2461 """Content Too Short exception.
2463 This exception may be raised by FileDownloader objects when a file they
2464 download is too small for what the server announced first, indicating
2465 the connection was probably interrupted.
2468 def __init__(self
, downloaded
, expected
):
2469 super(ContentTooShortError
, self
).__init
__(
2470 'Downloaded {0} bytes, expected {1} bytes'.format(downloaded
, expected
)
2473 self
.downloaded
= downloaded
2474 self
.expected
= expected
2477 class XAttrMetadataError(YoutubeDLError
):
2478 def __init__(self
, code
=None, msg
='Unknown error'):
2479 super(XAttrMetadataError
, self
).__init
__(msg
)
2483 # Parsing code and msg
2484 if (self
.code
in (errno
.ENOSPC
, errno
.EDQUOT
)
2485 or 'No space left' in self
.msg
or 'Disk quota exceeded' in self
.msg
):
2486 self
.reason
= 'NO_SPACE'
2487 elif self
.code
== errno
.E2BIG
or 'Argument list too long' in self
.msg
:
2488 self
.reason
= 'VALUE_TOO_LONG'
2490 self
.reason
= 'NOT_SUPPORTED'
2493 class XAttrUnavailableError(YoutubeDLError
):
2497 def _create_http_connection(ydl_handler
, http_class
, is_https
, *args
, **kwargs
):
2498 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
2499 # expected HTTP responses to meet HTTP/1.0 or later (see also
2500 # https://github.com/ytdl-org/youtube-dl/issues/6727)
2501 if sys
.version_info
< (3, 0):
2502 kwargs
['strict'] = True
2503 hc
= http_class(*args
, **compat_kwargs(kwargs
))
2504 source_address
= ydl_handler
._params
.get('source_address')
2506 if source_address
is not None:
2507 # This is to workaround _create_connection() from socket where it will try all
2508 # address data from getaddrinfo() including IPv6. This filters the result from
2509 # getaddrinfo() based on the source_address value.
2510 # This is based on the cpython socket.create_connection() function.
2511 # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
2512 def _create_connection(address
, timeout
=socket
._GLOBAL
_DEFAULT
_TIMEOUT
, source_address
=None):
2513 host
, port
= address
2515 addrs
= socket
.getaddrinfo(host
, port
, 0, socket
.SOCK_STREAM
)
2516 af
= socket
.AF_INET
if '.' in source_address
[0] else socket
.AF_INET6
2517 ip_addrs
= [addr
for addr
in addrs
if addr
[0] == af
]
2518 if addrs
and not ip_addrs
:
2519 ip_version
= 'v4' if af
== socket
.AF_INET
else 'v6'
2521 "No remote IP%s addresses available for connect, can't use '%s' as source address"
2522 % (ip_version
, source_address
[0]))
2523 for res
in ip_addrs
:
2524 af
, socktype
, proto
, canonname
, sa
= res
2527 sock
= socket
.socket(af
, socktype
, proto
)
2528 if timeout
is not socket
._GLOBAL
_DEFAULT
_TIMEOUT
:
2529 sock
.settimeout(timeout
)
2530 sock
.bind(source_address
)
2532 err
= None # Explicitly break reference cycle
2534 except socket
.error
as _
:
2536 if sock
is not None:
2541 raise socket
.error('getaddrinfo returns an empty list')
2542 if hasattr(hc
, '_create_connection'):
2543 hc
._create
_connection
= _create_connection
2544 sa
= (source_address
, 0)
2545 if hasattr(hc
, 'source_address'): # Python 2.7+
2546 hc
.source_address
= sa
2548 def _hc_connect(self
, *args
, **kwargs
):
2549 sock
= _create_connection(
2550 (self
.host
, self
.port
), self
.timeout
, sa
)
2552 self
.sock
= ssl
.wrap_socket(
2553 sock
, self
.key_file
, self
.cert_file
,
2554 ssl_version
=ssl
.PROTOCOL_TLSv1
)
2557 hc
.connect
= functools
.partial(_hc_connect
, hc
)
2562 def handle_youtubedl_headers(headers
):
2563 filtered_headers
= headers
2565 if 'Youtubedl-no-compression' in filtered_headers
:
2566 filtered_headers
= dict((k
, v
) for k
, v
in filtered_headers
.items() if k
.lower() != 'accept-encoding')
2567 del filtered_headers
['Youtubedl-no-compression']
2569 return filtered_headers
2572 class YoutubeDLHandler(compat_urllib_request
.HTTPHandler
):
2573 """Handler for HTTP requests and responses.
2575 This class, when installed with an OpenerDirector, automatically adds
2576 the standard headers to every HTTP request and handles gzipped and
2577 deflated responses from web servers. If compression is to be avoided in
2578 a particular request, the original request in the program code only has
2579 to include the HTTP header "Youtubedl-no-compression", which will be
2580 removed before making the real request.
2582 Part of this code was copied from:
2584 http://techknack.net/python-urllib2-handlers/
2586 Andrew Rowls, the author of that code, agreed to release it to the
2590 def __init__(self
, params
, *args
, **kwargs
):
2591 compat_urllib_request
.HTTPHandler
.__init
__(self
, *args
, **kwargs
)
2592 self
._params
= params
2594 def http_open(self
, req
):
2595 conn_class
= compat_http_client
.HTTPConnection
2597 socks_proxy
= req
.headers
.get('Ytdl-socks-proxy')
2599 conn_class
= make_socks_conn_class(conn_class
, socks_proxy
)
2600 del req
.headers
['Ytdl-socks-proxy']
2602 return self
.do_open(functools
.partial(
2603 _create_http_connection
, self
, conn_class
, False),
2609 return zlib
.decompress(data
, -zlib
.MAX_WBITS
)
2611 return zlib
.decompress(data
)
2613 def http_request(self
, req
):
2614 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
2615 # always respected by websites, some tend to give out URLs with non percent-encoded
2616 # non-ASCII characters (see telemb.py, ard.py [#3412])
2617 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
2618 # To work around aforementioned issue we will replace request's original URL with
2619 # percent-encoded one
2620 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
2621 # the code of this workaround has been moved here from YoutubeDL.urlopen()
2622 url
= req
.get_full_url()
2623 url_escaped
= escape_url(url
)
2625 # Substitute URL if any change after escaping
2626 if url
!= url_escaped
:
2627 req
= update_Request(req
, url
=url_escaped
)
2629 for h
, v
in std_headers
.items():
2630 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
2631 # The dict keys are capitalized because of this bug by urllib
2632 if h
.capitalize() not in req
.headers
:
2633 req
.add_header(h
, v
)
2635 req
.headers
= handle_youtubedl_headers(req
.headers
)
2637 if sys
.version_info
< (2, 7) and '#' in req
.get_full_url():
2638 # Python 2.6 is brain-dead when it comes to fragments
2639 req
._Request
__original
= req
._Request
__original
.partition('#')[0]
2640 req
._Request
__r
_type
= req
._Request
__r
_type
.partition('#')[0]
2644 def http_response(self
, req
, resp
):
2647 if resp
.headers
.get('Content-encoding', '') == 'gzip':
2648 content
= resp
.read()
2649 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
), mode
='rb')
2651 uncompressed
= io
.BytesIO(gz
.read())
2652 except IOError as original_ioerror
:
2653 # There may be junk add the end of the file
2654 # See http://stackoverflow.com/q/4928560/35070 for details
2655 for i
in range(1, 1024):
2657 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
[:-i
]), mode
='rb')
2658 uncompressed
= io
.BytesIO(gz
.read())
2663 raise original_ioerror
2664 resp
= compat_urllib_request
.addinfourl(uncompressed
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
2665 resp
.msg
= old_resp
.msg
2666 del resp
.headers
['Content-encoding']
2668 if resp
.headers
.get('Content-encoding', '') == 'deflate':
2669 gz
= io
.BytesIO(self
.deflate(resp
.read()))
2670 resp
= compat_urllib_request
.addinfourl(gz
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
2671 resp
.msg
= old_resp
.msg
2672 del resp
.headers
['Content-encoding']
2673 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
2674 # https://github.com/ytdl-org/youtube-dl/issues/6457).
2675 if 300 <= resp
.code
< 400:
2676 location
= resp
.headers
.get('Location')
2678 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
2679 if sys
.version_info
>= (3, 0):
2680 location
= location
.encode('iso-8859-1').decode('utf-8')
2682 location
= location
.decode('utf-8')
2683 location_escaped
= escape_url(location
)
2684 if location
!= location_escaped
:
2685 del resp
.headers
['Location']
2686 if sys
.version_info
< (3, 0):
2687 location_escaped
= location_escaped
.encode('utf-8')
2688 resp
.headers
['Location'] = location_escaped
2691 https_request
= http_request
2692 https_response
= http_response
2695 def make_socks_conn_class(base_class
, socks_proxy
):
2696 assert issubclass(base_class
, (
2697 compat_http_client
.HTTPConnection
, compat_http_client
.HTTPSConnection
))
2699 url_components
= compat_urlparse
.urlparse(socks_proxy
)
2700 if url_components
.scheme
.lower() == 'socks5':
2701 socks_type
= ProxyType
.SOCKS5
2702 elif url_components
.scheme
.lower() in ('socks', 'socks4'):
2703 socks_type
= ProxyType
.SOCKS4
2704 elif url_components
.scheme
.lower() == 'socks4a':
2705 socks_type
= ProxyType
.SOCKS4A
2707 def unquote_if_non_empty(s
):
2710 return compat_urllib_parse_unquote_plus(s
)
2714 url_components
.hostname
, url_components
.port
or 1080,
2716 unquote_if_non_empty(url_components
.username
),
2717 unquote_if_non_empty(url_components
.password
),
2720 class SocksConnection(base_class
):
2722 self
.sock
= sockssocket()
2723 self
.sock
.setproxy(*proxy_args
)
2724 if type(self
.timeout
) in (int, float):
2725 self
.sock
.settimeout(self
.timeout
)
2726 self
.sock
.connect((self
.host
, self
.port
))
2728 if isinstance(self
, compat_http_client
.HTTPSConnection
):
2729 if hasattr(self
, '_context'): # Python > 2.6
2730 self
.sock
= self
._context
.wrap_socket(
2731 self
.sock
, server_hostname
=self
.host
)
2733 self
.sock
= ssl
.wrap_socket(self
.sock
)
2735 return SocksConnection
2738 class YoutubeDLHTTPSHandler(compat_urllib_request
.HTTPSHandler
):
2739 def __init__(self
, params
, https_conn_class
=None, *args
, **kwargs
):
2740 compat_urllib_request
.HTTPSHandler
.__init
__(self
, *args
, **kwargs
)
2741 self
._https
_conn
_class
= https_conn_class
or compat_http_client
.HTTPSConnection
2742 self
._params
= params
2744 def https_open(self
, req
):
2746 conn_class
= self
._https
_conn
_class
2748 if hasattr(self
, '_context'): # python > 2.6
2749 kwargs
['context'] = self
._context
2750 if hasattr(self
, '_check_hostname'): # python 3.x
2751 kwargs
['check_hostname'] = self
._check
_hostname
2753 socks_proxy
= req
.headers
.get('Ytdl-socks-proxy')
2755 conn_class
= make_socks_conn_class(conn_class
, socks_proxy
)
2756 del req
.headers
['Ytdl-socks-proxy']
2758 return self
.do_open(functools
.partial(
2759 _create_http_connection
, self
, conn_class
, True),
2763 class YoutubeDLCookieJar(compat_cookiejar
.MozillaCookieJar
):
2765 See [1] for cookie file format.
2767 1. https://curl.haxx.se/docs/http-cookies.html
2769 _HTTPONLY_PREFIX
= '#HttpOnly_'
2771 _HEADER
= '''# Netscape HTTP Cookie File
2772 # This file is generated by youtube-dlc. Do not edit.
2775 _CookieFileEntry
= collections
.namedtuple(
2777 ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))
2779 def save(self
, filename
=None, ignore_discard
=False, ignore_expires
=False):
2781 Save cookies to a file.
2783 Most of the code is taken from CPython 3.8 and slightly adapted
2784 to support cookie files with UTF-8 in both python 2 and 3.
2786 if filename
is None:
2787 if self
.filename
is not None:
2788 filename
= self
.filename
2790 raise ValueError(compat_cookiejar
.MISSING_FILENAME_TEXT
)
2792 # Store session cookies with `expires` set to 0 instead of an empty
2795 if cookie
.expires
is None:
2798 with io
.open(filename
, 'w', encoding
='utf-8') as f
:
2799 f
.write(self
._HEADER
)
2802 if not ignore_discard
and cookie
.discard
:
2804 if not ignore_expires
and cookie
.is_expired(now
):
2810 if cookie
.domain
.startswith('.'):
2811 initial_dot
= 'TRUE'
2813 initial_dot
= 'FALSE'
2814 if cookie
.expires
is not None:
2815 expires
= compat_str(cookie
.expires
)
2818 if cookie
.value
is None:
2819 # cookies.txt regards 'Set-Cookie: foo' as a cookie
2820 # with no name, whereas http.cookiejar regards it as a
2821 # cookie with no value.
2826 value
= cookie
.value
2828 '\t'.join([cookie
.domain
, initial_dot
, cookie
.path
,
2829 secure
, expires
, name
, value
]) + '\n')
2831 def load(self
, filename
=None, ignore_discard
=False, ignore_expires
=False):
2832 """Load cookies from a file."""
2833 if filename
is None:
2834 if self
.filename
is not None:
2835 filename
= self
.filename
2837 raise ValueError(compat_cookiejar
.MISSING_FILENAME_TEXT
)
2839 def prepare_line(line
):
2840 if line
.startswith(self
._HTTPONLY
_PREFIX
):
2841 line
= line
[len(self
._HTTPONLY
_PREFIX
):]
2842 # comments and empty lines are fine
2843 if line
.startswith('#') or not line
.strip():
2845 cookie_list
= line
.split('\t')
2846 if len(cookie_list
) != self
._ENTRY
_LEN
:
2847 raise compat_cookiejar
.LoadError('invalid length %d' % len(cookie_list
))
2848 cookie
= self
._CookieFileEntry
(*cookie_list
)
2849 if cookie
.expires_at
and not cookie
.expires_at
.isdigit():
2850 raise compat_cookiejar
.LoadError('invalid expires at %s' % cookie
.expires_at
)
2854 with io
.open(filename
, encoding
='utf-8') as f
:
2857 cf
.write(prepare_line(line
))
2858 except compat_cookiejar
.LoadError
as e
:
2860 'WARNING: skipping cookie file entry due to %s: %r\n'
2861 % (e
, line
), sys
.stderr
)
2864 self
._really
_load
(cf
, filename
, ignore_discard
, ignore_expires
)
2865 # Session cookies are denoted by either `expires` field set to
2866 # an empty string or 0. MozillaCookieJar only recognizes the former
2867 # (see [1]). So we need force the latter to be recognized as session
2868 # cookies on our own.
2869 # Session cookies may be important for cookies-based authentication,
2870 # e.g. usually, when user does not check 'Remember me' check box while
2871 # logging in on a site, some important cookies are stored as session
2872 # cookies so that not recognizing them will result in failed login.
2873 # 1. https://bugs.python.org/issue17164
2875 # Treat `expires=0` cookies as session cookies
2876 if cookie
.expires
== 0:
2877 cookie
.expires
= None
2878 cookie
.discard
= True
2881 class YoutubeDLCookieProcessor(compat_urllib_request
.HTTPCookieProcessor
):
2882 def __init__(self
, cookiejar
=None):
2883 compat_urllib_request
.HTTPCookieProcessor
.__init
__(self
, cookiejar
)
2885 def http_response(self
, request
, response
):
2886 # Python 2 will choke on next HTTP request in row if there are non-ASCII
2887 # characters in Set-Cookie HTTP header of last response (see
2888 # https://github.com/ytdl-org/youtube-dl/issues/6769).
2889 # In order to at least prevent crashing we will percent encode Set-Cookie
2890 # header before HTTPCookieProcessor starts processing it.
2891 # if sys.version_info < (3, 0) and response.headers:
2892 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
2893 # set_cookie = response.headers.get(set_cookie_header)
2895 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
2896 # if set_cookie != set_cookie_escaped:
2897 # del response.headers[set_cookie_header]
2898 # response.headers[set_cookie_header] = set_cookie_escaped
2899 return compat_urllib_request
.HTTPCookieProcessor
.http_response(self
, request
, response
)
2901 https_request
= compat_urllib_request
.HTTPCookieProcessor
.http_request
2902 https_response
= http_response
2905 class YoutubeDLRedirectHandler(compat_urllib_request
.HTTPRedirectHandler
):
2906 if sys
.version_info
[0] < 3:
2907 def redirect_request(self
, req
, fp
, code
, msg
, headers
, newurl
):
2908 # On python 2 urlh.geturl() may sometimes return redirect URL
2909 # as byte string instead of unicode. This workaround allows
2910 # to force it always return unicode.
2911 return compat_urllib_request
.HTTPRedirectHandler
.redirect_request(self
, req
, fp
, code
, msg
, headers
, compat_str(newurl
))
2914 def extract_timezone(date_str
):
2916 r
'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
2919 timezone
= datetime
.timedelta()
2921 date_str
= date_str
[:-len(m
.group('tz'))]
2922 if not m
.group('sign'):
2923 timezone
= datetime
.timedelta()
2925 sign
= 1 if m
.group('sign') == '+' else -1
2926 timezone
= datetime
.timedelta(
2927 hours
=sign
* int(m
.group('hours')),
2928 minutes
=sign
* int(m
.group('minutes')))
2929 return timezone
, date_str
2932 def parse_iso8601(date_str
, delimiter
='T', timezone
=None):
2933 """ Return a UNIX timestamp from the given date """
2935 if date_str
is None:
2938 date_str
= re
.sub(r
'\.[0-9]+', '', date_str
)
2940 if timezone
is None:
2941 timezone
, date_str
= extract_timezone(date_str
)
2944 date_format
= '%Y-%m-%d{0}%H:%M:%S'.format(delimiter
)
2945 dt
= datetime
.datetime
.strptime(date_str
, date_format
) - timezone
2946 return calendar
.timegm(dt
.timetuple())
2951 def date_formats(day_first
=True):
2952 return DATE_FORMATS_DAY_FIRST
if day_first
else DATE_FORMATS_MONTH_FIRST
2955 def unified_strdate(date_str
, day_first
=True):
2956 """Return a string with the date in the format YYYYMMDD"""
2958 if date_str
is None:
2962 date_str
= date_str
.replace(',', ' ')
2963 # Remove AM/PM + timezone
2964 date_str
= re
.sub(r
'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str
)
2965 _
, date_str
= extract_timezone(date_str
)
2967 for expression
in date_formats(day_first
):
2969 upload_date
= datetime
.datetime
.strptime(date_str
, expression
).strftime('%Y%m%d')
2972 if upload_date
is None:
2973 timetuple
= email
.utils
.parsedate_tz(date_str
)
2976 upload_date
= datetime
.datetime(*timetuple
[:6]).strftime('%Y%m%d')
2979 if upload_date
is not None:
2980 return compat_str(upload_date
)
2983 def unified_timestamp(date_str
, day_first
=True):
2984 if date_str
is None:
2987 date_str
= re
.sub(r
'[,|]', '', date_str
)
2989 pm_delta
= 12 if re
.search(r
'(?i)PM', date_str
) else 0
2990 timezone
, date_str
= extract_timezone(date_str
)
2992 # Remove AM/PM + timezone
2993 date_str
= re
.sub(r
'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str
)
2995 # Remove unrecognized timezones from ISO 8601 alike timestamps
2996 m
= re
.search(r
'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str
)
2998 date_str
= date_str
[:-len(m
.group('tz'))]
3000 # Python only supports microseconds, so remove nanoseconds
3001 m
= re
.search(r
'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str
)
3003 date_str
= m
.group(1)
3005 for expression
in date_formats(day_first
):
3007 dt
= datetime
.datetime
.strptime(date_str
, expression
) - timezone
+ datetime
.timedelta(hours
=pm_delta
)
3008 return calendar
.timegm(dt
.timetuple())
3011 timetuple
= email
.utils
.parsedate_tz(date_str
)
3013 return calendar
.timegm(timetuple
) + pm_delta
* 3600
3016 def determine_ext(url
, default_ext
='unknown_video'):
3017 if url
is None or '.' not in url
:
3019 guess
= url
.partition('?')[0].rpartition('.')[2]
3020 if re
.match(r
'^[A-Za-z0-9]+$', guess
):
3022 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
3023 elif guess
.rstrip('/') in KNOWN_EXTENSIONS
:
3024 return guess
.rstrip('/')
3029 def subtitles_filename(filename
, sub_lang
, sub_format
, expected_real_ext
=None):
3030 return replace_extension(filename
, sub_lang
+ '.' + sub_format
, expected_real_ext
)
3033 def date_from_str(date_str
):
3035 Return a datetime object from a string in the format YYYYMMDD or
3036 (now|today)[+-][0-9](day|week|month|year)(s)?"""
3037 today
= datetime
.date
.today()
3038 if date_str
in ('now', 'today'):
3040 if date_str
== 'yesterday':
3041 return today
- datetime
.timedelta(days
=1)
3042 match
= re
.match(r
'(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str
)
3043 if match
is not None:
3044 sign
= match
.group('sign')
3045 time
= int(match
.group('time'))
3048 unit
= match
.group('unit')
3049 # A bad approximation?
3053 elif unit
== 'year':
3057 delta
= datetime
.timedelta(**{unit: time}
)
3058 return today
+ delta
3059 return datetime
.datetime
.strptime(date_str
, '%Y%m%d').date()
3062 def hyphenate_date(date_str
):
3064 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
3065 match
= re
.match(r
'^(\d\d\d\d)(\d\d)(\d\d)$', date_str
)
3066 if match
is not None:
3067 return '-'.join(match
.groups())
3072 class DateRange(object):
3073 """Represents a time interval between two dates"""
3075 def __init__(self
, start
=None, end
=None):
3076 """start and end must be strings in the format accepted by date"""
3077 if start
is not None:
3078 self
.start
= date_from_str(start
)
3080 self
.start
= datetime
.datetime
.min.date()
3082 self
.end
= date_from_str(end
)
3084 self
.end
= datetime
.datetime
.max.date()
3085 if self
.start
> self
.end
:
3086 raise ValueError('Date range: "%s" , the start date must be before the end date' % self
)
3090 """Returns a range that only contains the given day"""
3091 return cls(day
, day
)
3093 def __contains__(self
, date
):
3094 """Check if the date is in the range"""
3095 if not isinstance(date
, datetime
.date
):
3096 date
= date_from_str(date
)
3097 return self
.start
<= date
<= self
.end
3100 return '%s - %s' % (self
.start
.isoformat(), self
.end
.isoformat())
3103 def platform_name():
3104 """ Returns the platform name as a compat_str """
3105 res
= platform
.platform()
3106 if isinstance(res
, bytes):
3107 res
= res
.decode(preferredencoding())
3109 assert isinstance(res
, compat_str
)
3113 def _windows_write_string(s
, out
):
3114 """ Returns True if the string was written using special methods,
3115 False if it has yet to be written out."""
3116 # Adapted from http://stackoverflow.com/a/3259271/35070
3119 import ctypes
.wintypes
3127 fileno
= out
.fileno()
3128 except AttributeError:
3129 # If the output stream doesn't have a fileno, it's virtual
3131 except io
.UnsupportedOperation
:
3132 # Some strange Windows pseudo files?
3134 if fileno
not in WIN_OUTPUT_IDS
:
3137 GetStdHandle
= compat_ctypes_WINFUNCTYPE(
3138 ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.DWORD
)(
3139 ('GetStdHandle', ctypes
.windll
.kernel32
))
3140 h
= GetStdHandle(WIN_OUTPUT_IDS
[fileno
])
3142 WriteConsoleW
= compat_ctypes_WINFUNCTYPE(
3143 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.LPWSTR
,
3144 ctypes
.wintypes
.DWORD
, ctypes
.POINTER(ctypes
.wintypes
.DWORD
),
3145 ctypes
.wintypes
.LPVOID
)(('WriteConsoleW', ctypes
.windll
.kernel32
))
3146 written
= ctypes
.wintypes
.DWORD(0)
3148 GetFileType
= compat_ctypes_WINFUNCTYPE(ctypes
.wintypes
.DWORD
, ctypes
.wintypes
.DWORD
)(('GetFileType', ctypes
.windll
.kernel32
))
3149 FILE_TYPE_CHAR
= 0x0002
3150 FILE_TYPE_REMOTE
= 0x8000
3151 GetConsoleMode
= compat_ctypes_WINFUNCTYPE(
3152 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
,
3153 ctypes
.POINTER(ctypes
.wintypes
.DWORD
))(
3154 ('GetConsoleMode', ctypes
.windll
.kernel32
))
3155 INVALID_HANDLE_VALUE
= ctypes
.wintypes
.DWORD(-1).value
3157 def not_a_console(handle
):
3158 if handle
== INVALID_HANDLE_VALUE
or handle
is None:
3160 return ((GetFileType(handle
) & ~FILE_TYPE_REMOTE
) != FILE_TYPE_CHAR
3161 or GetConsoleMode(handle
, ctypes
.byref(ctypes
.wintypes
.DWORD())) == 0)
3163 if not_a_console(h
):
3166 def next_nonbmp_pos(s
):
3168 return next(i
for i
, c
in enumerate(s
) if ord(c
) > 0xffff)
3169 except StopIteration:
3173 count
= min(next_nonbmp_pos(s
), 1024)
3175 ret
= WriteConsoleW(
3176 h
, s
, count
if count
else 2, ctypes
.byref(written
), None)
3178 raise OSError('Failed to write string')
3179 if not count
: # We just wrote a non-BMP character
3180 assert written
.value
== 2
3183 assert written
.value
> 0
3184 s
= s
[written
.value
:]
3188 def write_string(s
, out
=None, encoding
=None):
3191 assert type(s
) == compat_str
3193 if sys
.platform
== 'win32' and encoding
is None and hasattr(out
, 'fileno'):
3194 if _windows_write_string(s
, out
):
3197 if ('b' in getattr(out
, 'mode', '')
3198 or sys
.version_info
[0] < 3): # Python 2 lies about mode of sys.stderr
3199 byt
= s
.encode(encoding
or preferredencoding(), 'ignore')
3201 elif hasattr(out
, 'buffer'):
3202 enc
= encoding
or getattr(out
, 'encoding', None) or preferredencoding()
3203 byt
= s
.encode(enc
, 'ignore')
3204 out
.buffer.write(byt
)
3210 def bytes_to_intlist(bs
):
3213 if isinstance(bs
[0], int): # Python 3
3216 return [ord(c
) for c
in bs
]
3219 def intlist_to_bytes(xs
):
3222 return compat_struct_pack('%dB' % len(xs
), *xs
)
3225 # Cross-platform file locking
3226 if sys
.platform
== 'win32':
3227 import ctypes
.wintypes
3230 class OVERLAPPED(ctypes
.Structure
):
3232 ('Internal', ctypes
.wintypes
.LPVOID
),
3233 ('InternalHigh', ctypes
.wintypes
.LPVOID
),
3234 ('Offset', ctypes
.wintypes
.DWORD
),
3235 ('OffsetHigh', ctypes
.wintypes
.DWORD
),
3236 ('hEvent', ctypes
.wintypes
.HANDLE
),
3239 kernel32
= ctypes
.windll
.kernel32
3240 LockFileEx
= kernel32
.LockFileEx
3241 LockFileEx
.argtypes
= [
3242 ctypes
.wintypes
.HANDLE
, # hFile
3243 ctypes
.wintypes
.DWORD
, # dwFlags
3244 ctypes
.wintypes
.DWORD
, # dwReserved
3245 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
3246 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
3247 ctypes
.POINTER(OVERLAPPED
) # Overlapped
3249 LockFileEx
.restype
= ctypes
.wintypes
.BOOL
3250 UnlockFileEx
= kernel32
.UnlockFileEx
3251 UnlockFileEx
.argtypes
= [
3252 ctypes
.wintypes
.HANDLE
, # hFile
3253 ctypes
.wintypes
.DWORD
, # dwReserved
3254 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
3255 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
3256 ctypes
.POINTER(OVERLAPPED
) # Overlapped
3258 UnlockFileEx
.restype
= ctypes
.wintypes
.BOOL
3259 whole_low
= 0xffffffff
3260 whole_high
= 0x7fffffff
3262 def _lock_file(f
, exclusive
):
3263 overlapped
= OVERLAPPED()
3264 overlapped
.Offset
= 0
3265 overlapped
.OffsetHigh
= 0
3266 overlapped
.hEvent
= 0
3267 f
._lock
_file
_overlapped
_p
= ctypes
.pointer(overlapped
)
3268 handle
= msvcrt
.get_osfhandle(f
.fileno())
3269 if not LockFileEx(handle
, 0x2 if exclusive
else 0x0, 0,
3270 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
3271 raise OSError('Locking file failed: %r' % ctypes
.FormatError())
3273 def _unlock_file(f
):
3274 assert f
._lock
_file
_overlapped
_p
3275 handle
= msvcrt
.get_osfhandle(f
.fileno())
3276 if not UnlockFileEx(handle
, 0,
3277 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
3278 raise OSError('Unlocking file failed: %r' % ctypes
.FormatError())
3281 # Some platforms, such as Jython, is missing fcntl
3285 def _lock_file(f
, exclusive
):
3286 fcntl
.flock(f
, fcntl
.LOCK_EX
if exclusive
else fcntl
.LOCK_SH
)
3288 def _unlock_file(f
):
3289 fcntl
.flock(f
, fcntl
.LOCK_UN
)
3291 UNSUPPORTED_MSG
= 'file locking is not supported on this platform'
3293 def _lock_file(f
, exclusive
):
3294 raise IOError(UNSUPPORTED_MSG
)
3296 def _unlock_file(f
):
3297 raise IOError(UNSUPPORTED_MSG
)
3300 class locked_file(object):
3301 def __init__(self
, filename
, mode
, encoding
=None):
3302 assert mode
in ['r', 'a', 'w']
3303 self
.f
= io
.open(filename
, mode
, encoding
=encoding
)
3306 def __enter__(self
):
3307 exclusive
= self
.mode
!= 'r'
3309 _lock_file(self
.f
, exclusive
)
3315 def __exit__(self
, etype
, value
, traceback
):
3317 _unlock_file(self
.f
)
3324 def write(self
, *args
):
3325 return self
.f
.write(*args
)
3327 def read(self
, *args
):
3328 return self
.f
.read(*args
)
3331 def get_filesystem_encoding():
3332 encoding
= sys
.getfilesystemencoding()
3333 return encoding
if encoding
is not None else 'utf-8'
3336 def shell_quote(args
):
3338 encoding
= get_filesystem_encoding()
3340 if isinstance(a
, bytes):
3341 # We may get a filename encoded with 'encodeFilename'
3342 a
= a
.decode(encoding
)
3343 quoted_args
.append(compat_shlex_quote(a
))
3344 return ' '.join(quoted_args
)
3347 def smuggle_url(url
, data
):
3348 """ Pass additional data in a URL for internal use. """
3350 url
, idata
= unsmuggle_url(url
, {})
3352 sdata
= compat_urllib_parse_urlencode(
3353 {'__youtubedl_smuggle': json.dumps(data)}
)
3354 return url
+ '#' + sdata
3357 def unsmuggle_url(smug_url
, default
=None):
3358 if '#__youtubedl_smuggle' not in smug_url
:
3359 return smug_url
, default
3360 url
, _
, sdata
= smug_url
.rpartition('#')
3361 jsond
= compat_parse_qs(sdata
)['__youtubedl_smuggle'][0]
3362 data
= json
.loads(jsond
)
3366 def format_bytes(bytes):
3369 if type(bytes) is str:
3370 bytes = float(bytes)
3374 exponent
= int(math
.log(bytes, 1024.0))
3375 suffix
= ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent
]
3376 converted
= float(bytes) / float(1024 ** exponent
)
3377 return '%.2f%s' % (converted
, suffix
)
3380 def lookup_unit_table(unit_table
, s
):
3381 units_re
= '|'.join(re
.escape(u
) for u
in unit_table
)
3383 r
'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re
, s
)
3386 num_str
= m
.group('num').replace(',', '.')
3387 mult
= unit_table
[m
.group('unit')]
3388 return int(float(num_str
) * mult
)
3391 def parse_filesize(s
):
3395 # The lower-case forms are of course incorrect and unofficial,
3396 # but we support those too
3413 'megabytes': 1000 ** 2,
3414 'mebibytes': 1024 ** 2,
3420 'gigabytes': 1000 ** 3,
3421 'gibibytes': 1024 ** 3,
3427 'terabytes': 1000 ** 4,
3428 'tebibytes': 1024 ** 4,
3434 'petabytes': 1000 ** 5,
3435 'pebibytes': 1024 ** 5,
3441 'exabytes': 1000 ** 6,
3442 'exbibytes': 1024 ** 6,
3448 'zettabytes': 1000 ** 7,
3449 'zebibytes': 1024 ** 7,
3455 'yottabytes': 1000 ** 8,
3456 'yobibytes': 1024 ** 8,
3459 return lookup_unit_table(_UNIT_TABLE
, s
)
3468 if re
.match(r
'^[\d,.]+$', s
):
3469 return str_to_int(s
)
3480 return lookup_unit_table(_UNIT_TABLE
, s
)
3483 def parse_resolution(s
):
3487 mobj
= re
.search(r
'\b(?P<w>\d+)\s*[xX×]\s*(?P<h>\d+)\b', s
)
3490 'width': int(mobj
.group('w')),
3491 'height': int(mobj
.group('h')),
3494 mobj
= re
.search(r
'\b(\d+)[pPiI]\b', s
)
3496 return {'height': int(mobj.group(1))}
3498 mobj
= re
.search(r
'\b([48])[kK]\b', s
)
3500 return {'height': int(mobj.group(1)) * 540}
3505 def parse_bitrate(s
):
3506 if not isinstance(s
, compat_str
):
3508 mobj
= re
.search(r
'\b(\d+)\s*kbps', s
)
3510 return int(mobj
.group(1))
3513 def month_by_name(name
, lang
='en'):
3514 """ Return the number of a month by (locale-independently) English name """
3516 month_names
= MONTH_NAMES
.get(lang
, MONTH_NAMES
['en'])
3519 return month_names
.index(name
) + 1
3524 def month_by_abbreviation(abbrev
):
3525 """ Return the number of a month by (locale-independently) English
3529 return [s
[:3] for s
in ENGLISH_MONTH_NAMES
].index(abbrev
) + 1
3534 def fix_xml_ampersands(xml_str
):
3535 """Replace all the '&' by '&' in XML"""
3537 r
'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
3542 def setproctitle(title
):
3543 assert isinstance(title
, compat_str
)
3545 # ctypes in Jython is not complete
3546 # http://bugs.jython.org/issue2148
3547 if sys
.platform
.startswith('java'):
3551 libc
= ctypes
.cdll
.LoadLibrary('libc.so.6')
3555 # LoadLibrary in Windows Python 2.7.13 only expects
3556 # a bytestring, but since unicode_literals turns
3557 # every string into a unicode string, it fails.
3559 title_bytes
= title
.encode('utf-8')
3560 buf
= ctypes
.create_string_buffer(len(title_bytes
))
3561 buf
.value
= title_bytes
3563 libc
.prctl(15, buf
, 0, 0, 0)
3564 except AttributeError:
3565 return # Strange libc, just skip this
3568 def remove_start(s
, start
):
3569 return s
[len(start
):] if s
is not None and s
.startswith(start
) else s
3572 def remove_end(s
, end
):
3573 return s
[:-len(end
)] if s
is not None and s
.endswith(end
) else s
3576 def remove_quotes(s
):
3577 if s
is None or len(s
) < 2:
3579 for quote
in ('"', "'", ):
3580 if s
[0] == quote
and s
[-1] == quote
:
3585 def get_domain(url
):
3586 domain
= re
.match(r
'(?:https?:\/\/)?(?:www\.)?(?P<domain>[^\n\/]+\.[^\n\/]+)(?:\/(.*))?', url
)
3587 return domain
.group('domain') if domain
else None
3590 def url_basename(url
):
3591 path
= compat_urlparse
.urlparse(url
).path
3592 return path
.strip('/').split('/')[-1]
3596 return re
.match(r
'https?://[^?#&]+/', url
).group()
3599 def urljoin(base
, path
):
3600 if isinstance(path
, bytes):
3601 path
= path
.decode('utf-8')
3602 if not isinstance(path
, compat_str
) or not path
:
3604 if re
.match(r
'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path
):
3606 if isinstance(base
, bytes):
3607 base
= base
.decode('utf-8')
3608 if not isinstance(base
, compat_str
) or not re
.match(
3609 r
'^(?:https?:)?//', base
):
3611 return compat_urlparse
.urljoin(base
, path
)
3614 class HEADRequest(compat_urllib_request
.Request
):
3615 def get_method(self
):
3619 class PUTRequest(compat_urllib_request
.Request
):
3620 def get_method(self
):
3624 def int_or_none(v
, scale
=1, default
=None, get_attr
=None, invscale
=1):
3627 v
= getattr(v
, get_attr
, None)
3633 return int(v
) * invscale
// scale
3634 except (ValueError, TypeError):
3638 def str_or_none(v
, default
=None):
3639 return default
if v
is None else compat_str(v
)
3642 def str_to_int(int_str
):
3643 """ A more relaxed version of int_or_none """
3644 if isinstance(int_str
, compat_integer_types
):
3646 elif isinstance(int_str
, compat_str
):
3647 int_str
= re
.sub(r
'[,\.\+]', '', int_str
)
3648 return int_or_none(int_str
)
3651 def float_or_none(v
, scale
=1, invscale
=1, default
=None):
3655 return float(v
) * invscale
/ scale
3656 except (ValueError, TypeError):
3660 def bool_or_none(v
, default
=None):
3661 return v
if isinstance(v
, bool) else default
3664 def strip_or_none(v
, default
=None):
3665 return v
.strip() if isinstance(v
, compat_str
) else default
3668 def url_or_none(url
):
3669 if not url
or not isinstance(url
, compat_str
):
3672 return url
if re
.match(r
'^(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url
) else None
3675 def parse_duration(s
):
3676 if not isinstance(s
, compat_basestring
):
3681 days
, hours
, mins
, secs
, ms
= [None] * 5
3682 m
= re
.match(r
'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s
)
3684 days
, hours
, mins
, secs
, ms
= m
.groups()
3689 [0-9]+\s*y(?:ears?)?\s*
3692 [0-9]+\s*m(?:onths?)?\s*
3695 [0-9]+\s*w(?:eeks?)?\s*
3698 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
3702 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
3705 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
3708 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
3711 days
, hours
, mins
, secs
, ms
= m
.groups()
3713 m
= re
.match(r
'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s
)
3715 hours
, mins
= m
.groups()
3721 duration
+= float(secs
)
3723 duration
+= float(mins
) * 60
3725 duration
+= float(hours
) * 60 * 60
3727 duration
+= float(days
) * 24 * 60 * 60
3729 duration
+= float(ms
)
3733 def prepend_extension(filename
, ext
, expected_real_ext
=None):
3734 name
, real_ext
= os
.path
.splitext(filename
)
3736 '{0}.{1}{2}'.format(name
, ext
, real_ext
)
3737 if not expected_real_ext
or real_ext
[1:] == expected_real_ext
3738 else '{0}.{1}'.format(filename
, ext
))
3741 def replace_extension(filename
, ext
, expected_real_ext
=None):
3742 name
, real_ext
= os
.path
.splitext(filename
)
3743 return '{0}.{1}'.format(
3744 name
if not expected_real_ext
or real_ext
[1:] == expected_real_ext
else filename
,
3748 def check_executable(exe
, args
=[]):
3749 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
3750 args can be a list of arguments for a short output (like -version) """
3752 process_communicate_or_kill(subprocess
.Popen(
3753 [exe
] + args
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
))
3759 def get_exe_version(exe
, args
=['--version'],
3760 version_re
=None, unrecognized
='present'):
3761 """ Returns the version of the specified executable,
3762 or False if the executable is not present """
3764 # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
3765 # SIGTTOU if youtube-dlc is run in the background.
3766 # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
3767 out
, _
= process_communicate_or_kill(subprocess
.Popen(
3768 [encodeArgument(exe
)] + args
,
3769 stdin
=subprocess
.PIPE
,
3770 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
))
3773 if isinstance(out
, bytes): # Python 2.x
3774 out
= out
.decode('ascii', 'ignore')
3775 return detect_exe_version(out
, version_re
, unrecognized
)
3778 def detect_exe_version(output
, version_re
=None, unrecognized
='present'):
3779 assert isinstance(output
, compat_str
)
3780 if version_re
is None:
3781 version_re
= r
'version\s+([-0-9._a-zA-Z]+)'
3782 m
= re
.search(version_re
, output
)
3789 class PagedList(object):
3791 # This is only useful for tests
3792 return len(self
.getslice())
3795 class OnDemandPagedList(PagedList
):
3796 def __init__(self
, pagefunc
, pagesize
, use_cache
=True):
3797 self
._pagefunc
= pagefunc
3798 self
._pagesize
= pagesize
3799 self
._use
_cache
= use_cache
3803 def getslice(self
, start
=0, end
=None):
3805 for pagenum
in itertools
.count(start
// self
._pagesize
):
3806 firstid
= pagenum
* self
._pagesize
3807 nextfirstid
= pagenum
* self
._pagesize
+ self
._pagesize
3808 if start
>= nextfirstid
:
3813 page_results
= self
._cache
.get(pagenum
)
3814 if page_results
is None:
3815 page_results
= list(self
._pagefunc
(pagenum
))
3817 self
._cache
[pagenum
] = page_results
3820 start
% self
._pagesize
3821 if firstid
<= start
< nextfirstid
3825 ((end
- 1) % self
._pagesize
) + 1
3826 if (end
is not None and firstid
<= end
<= nextfirstid
)
3829 if startv
!= 0 or endv
is not None:
3830 page_results
= page_results
[startv
:endv
]
3831 res
.extend(page_results
)
3833 # A little optimization - if current page is not "full", ie. does
3834 # not contain page_size videos then we can assume that this page
3835 # is the last one - there are no more ids on further pages -
3836 # i.e. no need to query again.
3837 if len(page_results
) + startv
< self
._pagesize
:
3840 # If we got the whole page, but the next page is not interesting,
3841 # break out early as well
3842 if end
== nextfirstid
:
3847 class InAdvancePagedList(PagedList
):
3848 def __init__(self
, pagefunc
, pagecount
, pagesize
):
3849 self
._pagefunc
= pagefunc
3850 self
._pagecount
= pagecount
3851 self
._pagesize
= pagesize
3853 def getslice(self
, start
=0, end
=None):
3855 start_page
= start
// self
._pagesize
3857 self
._pagecount
if end
is None else (end
// self
._pagesize
+ 1))
3858 skip_elems
= start
- start_page
* self
._pagesize
3859 only_more
= None if end
is None else end
- start
3860 for pagenum
in range(start_page
, end_page
):
3861 page
= list(self
._pagefunc
(pagenum
))
3863 page
= page
[skip_elems
:]
3865 if only_more
is not None:
3866 if len(page
) < only_more
:
3867 only_more
-= len(page
)
3869 page
= page
[:only_more
]
3876 def uppercase_escape(s
):
3877 unicode_escape
= codecs
.getdecoder('unicode_escape')
3879 r
'\\U[0-9a-fA-F]{8}',
3880 lambda m
: unicode_escape(m
.group(0))[0],
3884 def lowercase_escape(s
):
3885 unicode_escape
= codecs
.getdecoder('unicode_escape')
3887 r
'\\u[0-9a-fA-F]{4}',
3888 lambda m
: unicode_escape(m
.group(0))[0],
3892 def escape_rfc3986(s
):
3893 """Escape non-ASCII characters as suggested by RFC 3986"""
3894 if sys
.version_info
< (3, 0) and isinstance(s
, compat_str
):
3895 s
= s
.encode('utf-8')
3896 return compat_urllib_parse
.quote(s
, b
"%/;:@&=+$,!~*'()?#[]")
3899 def escape_url(url
):
3900 """Escape URL as suggested by RFC 3986"""
3901 url_parsed
= compat_urllib_parse_urlparse(url
)
3902 return url_parsed
._replace
(
3903 netloc
=url_parsed
.netloc
.encode('idna').decode('ascii'),
3904 path
=escape_rfc3986(url_parsed
.path
),
3905 params
=escape_rfc3986(url_parsed
.params
),
3906 query
=escape_rfc3986(url_parsed
.query
),
3907 fragment
=escape_rfc3986(url_parsed
.fragment
)
3911 def read_batch_urls(batch_fd
):
3913 if not isinstance(url
, compat_str
):
3914 url
= url
.decode('utf-8', 'replace')
3915 BOM_UTF8
= ('\xef\xbb\xbf', '\ufeff')
3916 for bom
in BOM_UTF8
:
3917 if url
.startswith(bom
):
3918 url
= url
[len(bom
):]
3920 if not url
or url
.startswith(('#', ';', ']')):
3922 # "#" cannot be stripped out since it is part of the URI
3923 # However, it can be safely stipped out if follwing a whitespace
3924 return re
.split(r
'\s#', url
, 1)[0].rstrip()
3926 with contextlib
.closing(batch_fd
) as fd
:
3927 return [url
for url
in map(fixup
, fd
) if url
]
3930 def urlencode_postdata(*args
, **kargs
):
3931 return compat_urllib_parse_urlencode(*args
, **kargs
).encode('ascii')
3934 def update_url_query(url
, query
):
3937 parsed_url
= compat_urlparse
.urlparse(url
)
3938 qs
= compat_parse_qs(parsed_url
.query
)
3940 return compat_urlparse
.urlunparse(parsed_url
._replace
(
3941 query
=compat_urllib_parse_urlencode(qs
, True)))
3944 def update_Request(req
, url
=None, data
=None, headers
={}, query={}
):
3945 req_headers
= req
.headers
.copy()
3946 req_headers
.update(headers
)
3947 req_data
= data
or req
.data
3948 req_url
= update_url_query(url
or req
.get_full_url(), query
)
3949 req_get_method
= req
.get_method()
3950 if req_get_method
== 'HEAD':
3951 req_type
= HEADRequest
3952 elif req_get_method
== 'PUT':
3953 req_type
= PUTRequest
3955 req_type
= compat_urllib_request
.Request
3957 req_url
, data
=req_data
, headers
=req_headers
,
3958 origin_req_host
=req
.origin_req_host
, unverifiable
=req
.unverifiable
)
3959 if hasattr(req
, 'timeout'):
3960 new_req
.timeout
= req
.timeout
3964 def _multipart_encode_impl(data
, boundary
):
3965 content_type
= 'multipart/form-data; boundary=%s' % boundary
3968 for k
, v
in data
.items():
3969 out
+= b
'--' + boundary
.encode('ascii') + b
'\r\n'
3970 if isinstance(k
, compat_str
):
3971 k
= k
.encode('utf-8')
3972 if isinstance(v
, compat_str
):
3973 v
= v
.encode('utf-8')
3974 # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
3975 # suggests sending UTF-8 directly. Firefox sends UTF-8, too
3976 content
= b
'Content-Disposition: form-data; name="' + k
+ b
'"\r\n\r\n' + v
+ b
'\r\n'
3977 if boundary
.encode('ascii') in content
:
3978 raise ValueError('Boundary overlaps with data')
3981 out
+= b
'--' + boundary
.encode('ascii') + b
'--\r\n'
3983 return out
, content_type
3986 def multipart_encode(data
, boundary
=None):
3988 Encode a dict to RFC 7578-compliant form-data
3991 A dict where keys and values can be either Unicode or bytes-like
3994 If specified a Unicode object, it's used as the boundary. Otherwise
3995 a random boundary is generated.
3997 Reference: https://tools.ietf.org/html/rfc7578
3999 has_specified_boundary
= boundary
is not None
4002 if boundary
is None:
4003 boundary
= '---------------' + str(random
.randrange(0x0fffffff, 0xffffffff))
4006 out
, content_type
= _multipart_encode_impl(data
, boundary
)
4009 if has_specified_boundary
:
4013 return out
, content_type
4016 def dict_get(d
, key_or_keys
, default
=None, skip_false_values
=True):
4017 if isinstance(key_or_keys
, (list, tuple)):
4018 for key
in key_or_keys
:
4019 if key
not in d
or d
[key
] is None or skip_false_values
and not d
[key
]:
4023 return d
.get(key_or_keys
, default
)
4026 def try_get(src
, getter
, expected_type
=None):
4027 if not isinstance(getter
, (list, tuple)):
4032 except (AttributeError, KeyError, TypeError, IndexError):
4035 if expected_type
is None or isinstance(v
, expected_type
):
4039 def merge_dicts(*dicts
):
4041 for a_dict
in dicts
:
4042 for k
, v
in a_dict
.items():
4046 or (isinstance(v
, compat_str
) and v
4047 and isinstance(merged
[k
], compat_str
)
4048 and not merged
[k
])):
4053 def encode_compat_str(string
, encoding
=preferredencoding(), errors
='strict'):
4054 return string
if isinstance(string
, compat_str
) else compat_str(string
, encoding
, errors
)
4066 TV_PARENTAL_GUIDELINES
= {
4076 def parse_age_limit(s
):
4078 return s
if 0 <= s
<= 21 else None
4079 if not isinstance(s
, compat_basestring
):
4081 m
= re
.match(r
'^(?P<age>\d{1,2})\+?$', s
)
4083 return int(m
.group('age'))
4085 return US_RATINGS
[s
]
4086 m
= re
.match(r
'^TV[_-]?(%s)$' % '|'.join(k
[3:] for k
in TV_PARENTAL_GUIDELINES
), s
)
4088 return TV_PARENTAL_GUIDELINES
['TV-' + m
.group(1)]
4092 def strip_jsonp(code
):
4095 (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
4096 (?:\s*&&\s*(?P=func_name))?
4097 \s*\(\s*(?P<callback_data>.*)\);?
4098 \s*?(?://[^\n]*)*$''',
4099 r
'\g<callback_data>', code
)
4102 def js_to_json(code
):
4103 COMMENT_RE
= r
'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
4104 SKIP_RE
= r
'\s*(?:{comment})?\s*'.format(comment
=COMMENT_RE
)
4106 (r
'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip
=SKIP_RE
), 16),
4107 (r
'(?s)^(0+[0-7]+){skip}:?$'.format(skip
=SKIP_RE
), 8),
4112 if v
in ('true', 'false', 'null'):
4114 elif v
.startswith('/*') or v
.startswith('//') or v
.startswith('!') or v
== ',':
4117 if v
[0] in ("'", '"'):
4118 v
= re
.sub(r
'(?s)\\.|"', lambda m
: {
4123 }.get(m
.group(0), m
.group(0)), v
[1:-1])
4125 for regex
, base
in INTEGER_TABLE
:
4126 im
= re
.match(regex
, v
)
4128 i
= int(im
.group(1), base
)
4129 return '"%d":' % i
if v
.endswith(':') else '%d' % i
4133 return re
.sub(r
'''(?sx)
4134 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
4135 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
4136 {comment}|,(?={skip}[\]}}])|
4137 (?:(?<![0-9])[eE]|[a-df-zA-DF-Z_])[.a-zA-Z_0-9]*|
4138 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
4141 '''.format(comment
=COMMENT_RE
, skip
=SKIP_RE
), fix_kv
, code
)
4144 def qualities(quality_ids
):
4145 """ Get a numeric quality value out of a list of possible values """
4148 return quality_ids
.index(qid
)
4154 DEFAULT_OUTTMPL
= '%(title)s [%(id)s].%(ext)s'
4157 def limit_length(s
, length
):
4158 """ Add ellipses to overly long strings """
4163 return s
[:length
- len(ELLIPSES
)] + ELLIPSES
4167 def version_tuple(v
):
4168 return tuple(int(e
) for e
in re
.split(r
'[-.]', v
))
4171 def is_outdated_version(version
, limit
, assume_new
=True):
4173 return not assume_new
4175 return version_tuple(version
) < version_tuple(limit
)
4177 return not assume_new
4180 def ytdl_is_updateable():
4181 """ Returns if youtube-dlc can be updated with -U """
4184 from zipimport
import zipimporter
4186 return isinstance(globals().get('__loader__'), zipimporter
) or hasattr(sys
, 'frozen')
4189 def args_to_str(args
):
4190 # Get a short string representation for a subprocess command
4191 return ' '.join(compat_shlex_quote(a
) for a
in args
)
4194 def error_to_compat_str(err
):
4196 # On python 2 error byte string must be decoded with proper
4197 # encoding rather than ascii
4198 if sys
.version_info
[0] < 3:
4199 err_str
= err_str
.decode(preferredencoding())
4203 def mimetype2ext(mt
):
4209 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
4210 # it's the most popular one
4211 'audio/mpeg': 'mp3',
4212 'audio/x-wav': 'wav',
4217 _
, _
, res
= mt
.rpartition('/')
4218 res
= res
.split(';')[0].strip().lower()
4222 'smptett+xml': 'tt',
4226 'x-mp4-fragmented': 'mp4',
4227 'x-ms-sami': 'sami',
4230 'x-mpegurl': 'm3u8',
4231 'vnd.apple.mpegurl': 'm3u8',
4235 'vnd.ms-sstr+xml': 'ism',
4242 def parse_codecs(codecs_str
):
4243 # http://tools.ietf.org/html/rfc6381
4246 split_codecs
= list(filter(None, map(
4247 lambda str: str.strip(), codecs_str
.strip().strip(',').split(','))))
4248 vcodec
, acodec
= None, None
4249 for full_codec
in split_codecs
:
4250 codec
= full_codec
.split('.')[0]
4251 if codec
in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'):
4254 elif codec
in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
4258 write_string('WARNING: Unknown codec %s\n' % full_codec
, sys
.stderr
)
4259 if not vcodec
and not acodec
:
4260 if len(split_codecs
) == 2:
4262 'vcodec': split_codecs
[0],
4263 'acodec': split_codecs
[1],
4267 'vcodec': vcodec
or 'none',
4268 'acodec': acodec
or 'none',
4273 def urlhandle_detect_ext(url_handle
):
4274 getheader
= url_handle
.headers
.get
4276 cd
= getheader('Content-Disposition')
4278 m
= re
.match(r
'attachment;\s*filename="(?P<filename>[^"]+)"', cd
)
4280 e
= determine_ext(m
.group('filename'), default_ext
=None)
4284 return mimetype2ext(getheader('Content-Type'))
4287 def encode_data_uri(data
, mime_type
):
4288 return 'data:%s;base64,%s' % (mime_type
, base64
.b64encode(data
).decode('ascii'))
4291 def age_restricted(content_limit
, age_limit
):
4292 """ Returns True iff the content should be blocked """
4294 if age_limit
is None: # No limit set
4296 if content_limit
is None:
4297 return False # Content available for everyone
4298 return age_limit
< content_limit
4301 def is_html(first_bytes
):
4302 """ Detect whether a file contains HTML by examining its first bytes. """
4305 (b
'\xef\xbb\xbf', 'utf-8'),
4306 (b
'\x00\x00\xfe\xff', 'utf-32-be'),
4307 (b
'\xff\xfe\x00\x00', 'utf-32-le'),
4308 (b
'\xff\xfe', 'utf-16-le'),
4309 (b
'\xfe\xff', 'utf-16-be'),
4311 for bom
, enc
in BOMS
:
4312 if first_bytes
.startswith(bom
):
4313 s
= first_bytes
[len(bom
):].decode(enc
, 'replace')
4316 s
= first_bytes
.decode('utf-8', 'replace')
4318 return re
.match(r
'^\s*<', s
)
4321 def determine_protocol(info_dict
):
4322 protocol
= info_dict
.get('protocol')
4323 if protocol
is not None:
4326 url
= info_dict
['url']
4327 if url
.startswith('rtmp'):
4329 elif url
.startswith('mms'):
4331 elif url
.startswith('rtsp'):
4334 ext
= determine_ext(url
)
4340 return compat_urllib_parse_urlparse(url
).scheme
4343 def render_table(header_row
, data
, delim
=False, extraGap
=0, hideEmpty
=False):
4344 """ Render a list of rows, each as a list of values """
4346 def get_max_lens(table
):
4347 return [max(len(compat_str(v
)) for v
in col
) for col
in zip(*table
)]
4349 def filter_using_list(row
, filterArray
):
4350 return [col
for (take
, col
) in zip(filterArray
, row
) if take
]
4353 max_lens
= get_max_lens(data
)
4354 header_row
= filter_using_list(header_row
, max_lens
)
4355 data
= [filter_using_list(row
, max_lens
) for row
in data
]
4357 table
= [header_row
] + data
4358 max_lens
= get_max_lens(table
)
4360 table
= [header_row
] + [['-' * ml
for ml
in max_lens
]] + data
4361 format_str
= ' '.join('%-' + compat_str(ml
+ extraGap
) + 's' for ml
in max_lens
[:-1]) + ' %s'
4362 return '\n'.join(format_str
% tuple(row
) for row
in table
)
4365 def _match_one(filter_part
, dct
):
4366 COMPARISON_OPERATORS
= {
4374 operator_rex
= re
.compile(r
'''(?x)\s*
4376 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
4378 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
4379 (?P<quote>["\'])(?P
<quotedstrval
>(?
:\\.|
(?
!(?P
=quote
)|
\\).)+?
)(?P
=quote
)|
4380 (?P
<strval
>(?
![0-9.])[a
-z0
-9A
-Z
]*)
4383 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
4384 m = operator_rex.search(filter_part)
4386 op = COMPARISON_OPERATORS[m.group('op')]
4387 actual_value = dct.get(m.group('key'))
4388 if (m.group('quotedstrval') is not None
4389 or m.group('strval') is not None
4390 # If the original field is a string and matching comparisonvalue is
4391 # a number we should respect the origin of the original field
4392 # and process comparison value as a string (see
4393 # https://github.com/ytdl-org/youtube-dl/issues/11082).
4394 or actual_value is not None and m.group('intval') is not None
4395 and isinstance(actual_value, compat_str)):
4396 if m.group('op') not in ('=', '!='):
4398 'Operator %s does not support string values!' % m.group('op'))
4399 comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
4400 quote = m.group('quote')
4401 if quote is not None:
4402 comparison_value = comparison_value.replace(r'\%s' % quote, quote)
4405 comparison_value = int(m.group('intval'))
4407 comparison_value = parse_filesize(m.group('intval'))
4408 if comparison_value is None:
4409 comparison_value = parse_filesize(m.group('intval') + 'B')
4410 if comparison_value is None:
4412 'Invalid integer value %r in filter part %r' % (
4413 m.group('intval'), filter_part))
4414 if actual_value is None:
4415 return m.group('none_inclusive')
4416 return op(actual_value, comparison_value)
4419 '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
4420 '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
4422 operator_rex = re.compile(r'''(?x
)\s
*
4423 (?P
<op
>%s)\s
*(?P
<key
>[a
-z_
]+)
4425 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
4426 m = operator_rex.search(filter_part)
4428 op = UNARY_OPERATORS[m.group('op')]
4429 actual_value = dct.get(m.group('key'))
4430 return op(actual_value)
4432 raise ValueError('Invalid filter part %r' % filter_part)
4435 def match_str(filter_str, dct):
4436 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
4439 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
4442 def match_filter_func(filter_str):
4443 def _match_func(info_dict):
4444 if match_str(filter_str, info_dict):
4447 video_title = info_dict.get('title', info_dict.get('id', 'video'))
4448 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
4452 def parse_dfxp_time_expr(time_expr):
4456 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
4458 return float(mobj.group('time_offset'))
4460 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
4462 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
4465 def srt_subtitles_timecode(seconds):
4466 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
4469 def dfxp2srt(dfxp_data):
4471 @param dfxp_data A
bytes-like
object containing DFXP data
4472 @returns A
unicode object containing converted SRT data
4474 LEGACY_NAMESPACES = (
4475 (b'http://www.w3.org/ns/ttml', [
4476 b'http://www.w3.org/2004/11/ttaf1',
4477 b'http://www.w3.org/2006/04/ttaf1',
4478 b'http://www.w3.org/2006/10/ttaf1',
4480 (b'http://www.w3.org/ns/ttml#styling', [
4481 b'http://www.w3.org/ns/ttml#style',
4485 SUPPORTED_STYLING = [
4494 _x = functools.partial(xpath_with_ns, ns_map={
4495 'xml': 'http://www.w3.org/XML/1998/namespace',
4496 'ttml': 'http://www.w3.org/ns/ttml',
4497 'tts': 'http://www.w3.org/ns/ttml#styling',
4503 class TTMLPElementParser(object):
4505 _unclosed_elements = []
4506 _applied_styles = []
4508 def start(self, tag, attrib):
4509 if tag in (_x('ttml:br'), 'br'):
4512 unclosed_elements = []
4514 element_style_id = attrib.get('style')
4516 style.update(default_style)
4517 if element_style_id:
4518 style.update(styles.get(element_style_id, {}))
4519 for prop in SUPPORTED_STYLING:
4520 prop_val = attrib.get(_x('tts:' + prop))
4522 style[prop] = prop_val
4525 for k, v in sorted(style.items()):
4526 if self._applied_styles and self._applied_styles[-1].get(k) == v:
4529 font += ' color="%s"' % v
4530 elif k == 'fontSize':
4531 font += ' size="%s"' % v
4532 elif k == 'fontFamily':
4533 font += ' face="%s"' % v
4534 elif k == 'fontWeight' and v == 'bold':
4536 unclosed_elements.append('b')
4537 elif k == 'fontStyle' and v == 'italic':
4539 unclosed_elements.append('i')
4540 elif k == 'textDecoration' and v == 'underline':
4542 unclosed_elements.append('u')
4544 self._out += '<font' + font + '>'
4545 unclosed_elements.append('font')
4547 if self._applied_styles:
4548 applied_style.update(self._applied_styles[-1])
4549 applied_style.update(style)
4550 self._applied_styles.append(applied_style)
4551 self._unclosed_elements.append(unclosed_elements)
4554 if tag not in (_x('ttml:br'), 'br'):
4555 unclosed_elements = self._unclosed_elements.pop()
4556 for element in reversed(unclosed_elements):
4557 self._out += '</%s>' % element
4558 if unclosed_elements and self._applied_styles:
4559 self._applied_styles.pop()
4561 def data(self, data):
4565 return self._out.strip()
4567 def parse_node(node):
4568 target = TTMLPElementParser()
4569 parser = xml.etree.ElementTree.XMLParser(target=target)
4570 parser.feed(xml.etree.ElementTree.tostring(node))
4571 return parser.close()
4573 for k, v in LEGACY_NAMESPACES:
4575 dfxp_data = dfxp_data.replace(ns, k)
4577 dfxp = compat_etree_fromstring(dfxp_data)
4579 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
4582 raise ValueError('Invalid dfxp/TTML subtitle')
4586 for style in dfxp.findall(_x('.//ttml:style')):
4587 style_id = style.get('id') or style.get(_x('xml:id'))
4590 parent_style_id = style.get('style')
4592 if parent_style_id not in styles:
4595 styles[style_id] = styles[parent_style_id].copy()
4596 for prop in SUPPORTED_STYLING:
4597 prop_val = style.get(_x('tts:' + prop))
4599 styles.setdefault(style_id, {})[prop] = prop_val
4605 for p in ('body', 'div'):
4606 ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
4609 style = styles.get(ele.get('style'))
4612 default_style.update(style)
4614 for para, index in zip(paras, itertools.count(1)):
4615 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
4616 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
4617 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
4618 if begin_time is None:
4623 end_time = begin_time + dur
4624 out.append('%d\n%s --> %s\n%s\n\n' % (
4626 srt_subtitles_timecode(begin_time),
4627 srt_subtitles_timecode(end_time),
4633 def cli_option(params, command_option, param):
4634 param = params.get(param)
4636 param = compat_str(param)
4637 return [command_option, param] if param is not None else []
4640 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
4641 param = params.get(param)
4644 assert isinstance(param, bool)
4646 return [command_option + separator + (true_value if param else false_value)]
4647 return [command_option, true_value if param else false_value]
4650 def cli_valueless_option(params, command_option, param, expected_value=True):
4651 param = params.get(param)
4652 return [command_option] if param == expected_value else []
4655 def cli_configuration_args(params, param, default=[]):
4656 ex_args = params.get(param)
4659 assert isinstance(ex_args, list)
4663 class ISO639Utils(object):
4664 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
4723 'iw': 'heb', # Replaced by he in 1989 revision
4733 'in': 'ind', # Replaced by id in 1989 revision
4848 'ji': 'yid', # Replaced by yi in 1989 revision
4856 def short2long(cls, code):
4857 """Convert language code from ISO 639-1 to ISO 639-2/T"""
4858 return cls._lang_map.get(code[:2])
4861 def long2short(cls, code):
4862 """Convert language code from ISO 639-2/T to ISO 639-1"""
4863 for short_name, long_name in cls._lang_map.items():
4864 if long_name == code:
4868 class ISO3166Utils(object):
4869 # From http://data.okfn.org/data/core/country-list
4871 'AF': 'Afghanistan',
4872 'AX': 'Åland Islands',
4875 'AS': 'American Samoa',
4880 'AG': 'Antigua and Barbuda',
4897 'BO': 'Bolivia, Plurinational State of',
4898 'BQ': 'Bonaire, Sint Eustatius and Saba',
4899 'BA': 'Bosnia and Herzegovina',
4901 'BV': 'Bouvet Island',
4903 'IO': 'British Indian Ocean Territory',
4904 'BN': 'Brunei Darussalam',
4906 'BF': 'Burkina Faso',
4912 'KY': 'Cayman Islands',
4913 'CF': 'Central African Republic',
4917 'CX': 'Christmas Island',
4918 'CC': 'Cocos (Keeling) Islands',
4922 'CD': 'Congo, the Democratic Republic of the',
4923 'CK': 'Cook Islands',
4925 'CI': 'Côte d\'Ivoire',
4930 'CZ': 'Czech Republic',
4934 'DO': 'Dominican Republic',
4937 'SV': 'El Salvador',
4938 'GQ': 'Equatorial Guinea',
4942 'FK': 'Falkland Islands (Malvinas)',
4943 'FO': 'Faroe Islands',
4947 'GF': 'French Guiana',
4948 'PF': 'French Polynesia',
4949 'TF': 'French Southern Territories',
4964 'GW': 'Guinea-Bissau',
4967 'HM': 'Heard Island and McDonald Islands',
4968 'VA': 'Holy See (Vatican City State)',
4975 'IR': 'Iran, Islamic Republic of',
4978 'IM': 'Isle of Man',
4988 'KP': 'Korea, Democratic People\'s Republic of',
4989 'KR': 'Korea, Republic of',
4992 'LA': 'Lao People\'s Democratic Republic',
4998 'LI': 'Liechtenstein',
5002 'MK': 'Macedonia, the Former Yugoslav Republic of',
5009 'MH': 'Marshall Islands',
5015 'FM': 'Micronesia, Federated States of',
5016 'MD': 'Moldova, Republic of',
5027 'NL': 'Netherlands',
5028 'NC': 'New Caledonia',
5029 'NZ': 'New Zealand',
5034 'NF': 'Norfolk Island',
5035 'MP': 'Northern Mariana Islands',
5040 'PS': 'Palestine, State of',
5042 'PG': 'Papua New Guinea',
5045 'PH': 'Philippines',
5049 'PR': 'Puerto Rico',
5053 'RU': 'Russian Federation',
5055 'BL': 'Saint Barthélemy',
5056 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
5057 'KN': 'Saint Kitts and Nevis',
5058 'LC': 'Saint Lucia',
5059 'MF': 'Saint Martin (French part)',
5060 'PM': 'Saint Pierre and Miquelon',
5061 'VC': 'Saint Vincent and the Grenadines',
5064 'ST': 'Sao Tome and Principe',
5065 'SA': 'Saudi Arabia',
5069 'SL': 'Sierra Leone',
5071 'SX': 'Sint Maarten (Dutch part)',
5074 'SB': 'Solomon Islands',
5076 'ZA': 'South Africa',
5077 'GS': 'South Georgia and the South Sandwich Islands',
5078 'SS': 'South Sudan',
5083 'SJ': 'Svalbard and Jan Mayen',
5086 'CH': 'Switzerland',
5087 'SY': 'Syrian Arab Republic',
5088 'TW': 'Taiwan, Province of China',
5090 'TZ': 'Tanzania, United Republic of',
5092 'TL': 'Timor-Leste',
5096 'TT': 'Trinidad and Tobago',
5099 'TM': 'Turkmenistan',
5100 'TC': 'Turks and Caicos Islands',
5104 'AE': 'United Arab Emirates',
5105 'GB': 'United Kingdom',
5106 'US': 'United States',
5107 'UM': 'United States Minor Outlying Islands',
5111 'VE': 'Venezuela, Bolivarian Republic of',
5113 'VG': 'Virgin Islands, British',
5114 'VI': 'Virgin Islands, U.S.',
5115 'WF': 'Wallis and Futuna',
5116 'EH': 'Western Sahara',
5123 def short2full(cls, code):
5124 """Convert an ISO 3166-2 country code to the corresponding full name"""
5125 return cls._country_map.get(code.upper())
5128 class GeoUtils(object):
5129 # Major IPv4 address blocks per country
5131 'AD': '46.172.224.0/19',
5132 'AE': '94.200.0.0/13',
5133 'AF': '149.54.0.0/17',
5134 'AG': '209.59.64.0/18',
5135 'AI': '204.14.248.0/21',
5136 'AL': '46.99.0.0/16',
5137 'AM': '46.70.0.0/15',
5138 'AO': '105.168.0.0/13',
5139 'AP': '182.50.184.0/21',
5140 'AQ': '23.154.160.0/24',
5141 'AR': '181.0.0.0/12',
5142 'AS': '202.70.112.0/20',
5143 'AT': '77.116.0.0/14',
5144 'AU': '1.128.0.0/11',
5145 'AW': '181.41.0.0/18',
5146 'AX': '185.217.4.0/22',
5147 'AZ': '5.197.0.0/16',
5148 'BA': '31.176.128.0/17',
5149 'BB': '65.48.128.0/17',
5150 'BD': '114.130.0.0/16',
5152 'BF': '102.178.0.0/15',
5153 'BG': '95.42.0.0/15',
5154 'BH': '37.131.0.0/17',
5155 'BI': '154.117.192.0/18',
5156 'BJ': '137.255.0.0/16',
5157 'BL': '185.212.72.0/23',
5158 'BM': '196.12.64.0/18',
5159 'BN': '156.31.0.0/16',
5160 'BO': '161.56.0.0/16',
5161 'BQ': '161.0.80.0/20',
5162 'BR': '191.128.0.0/12',
5163 'BS': '24.51.64.0/18',
5164 'BT': '119.2.96.0/19',
5165 'BW': '168.167.0.0/16',
5166 'BY': '178.120.0.0/13',
5167 'BZ': '179.42.192.0/18',
5168 'CA': '99.224.0.0/11',
5169 'CD': '41.243.0.0/16',
5170 'CF': '197.242.176.0/21',
5171 'CG': '160.113.0.0/16',
5172 'CH': '85.0.0.0/13',
5173 'CI': '102.136.0.0/14',
5174 'CK': '202.65.32.0/19',
5175 'CL': '152.172.0.0/14',
5176 'CM': '102.244.0.0/14',
5177 'CN': '36.128.0.0/10',
5178 'CO': '181.240.0.0/12',
5179 'CR': '201.192.0.0/12',
5180 'CU': '152.206.0.0/15',
5181 'CV': '165.90.96.0/19',
5182 'CW': '190.88.128.0/17',
5183 'CY': '31.153.0.0/16',
5184 'CZ': '88.100.0.0/14',
5186 'DJ': '197.241.0.0/17',
5187 'DK': '87.48.0.0/12',
5188 'DM': '192.243.48.0/20',
5189 'DO': '152.166.0.0/15',
5190 'DZ': '41.96.0.0/12',
5191 'EC': '186.68.0.0/15',
5192 'EE': '90.190.0.0/15',
5193 'EG': '156.160.0.0/11',
5194 'ER': '196.200.96.0/20',
5195 'ES': '88.0.0.0/11',
5196 'ET': '196.188.0.0/14',
5197 'EU': '2.16.0.0/13',
5198 'FI': '91.152.0.0/13',
5199 'FJ': '144.120.0.0/16',
5200 'FK': '80.73.208.0/21',
5201 'FM': '119.252.112.0/20',
5202 'FO': '88.85.32.0/19',
5204 'GA': '41.158.0.0/15',
5206 'GD': '74.122.88.0/21',
5207 'GE': '31.146.0.0/16',
5208 'GF': '161.22.64.0/18',
5209 'GG': '62.68.160.0/19',
5210 'GH': '154.160.0.0/12',
5211 'GI': '95.164.0.0/16',
5212 'GL': '88.83.0.0/19',
5213 'GM': '160.182.0.0/15',
5214 'GN': '197.149.192.0/18',
5215 'GP': '104.250.0.0/19',
5216 'GQ': '105.235.224.0/20',
5217 'GR': '94.64.0.0/13',
5218 'GT': '168.234.0.0/16',
5219 'GU': '168.123.0.0/16',
5220 'GW': '197.214.80.0/20',
5221 'GY': '181.41.64.0/18',
5222 'HK': '113.252.0.0/14',
5223 'HN': '181.210.0.0/16',
5224 'HR': '93.136.0.0/13',
5225 'HT': '148.102.128.0/17',
5226 'HU': '84.0.0.0/14',
5227 'ID': '39.192.0.0/10',
5228 'IE': '87.32.0.0/12',
5229 'IL': '79.176.0.0/13',
5230 'IM': '5.62.80.0/20',
5231 'IN': '117.192.0.0/10',
5232 'IO': '203.83.48.0/21',
5233 'IQ': '37.236.0.0/14',
5234 'IR': '2.176.0.0/12',
5235 'IS': '82.221.0.0/16',
5236 'IT': '79.0.0.0/10',
5237 'JE': '87.244.64.0/18',
5238 'JM': '72.27.0.0/17',
5239 'JO': '176.29.0.0/16',
5240 'JP': '133.0.0.0/8',
5241 'KE': '105.48.0.0/12',
5242 'KG': '158.181.128.0/17',
5243 'KH': '36.37.128.0/17',
5244 'KI': '103.25.140.0/22',
5245 'KM': '197.255.224.0/20',
5246 'KN': '198.167.192.0/19',
5247 'KP': '175.45.176.0/22',
5248 'KR': '175.192.0.0/10',
5249 'KW': '37.36.0.0/14',
5250 'KY': '64.96.0.0/15',
5251 'KZ': '2.72.0.0/13',
5252 'LA': '115.84.64.0/18',
5253 'LB': '178.135.0.0/16',
5254 'LC': '24.92.144.0/20',
5255 'LI': '82.117.0.0/19',
5256 'LK': '112.134.0.0/15',
5257 'LR': '102.183.0.0/16',
5258 'LS': '129.232.0.0/17',
5259 'LT': '78.56.0.0/13',
5260 'LU': '188.42.0.0/16',
5261 'LV': '46.109.0.0/16',
5262 'LY': '41.252.0.0/14',
5263 'MA': '105.128.0.0/11',
5264 'MC': '88.209.64.0/18',
5265 'MD': '37.246.0.0/16',
5266 'ME': '178.175.0.0/17',
5267 'MF': '74.112.232.0/21',
5268 'MG': '154.126.0.0/17',
5269 'MH': '117.103.88.0/21',
5270 'MK': '77.28.0.0/15',
5271 'ML': '154.118.128.0/18',
5272 'MM': '37.111.0.0/17',
5273 'MN': '49.0.128.0/17',
5274 'MO': '60.246.0.0/16',
5275 'MP': '202.88.64.0/20',
5276 'MQ': '109.203.224.0/19',
5277 'MR': '41.188.64.0/18',
5278 'MS': '208.90.112.0/22',
5279 'MT': '46.11.0.0/16',
5280 'MU': '105.16.0.0/12',
5281 'MV': '27.114.128.0/18',
5282 'MW': '102.70.0.0/15',
5283 'MX': '187.192.0.0/11',
5284 'MY': '175.136.0.0/13',
5285 'MZ': '197.218.0.0/15',
5286 'NA': '41.182.0.0/16',
5287 'NC': '101.101.0.0/18',
5288 'NE': '197.214.0.0/18',
5289 'NF': '203.17.240.0/22',
5290 'NG': '105.112.0.0/12',
5291 'NI': '186.76.0.0/15',
5292 'NL': '145.96.0.0/11',
5293 'NO': '84.208.0.0/13',
5294 'NP': '36.252.0.0/15',
5295 'NR': '203.98.224.0/19',
5296 'NU': '49.156.48.0/22',
5297 'NZ': '49.224.0.0/14',
5298 'OM': '5.36.0.0/15',
5299 'PA': '186.72.0.0/15',
5300 'PE': '186.160.0.0/14',
5301 'PF': '123.50.64.0/18',
5302 'PG': '124.240.192.0/19',
5303 'PH': '49.144.0.0/13',
5304 'PK': '39.32.0.0/11',
5305 'PL': '83.0.0.0/11',
5306 'PM': '70.36.0.0/20',
5307 'PR': '66.50.0.0/16',
5308 'PS': '188.161.0.0/16',
5309 'PT': '85.240.0.0/13',
5310 'PW': '202.124.224.0/20',
5311 'PY': '181.120.0.0/14',
5312 'QA': '37.210.0.0/15',
5313 'RE': '102.35.0.0/16',
5314 'RO': '79.112.0.0/13',
5315 'RS': '93.86.0.0/15',
5316 'RU': '5.136.0.0/13',
5317 'RW': '41.186.0.0/16',
5318 'SA': '188.48.0.0/13',
5319 'SB': '202.1.160.0/19',
5320 'SC': '154.192.0.0/11',
5321 'SD': '102.120.0.0/13',
5322 'SE': '78.64.0.0/12',
5323 'SG': '8.128.0.0/10',
5324 'SI': '188.196.0.0/14',
5325 'SK': '78.98.0.0/15',
5326 'SL': '102.143.0.0/17',
5327 'SM': '89.186.32.0/19',
5328 'SN': '41.82.0.0/15',
5329 'SO': '154.115.192.0/18',
5330 'SR': '186.179.128.0/17',
5331 'SS': '105.235.208.0/21',
5332 'ST': '197.159.160.0/19',
5333 'SV': '168.243.0.0/16',
5334 'SX': '190.102.0.0/20',
5336 'SZ': '41.84.224.0/19',
5337 'TC': '65.255.48.0/20',
5338 'TD': '154.68.128.0/19',
5339 'TG': '196.168.0.0/14',
5340 'TH': '171.96.0.0/13',
5341 'TJ': '85.9.128.0/18',
5342 'TK': '27.96.24.0/21',
5343 'TL': '180.189.160.0/20',
5344 'TM': '95.85.96.0/19',
5345 'TN': '197.0.0.0/11',
5346 'TO': '175.176.144.0/21',
5347 'TR': '78.160.0.0/11',
5348 'TT': '186.44.0.0/15',
5349 'TV': '202.2.96.0/19',
5350 'TW': '120.96.0.0/11',
5351 'TZ': '156.156.0.0/14',
5352 'UA': '37.52.0.0/14',
5353 'UG': '102.80.0.0/13',
5355 'UY': '167.56.0.0/13',
5356 'UZ': '84.54.64.0/18',
5357 'VA': '212.77.0.0/19',
5358 'VC': '207.191.240.0/21',
5359 'VE': '186.88.0.0/13',
5360 'VG': '66.81.192.0/20',
5361 'VI': '146.226.0.0/16',
5362 'VN': '14.160.0.0/11',
5363 'VU': '202.80.32.0/20',
5364 'WF': '117.20.32.0/21',
5365 'WS': '202.4.32.0/19',
5366 'YE': '134.35.0.0/16',
5367 'YT': '41.242.116.0/22',
5368 'ZA': '41.0.0.0/11',
5369 'ZM': '102.144.0.0/13',
5370 'ZW': '102.177.192.0/18',
5374 def random_ipv4(cls, code_or_block):
5375 if len(code_or_block) == 2:
5376 block = cls._country_ip_map.get(code_or_block.upper())
5380 block = code_or_block
5381 addr, preflen = block.split('/')
5382 addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
5383 addr_max = addr_min | (0xffffffff >> int(preflen))
5384 return compat_str(socket.inet_ntoa(
5385 compat_struct_pack('!L', random.randint(addr_min, addr_max))))
5388 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
5389 def __init__(self, proxies=None):
5390 # Set default handlers
5391 for type in ('http', 'https'):
5392 setattr(self, '%s_open' % type,
5393 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
5394 meth(r, proxy, type))
5395 compat_urllib_request.ProxyHandler.__init__(self, proxies)
5397 def proxy_open(self, req, proxy, type):
5398 req_proxy = req.headers.get('Ytdl-request-proxy')
5399 if req_proxy is not None:
5401 del req.headers['Ytdl-request-proxy']
5403 if proxy == '__noproxy__':
5404 return None # No Proxy
5405 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
5406 req.add_header('Ytdl-socks-proxy', proxy)
5407 # youtube-dlc's http/https handlers do wrapping the socket with socks
5409 return compat_urllib_request.ProxyHandler.proxy_open(
5410 self, req, proxy, type)
5413 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
5414 # released into Public Domain
5415 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
5417 def long_to_bytes(n, blocksize=0):
5418 """long_to_bytes(n:long, blocksize:int) : string
5419 Convert a long integer to a byte string.
5421 If optional blocksize is given and greater than zero, pad the front of the
5422 byte string with binary zeros so that the length is a multiple of
5425 # after much testing, this algorithm was deemed to be the fastest
5429 s = compat_struct_pack('>I', n & 0xffffffff) + s
5431 # strip off leading zeros
5432 for i in range(len(s)):
5433 if s[i] != b'\000'[0]:
5436 # only happens when n == 0
5440 # add back some pad bytes. this could be done more efficiently w.r.t. the
5441 # de-padding being done above, but sigh...
5442 if blocksize > 0 and len(s) % blocksize:
5443 s = (blocksize - len(s) % blocksize) * b'\000' + s
5447 def bytes_to_long(s):
5448 """bytes_to_long(string) : long
5449 Convert a byte string to a long integer.
5451 This is (essentially) the inverse of long_to_bytes().
5456 extra = (4 - length % 4)
5457 s = b'\000' * extra + s
5458 length = length + extra
5459 for i in range(0, length, 4):
5460 acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
5464 def ohdave_rsa_encrypt(data, exponent, modulus):
5466 Implement OHDave
's RSA algorithm. See http://www.ohdave.com/rsa/
5469 data: data to encrypt, bytes-like object
5470 exponent, modulus: parameter e and N of RSA algorithm, both integer
5471 Output: hex string of encrypted data
5473 Limitation: supports one block encryption only
5476 payload = int(binascii.hexlify(data[::-1]), 16)
5477 encrypted = pow(payload, exponent, modulus)
5478 return '%x' % encrypted
5481 def pkcs1pad(data, length):
5483 Padding input data with PKCS#1 scheme
5485 @param {int[]} data input data
5486 @param {int} length target length
5487 @returns {int[]} padded data
5489 if len(data) > length - 11:
5490 raise ValueError('Input data too
long for PKCS
#1 padding')
5492 pseudo_random
= [random
.randint(0, 254) for _
in range(length
- len(data
) - 3)]
5493 return [0, 2] + pseudo_random
+ [0] + data
5496 def encode_base_n(num
, n
, table
=None):
5497 FULL_TABLE
= '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
5499 table
= FULL_TABLE
[:n
]
5502 raise ValueError('base %d exceeds table length %d' % (n
, len(table
)))
5509 ret
= table
[num
% n
] + ret
5514 def decode_packed_codes(code
):
5515 mobj
= re
.search(PACKED_CODES_RE
, code
)
5516 obfuscated_code
, base
, count
, symbols
= mobj
.groups()
5519 symbols
= symbols
.split('|')
5524 base_n_count
= encode_base_n(count
, base
)
5525 symbol_table
[base_n_count
] = symbols
[count
] or base_n_count
5528 r
'\b(\w+)\b', lambda mobj
: symbol_table
[mobj
.group(0)],
5532 def caesar(s
, alphabet
, shift
):
5537 alphabet
[(alphabet
.index(c
) + shift
) % l
] if c
in alphabet
else c
5542 return caesar(s
, r
'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''', 47)
5545 def parse_m3u8_attributes(attrib
):
5547 for (key
, val
) in re
.findall(r
'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib
):
5548 if val
.startswith('"'):
5554 def urshift(val
, n
):
5555 return val
>> n
if val
>= 0 else (val
+ 0x100000000) >> n
5558 # Based on png2str() written by @gdkchan and improved by @yokrysty
5559 # Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
5560 def decode_png(png_data
):
5561 # Reference: https://www.w3.org/TR/PNG/
5562 header
= png_data
[8:]
5564 if png_data
[:8] != b
'\x89PNG\x0d\x0a\x1a\x0a' or header
[4:8] != b
'IHDR':
5565 raise IOError('Not a valid PNG file.')
5567 int_map
= {1: '>B', 2: '>H', 4: '>I'}
5568 unpack_integer
= lambda x
: compat_struct_unpack(int_map
[len(x
)], x
)[0]
5573 length
= unpack_integer(header
[:4])
5576 chunk_type
= header
[:4]
5579 chunk_data
= header
[:length
]
5580 header
= header
[length
:]
5582 header
= header
[4:] # Skip CRC
5590 ihdr
= chunks
[0]['data']
5592 width
= unpack_integer(ihdr
[:4])
5593 height
= unpack_integer(ihdr
[4:8])
5597 for chunk
in chunks
:
5598 if chunk
['type'] == b
'IDAT':
5599 idat
+= chunk
['data']
5602 raise IOError('Unable to read PNG data.')
5604 decompressed_data
= bytearray(zlib
.decompress(idat
))
5609 def _get_pixel(idx
):
5614 for y
in range(height
):
5615 basePos
= y
* (1 + stride
)
5616 filter_type
= decompressed_data
[basePos
]
5620 pixels
.append(current_row
)
5622 for x
in range(stride
):
5623 color
= decompressed_data
[1 + basePos
+ x
]
5624 basex
= y
* stride
+ x
5629 left
= _get_pixel(basex
- 3)
5631 up
= _get_pixel(basex
- stride
)
5633 if filter_type
== 1: # Sub
5634 color
= (color
+ left
) & 0xff
5635 elif filter_type
== 2: # Up
5636 color
= (color
+ up
) & 0xff
5637 elif filter_type
== 3: # Average
5638 color
= (color
+ ((left
+ up
) >> 1)) & 0xff
5639 elif filter_type
== 4: # Paeth
5645 c
= _get_pixel(basex
- stride
- 3)
5653 if pa
<= pb
and pa
<= pc
:
5654 color
= (color
+ a
) & 0xff
5656 color
= (color
+ b
) & 0xff
5658 color
= (color
+ c
) & 0xff
5660 current_row
.append(color
)
5662 return width
, height
, pixels
5665 def write_xattr(path
, key
, value
):
5666 # This mess below finds the best xattr tool for the job
5668 # try the pyxattr module...
5671 if hasattr(xattr
, 'set'): # pyxattr
5672 # Unicode arguments are not supported in python-pyxattr until
5674 # See https://github.com/ytdl-org/youtube-dl/issues/5498
5675 pyxattr_required_version
= '0.5.0'
5676 if version_tuple(xattr
.__version
__) < version_tuple(pyxattr_required_version
):
5677 # TODO: fallback to CLI tools
5678 raise XAttrUnavailableError(
5679 'python-pyxattr is detected but is too old. '
5680 'youtube-dlc requires %s or above while your version is %s. '
5681 'Falling back to other xattr implementations' % (
5682 pyxattr_required_version
, xattr
.__version
__))
5684 setxattr
= xattr
.set
5686 setxattr
= xattr
.setxattr
5689 setxattr(path
, key
, value
)
5690 except EnvironmentError as e
:
5691 raise XAttrMetadataError(e
.errno
, e
.strerror
)
5694 if compat_os_name
== 'nt':
5695 # Write xattrs to NTFS Alternate Data Streams:
5696 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
5697 assert ':' not in key
5698 assert os
.path
.exists(path
)
5700 ads_fn
= path
+ ':' + key
5702 with open(ads_fn
, 'wb') as f
:
5704 except EnvironmentError as e
:
5705 raise XAttrMetadataError(e
.errno
, e
.strerror
)
5707 user_has_setfattr
= check_executable('setfattr', ['--version'])
5708 user_has_xattr
= check_executable('xattr', ['-h'])
5710 if user_has_setfattr
or user_has_xattr
:
5712 value
= value
.decode('utf-8')
5713 if user_has_setfattr
:
5714 executable
= 'setfattr'
5715 opts
= ['-n', key
, '-v', value
]
5716 elif user_has_xattr
:
5717 executable
= 'xattr'
5718 opts
= ['-w', key
, value
]
5720 cmd
= ([encodeFilename(executable
, True)]
5721 + [encodeArgument(o
) for o
in opts
]
5722 + [encodeFilename(path
, True)])
5725 p
= subprocess
.Popen(
5726 cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, stdin
=subprocess
.PIPE
)
5727 except EnvironmentError as e
:
5728 raise XAttrMetadataError(e
.errno
, e
.strerror
)
5729 stdout
, stderr
= process_communicate_or_kill(p
)
5730 stderr
= stderr
.decode('utf-8', 'replace')
5731 if p
.returncode
!= 0:
5732 raise XAttrMetadataError(p
.returncode
, stderr
)
5735 # On Unix, and can't find pyxattr, setfattr, or xattr.
5736 if sys
.platform
.startswith('linux'):
5737 raise XAttrUnavailableError(
5738 "Couldn't find a tool to set the xattrs. "
5739 "Install either the python 'pyxattr' or 'xattr' "
5740 "modules, or the GNU 'attr' package "
5741 "(which contains the 'setfattr' tool).")
5743 raise XAttrUnavailableError(
5744 "Couldn't find a tool to set the xattrs. "
5745 "Install either the python 'xattr' module, "
5746 "or the 'xattr' binary.")
5749 def random_birthday(year_field
, month_field
, day_field
):
5750 start_date
= datetime
.date(1950, 1, 1)
5751 end_date
= datetime
.date(1995, 12, 31)
5752 offset
= random
.randint(0, (end_date
- start_date
).days
)
5753 random_date
= start_date
+ datetime
.timedelta(offset
)
5755 year_field
: str(random_date
.year
),
5756 month_field
: str(random_date
.month
),
5757 day_field
: str(random_date
.day
),
5761 # Templates for internet shortcut files, which are plain text files.
5762 DOT_URL_LINK_TEMPLATE
= '''
5767 DOT_WEBLOC_LINK_TEMPLATE
= '''
5768 <?xml version="1.0" encoding="UTF-8"?>
5769 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
5770 <plist version="1.0">
5773 \t<string>%(url)s</string>
5778 DOT_DESKTOP_LINK_TEMPLATE
= '''
5788 def iri_to_uri(iri
):
5790 Converts an IRI (Internationalized Resource Identifier, allowing Unicode characters) to a URI (Uniform Resource Identifier, ASCII-only).
5792 The function doesn't add an additional layer of escaping; e.g., it doesn't escape `%3C` as `%253C`. Instead, it percent-escapes characters with an underlying UTF-8 encoding *besides* those already escaped, leaving the URI intact.
5795 iri_parts
= compat_urllib_parse_urlparse(iri
)
5797 if '[' in iri_parts
.netloc
:
5798 raise ValueError('IPv6 URIs are not, yet, supported.')
5799 # Querying `.netloc`, when there's only one bracket, also raises a ValueError.
5801 # The `safe` argument values, that the following code uses, contain the characters that should not be percent-encoded. Everything else but letters, digits and '_.-' will be percent-encoded with an underlying UTF-8 encoding. Everything already percent-encoded will be left as is.
5804 if iri_parts
.username
:
5805 net_location
+= compat_urllib_parse_quote(iri_parts
.username
, safe
=r
"!$%&'()*+,~")
5806 if iri_parts
.password
is not None:
5807 net_location
+= ':' + compat_urllib_parse_quote(iri_parts
.password
, safe
=r
"!$%&'()*+,~")
5810 net_location
+= iri_parts
.hostname
.encode('idna').decode('utf-8') # Punycode for Unicode hostnames.
5811 # The 'idna' encoding produces ASCII text.
5812 if iri_parts
.port
is not None and iri_parts
.port
!= 80:
5813 net_location
+= ':' + str(iri_parts
.port
)
5815 return compat_urllib_parse_urlunparse(
5819 compat_urllib_parse_quote_plus(iri_parts
.path
, safe
=r
"!$%&'()*+,/:;=@|~"),
5821 # Unsure about the `safe` argument, since this is a legacy way of handling parameters.
5822 compat_urllib_parse_quote_plus(iri_parts
.params
, safe
=r
"!$%&'()*+,/:;=@|~"),
5824 # Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component.
5825 compat_urllib_parse_quote_plus(iri_parts
.query
, safe
=r
"!$%&'()*+,/:;=?@{|}~"),
5827 compat_urllib_parse_quote_plus(iri_parts
.fragment
, safe
=r
"!#$%&'()*+,/:;=?@{|}~")))
5829 # Source for `safe` arguments: https://url.spec.whatwg.org/#percent-encoded-bytes.
5832 def to_high_limit_path(path
):
5833 if sys
.platform
in ['win32', 'cygwin']:
5834 # Work around MAX_PATH limitation on Windows. The maximum allowed length for the individual path segments may still be quite limited.
5835 return r
'\\?\ '.rstrip() + os
.path
.abspath(path
)
5840 def format_field(obj
, field
, template
='%s', ignore
=(None, ''), default
='', func
=None):
5841 val
= obj
.get(field
, default
)
5842 if func
and val
not in ignore
:
5844 return template
% val
if val
not in ignore
else default
5847 def clean_podcast_url(url
):
5848 return re
.sub(r
'''(?x)
5852 media\.blubrry\.com| # https://create.blubrry.com/resources/podcast-media-download-statistics/getting-started/
5855 (?:dts|www)\.podtrac\.com/(?:pts/)?redirect\.[0-9a-z]{3,4}| # http://analytics.podtrac.com/how-to-measure
5858 cn\.co| # https://podcorn.com/analytics-prefix/
5859 st\.fm # https://podsights.com/docs/
projects / yt-dlp.git / blob