2 # -*- coding: utf-8 -*-
4 from __future__
import unicode_literals
33 import xml
.etree
.ElementTree
40 compat_etree_fromstring
,
42 compat_html_entities_html5
,
47 compat_socket_create_connection
,
53 compat_urllib_parse_urlencode
,
54 compat_urllib_parse_urlparse
,
55 compat_urllib_parse_unquote_plus
,
56 compat_urllib_request
,
67 def register_socks_protocols():
68 # "Register" SOCKS protocols
69 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
70 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
71 for scheme
in ('socks', 'socks4', 'socks4a', 'socks5'):
72 if scheme
not in compat_urlparse
.uses_netloc
:
73 compat_urlparse
.uses_netloc
.append(scheme
)
76 # This is not clearly defined otherwise
77 compiled_regex_type
= type(re
.compile(''))
80 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)',
81 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
82 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
83 'Accept-Encoding': 'gzip, deflate',
84 'Accept-Language': 'en-us,en;q=0.5',
90 ENGLISH_MONTH_NAMES
= [
91 'January', 'February', 'March', 'April', 'May', 'June',
92 'July', 'August', 'September', 'October', 'November', 'December']
95 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
96 'flv', 'f4v', 'f4a', 'f4b',
97 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
107 'f4f', 'f4m', 'm3u8', 'smil')
109 # needed for sanitizing filenames in restricted mode
110 ACCENT_CHARS
= dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
111 itertools
.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
112 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
127 '%Y-%m-%d %H:%M:%S.%f',
130 '%Y-%m-%dT%H:%M:%SZ',
131 '%Y-%m-%dT%H:%M:%S.%fZ',
132 '%Y-%m-%dT%H:%M:%S.%f0Z',
134 '%Y-%m-%dT%H:%M:%S.%f',
138 DATE_FORMATS_DAY_FIRST
= list(DATE_FORMATS
)
139 DATE_FORMATS_DAY_FIRST
.extend([
148 DATE_FORMATS_MONTH_FIRST
= list(DATE_FORMATS
)
149 DATE_FORMATS_MONTH_FIRST
.extend([
158 def preferredencoding():
159 """Get preferred encoding.
161 Returns the best encoding scheme for the system, based on
162 locale.getpreferredencoding() and some further tweaks.
165 pref
= locale
.getpreferredencoding()
173 def write_json_file(obj
, fn
):
174 """ Encode obj as JSON and write it to fn, atomically if possible """
176 fn
= encodeFilename(fn
)
177 if sys
.version_info
< (3, 0) and sys
.platform
!= 'win32':
178 encoding
= get_filesystem_encoding()
179 # os.path.basename returns a bytes object, but NamedTemporaryFile
180 # will fail if the filename contains non ascii characters unless we
181 # use a unicode object
182 path_basename
= lambda f
: os
.path
.basename(fn
).decode(encoding
)
183 # the same for os.path.dirname
184 path_dirname
= lambda f
: os
.path
.dirname(fn
).decode(encoding
)
186 path_basename
= os
.path
.basename
187 path_dirname
= os
.path
.dirname
191 'prefix': path_basename(fn
) + '.',
192 'dir': path_dirname(fn
),
196 # In Python 2.x, json.dump expects a bytestream.
197 # In Python 3.x, it writes to a character stream
198 if sys
.version_info
< (3, 0):
206 tf
= tempfile
.NamedTemporaryFile(**compat_kwargs(args
))
211 if sys
.platform
== 'win32':
212 # Need to remove existing file on Windows, else os.rename raises
213 # WindowsError or FileExistsError.
218 os
.rename(tf
.name
, fn
)
227 if sys
.version_info
>= (2, 7):
228 def find_xpath_attr(node
, xpath
, key
, val
=None):
229 """ Find the xpath xpath[@key=val] """
230 assert re
.match(r
'^[a-zA-Z_-]+$', key
)
231 expr
= xpath
+ ('[@%s]' % key
if val
is None else "[@%s='%s']" % (key
, val
))
232 return node
.find(expr
)
234 def find_xpath_attr(node
, xpath
, key
, val
=None):
235 for f
in node
.findall(compat_xpath(xpath
)):
236 if key
not in f
.attrib
:
238 if val
is None or f
.attrib
.get(key
) == val
:
242 # On python2.6 the xml.etree.ElementTree.Element methods don't support
243 # the namespace parameter
246 def xpath_with_ns(path
, ns_map
):
247 components
= [c
.split(':') for c
in path
.split('/')]
251 replaced
.append(c
[0])
254 replaced
.append('{%s}%s' % (ns_map
[ns
], tag
))
255 return '/'.join(replaced
)
258 def xpath_element(node
, xpath
, name
=None, fatal
=False, default
=NO_DEFAULT
):
259 def _find_xpath(xpath
):
260 return node
.find(compat_xpath(xpath
))
262 if isinstance(xpath
, (str, compat_str
)):
263 n
= _find_xpath(xpath
)
271 if default
is not NO_DEFAULT
:
274 name
= xpath
if name
is None else name
275 raise ExtractorError('Could not find XML element %s' % name
)
281 def xpath_text(node
, xpath
, name
=None, fatal
=False, default
=NO_DEFAULT
):
282 n
= xpath_element(node
, xpath
, name
, fatal
=fatal
, default
=default
)
283 if n
is None or n
== default
:
286 if default
is not NO_DEFAULT
:
289 name
= xpath
if name
is None else name
290 raise ExtractorError('Could not find XML element\'s text %s' % name
)
296 def xpath_attr(node
, xpath
, key
, name
=None, fatal
=False, default
=NO_DEFAULT
):
297 n
= find_xpath_attr(node
, xpath
, key
)
299 if default
is not NO_DEFAULT
:
302 name
= '%s[@%s]' % (xpath
, key
) if name
is None else name
303 raise ExtractorError('Could not find XML attribute %s' % name
)
309 def get_element_by_id(id, html
):
310 """Return the content of the tag with the specified ID in the passed HTML document"""
311 return get_element_by_attribute('id', id, html
)
314 def get_element_by_class(class_name
, html
):
315 return get_element_by_attribute(
316 'class', r
'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
317 html, escape_value=False)
320 def get_element_by_attribute(attribute, value, html, escape_value=True):
321 """Return the content of the tag with the specified attribute in the passed HTML document"""
323 value = re.escape(value) if escape_value else value
325 m = re.search(r'''(?xs)
327 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^
']*'))*?
329 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^
"]*"|
='[^']*'))*?
333 ''' % (re.escape(attribute), value), html)
337 res = m.group('content
')
339 if res.startswith('"') or res.startswith("'"):
342 return unescapeHTML(res)
345 class HTMLAttributeParser(compat_HTMLParser):
346 """Trivial HTML parser to gather the attributes for a single element"""
349 compat_HTMLParser.__init__(self)
351 def handle_starttag(self, tag, attrs):
352 self.attrs = dict(attrs)
355 def extract_attributes(html_element):
356 """Given a string for an HTML element such as
358 a="foo" B="bar" c="&98;az" d=boz
359 empty= noval entity="&"
362 Decode and return a dictionary of attributes.
364 'a
': 'foo
', 'b
': 'bar
', c: 'baz
', d: 'boz
',
365 'empty
': '', 'noval
': None, 'entity
': '&',
366 'sq
': '"', 'dq': '\''
368 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
369 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
371 parser = HTMLAttributeParser()
372 parser.feed(html_element)
377 def clean_html(html):
378 """Clean an HTML snippet into a readable string"""
380 if html is None: # Convenience for sanitizing descriptions etc.
384 html = html.replace('\n', ' ')
385 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
386 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
388 html = re.sub('<.*?>', '', html)
389 # Replace html entities
390 html = unescapeHTML(html)
394 def sanitize_open(filename, open_mode):
395 """Try to open the given filename, and slightly tweak it if this fails.
397 Attempts to open the given filename. If this fails, it tries to change
398 the filename slightly, step by step, until it's either able to open it
399 or it fails and raises a final exception, like the standard open()
402 It returns the tuple (stream, definitive_file_name).
406 if sys.platform == 'win32':
408 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
409 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
410 stream = open(encodeFilename(filename), open_mode)
411 return (stream, filename)
412 except (IOError, OSError) as err:
413 if err.errno in (errno.EACCES,):
416 # In case of error, try to remove win32 forbidden chars
417 alt_filename = sanitize_path(filename)
418 if alt_filename == filename:
421 # An exception here should be caught in the caller
422 stream = open(encodeFilename(alt_filename), open_mode)
423 return (stream, alt_filename)
426 def timeconvert(timestr):
427 """Convert RFC 2822 defined time string into system timestamp"""
429 timetuple = email.utils.parsedate_tz(timestr)
430 if timetuple is not None:
431 timestamp = email.utils.mktime_tz(timetuple)
435 def sanitize_filename(s, restricted=False, is_id=False):
436 """Sanitizes a string so it could be used as part of a filename.
437 If restricted is set, use a stricter subset of allowed characters.
438 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
440 def replace_insane(char):
441 if restricted and char in ACCENT_CHARS:
442 return ACCENT_CHARS[char]
443 if char == '?' or ord(char) < 32 or ord(char) == 127:
446 return '' if restricted else '\''
448 return '_
-' if restricted else ' -'
449 elif char in '\\/|
*<>':
451 if restricted and (char in '!&\'()[]{}$
;`^
,#' or char.isspace()):
453 if restricted
and ord(char
) > 127:
458 s
= re
.sub(r
'[0-9]+(?::[0-9]+)+', lambda m
: m
.group(0).replace(':', '_'), s
)
459 result
= ''.join(map(replace_insane
, s
))
461 while '__' in result
:
462 result
= result
.replace('__', '_')
463 result
= result
.strip('_')
464 # Common case of "Foreign band name - English song title"
465 if restricted
and result
.startswith('-_'):
467 if result
.startswith('-'):
468 result
= '_' + result
[len('-'):]
469 result
= result
.lstrip('.')
475 def sanitize_path(s
):
476 """Sanitizes and normalizes path on Windows"""
477 if sys
.platform
!= 'win32':
479 drive_or_unc
, _
= os
.path
.splitdrive(s
)
480 if sys
.version_info
< (2, 7) and not drive_or_unc
:
481 drive_or_unc
, _
= os
.path
.splitunc(s
)
482 norm_path
= os
.path
.normpath(remove_start(s
, drive_or_unc
)).split(os
.path
.sep
)
486 path_part
if path_part
in ['.', '..'] else re
.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part
)
487 for path_part
in norm_path
]
489 sanitized_path
.insert(0, drive_or_unc
+ os
.path
.sep
)
490 return os
.path
.join(*sanitized_path
)
493 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
494 # unwanted failures due to missing protocol
495 def sanitize_url(url
):
496 return 'http:%s' % url
if url
.startswith('//') else url
499 def sanitized_Request(url
, *args
, **kwargs
):
500 return compat_urllib_request
.Request(sanitize_url(url
), *args
, **kwargs
)
503 def orderedSet(iterable
):
504 """ Remove all duplicates from the input iterable """
512 def _htmlentity_transform(entity_with_semicolon
):
513 """Transforms an HTML entity to a character."""
514 entity
= entity_with_semicolon
[:-1]
516 # Known non-numeric HTML entity
517 if entity
in compat_html_entities
.name2codepoint
:
518 return compat_chr(compat_html_entities
.name2codepoint
[entity
])
520 # TODO: HTML5 allows entities without a semicolon. For example,
521 # 'Éric' should be decoded as 'Éric'.
522 if entity_with_semicolon
in compat_html_entities_html5
:
523 return compat_html_entities_html5
[entity_with_semicolon
]
525 mobj
= re
.match(r
'#(x[0-9a-fA-F]+|[0-9]+)', entity
)
527 numstr
= mobj
.group(1)
528 if numstr
.startswith('x'):
530 numstr
= '0%s' % numstr
533 # See https://github.com/rg3/youtube-dl/issues/7518
535 return compat_chr(int(numstr
, base
))
539 # Unknown entity in name, return its literal representation
540 return '&%s;' % entity
546 assert type(s
) == compat_str
549 r
'&([^;]+;)', lambda m
: _htmlentity_transform(m
.group(1)), s
)
552 def get_subprocess_encoding():
553 if sys
.platform
== 'win32' and sys
.getwindowsversion()[0] >= 5:
554 # For subprocess calls, encode with locale encoding
555 # Refer to http://stackoverflow.com/a/9951851/35070
556 encoding
= preferredencoding()
558 encoding
= sys
.getfilesystemencoding()
564 def encodeFilename(s
, for_subprocess
=False):
566 @param s The name of the file
569 assert type(s
) == compat_str
571 # Python 3 has a Unicode API
572 if sys
.version_info
>= (3, 0):
575 # Pass '' directly to use Unicode APIs on Windows 2000 and up
576 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
577 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
578 if not for_subprocess
and sys
.platform
== 'win32' and sys
.getwindowsversion()[0] >= 5:
581 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
582 if sys
.platform
.startswith('java'):
585 return s
.encode(get_subprocess_encoding(), 'ignore')
588 def decodeFilename(b
, for_subprocess
=False):
590 if sys
.version_info
>= (3, 0):
593 if not isinstance(b
, bytes):
596 return b
.decode(get_subprocess_encoding(), 'ignore')
599 def encodeArgument(s
):
600 if not isinstance(s
, compat_str
):
601 # Legacy code that uses byte strings
602 # Uncomment the following line after fixing all post processors
603 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
604 s
= s
.decode('ascii')
605 return encodeFilename(s
, True)
608 def decodeArgument(b
):
609 return decodeFilename(b
, True)
612 def decodeOption(optval
):
615 if isinstance(optval
, bytes):
616 optval
= optval
.decode(preferredencoding())
618 assert isinstance(optval
, compat_str
)
622 def formatSeconds(secs
):
624 return '%d:%02d:%02d' % (secs
// 3600, (secs
% 3600) // 60, secs
% 60)
626 return '%d:%02d' % (secs
// 60, secs
% 60)
631 def make_HTTPS_handler(params
, **kwargs
):
632 opts_no_check_certificate
= params
.get('nocheckcertificate', False)
633 if hasattr(ssl
, 'create_default_context'): # Python >= 3.4 or 2.7.9
634 context
= ssl
.create_default_context(ssl
.Purpose
.SERVER_AUTH
)
635 if opts_no_check_certificate
:
636 context
.check_hostname
= False
637 context
.verify_mode
= ssl
.CERT_NONE
639 return YoutubeDLHTTPSHandler(params
, context
=context
, **kwargs
)
642 # (create_default_context present but HTTPSHandler has no context=)
645 if sys
.version_info
< (3, 2):
646 return YoutubeDLHTTPSHandler(params
, **kwargs
)
648 context
= ssl
.SSLContext(ssl
.PROTOCOL_TLSv1
)
649 context
.verify_mode
= (ssl
.CERT_NONE
650 if opts_no_check_certificate
651 else ssl
.CERT_REQUIRED
)
652 context
.set_default_verify_paths()
653 return YoutubeDLHTTPSHandler(params
, context
=context
, **kwargs
)
656 def bug_reports_message():
657 if ytdl_is_updateable():
658 update_cmd
= 'type youtube-dl -U to update'
660 update_cmd
= 'see https://yt-dl.org/update on how to update'
661 msg
= '; please report this issue on https://yt-dl.org/bug .'
662 msg
+= ' Make sure you are using the latest version; %s.' % update_cmd
663 msg
+= ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
667 class ExtractorError(Exception):
668 """Error during info extraction."""
670 def __init__(self
, msg
, tb
=None, expected
=False, cause
=None, video_id
=None):
671 """ tb, if given, is the original traceback (so that it can be printed out).
672 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
675 if sys
.exc_info()[0] in (compat_urllib_error
.URLError
, socket
.timeout
, UnavailableVideoError
):
677 if video_id
is not None:
678 msg
= video_id
+ ': ' + msg
680 msg
+= ' (caused by %r)' % cause
682 msg
+= bug_reports_message()
683 super(ExtractorError
, self
).__init
__(msg
)
686 self
.exc_info
= sys
.exc_info() # preserve original exception
688 self
.video_id
= video_id
690 def format_traceback(self
):
691 if self
.traceback
is None:
693 return ''.join(traceback
.format_tb(self
.traceback
))
696 class UnsupportedError(ExtractorError
):
697 def __init__(self
, url
):
698 super(UnsupportedError
, self
).__init
__(
699 'Unsupported URL: %s' % url
, expected
=True)
703 class RegexNotFoundError(ExtractorError
):
704 """Error when a regex didn't match"""
708 class DownloadError(Exception):
709 """Download Error exception.
711 This exception may be thrown by FileDownloader objects if they are not
712 configured to continue on errors. They will contain the appropriate
716 def __init__(self
, msg
, exc_info
=None):
717 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
718 super(DownloadError
, self
).__init
__(msg
)
719 self
.exc_info
= exc_info
722 class SameFileError(Exception):
723 """Same File exception.
725 This exception will be thrown by FileDownloader objects if they detect
726 multiple files would have to be downloaded to the same file on disk.
731 class PostProcessingError(Exception):
732 """Post Processing exception.
734 This exception may be raised by PostProcessor's .run() method to
735 indicate an error in the postprocessing task.
738 def __init__(self
, msg
):
742 class MaxDownloadsReached(Exception):
743 """ --max-downloads limit has been reached. """
747 class UnavailableVideoError(Exception):
748 """Unavailable Format exception.
750 This exception will be thrown when a video is requested
751 in a format that is not available for that video.
756 class ContentTooShortError(Exception):
757 """Content Too Short exception.
759 This exception may be raised by FileDownloader objects when a file they
760 download is too small for what the server announced first, indicating
761 the connection was probably interrupted.
764 def __init__(self
, downloaded
, expected
):
766 self
.downloaded
= downloaded
767 self
.expected
= expected
770 def _create_http_connection(ydl_handler
, http_class
, is_https
, *args
, **kwargs
):
771 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
772 # expected HTTP responses to meet HTTP/1.0 or later (see also
773 # https://github.com/rg3/youtube-dl/issues/6727)
774 if sys
.version_info
< (3, 0):
775 kwargs
[b
'strict'] = True
776 hc
= http_class(*args
, **kwargs
)
777 source_address
= ydl_handler
._params
.get('source_address')
778 if source_address
is not None:
779 sa
= (source_address
, 0)
780 if hasattr(hc
, 'source_address'): # Python 2.7+
781 hc
.source_address
= sa
783 def _hc_connect(self
, *args
, **kwargs
):
784 sock
= compat_socket_create_connection(
785 (self
.host
, self
.port
), self
.timeout
, sa
)
787 self
.sock
= ssl
.wrap_socket(
788 sock
, self
.key_file
, self
.cert_file
,
789 ssl_version
=ssl
.PROTOCOL_TLSv1
)
792 hc
.connect
= functools
.partial(_hc_connect
, hc
)
797 def handle_youtubedl_headers(headers
):
798 filtered_headers
= headers
800 if 'Youtubedl-no-compression' in filtered_headers
:
801 filtered_headers
= dict((k
, v
) for k
, v
in filtered_headers
.items() if k
.lower() != 'accept-encoding')
802 del filtered_headers
['Youtubedl-no-compression']
804 return filtered_headers
807 class YoutubeDLHandler(compat_urllib_request
.HTTPHandler
):
808 """Handler for HTTP requests and responses.
810 This class, when installed with an OpenerDirector, automatically adds
811 the standard headers to every HTTP request and handles gzipped and
812 deflated responses from web servers. If compression is to be avoided in
813 a particular request, the original request in the program code only has
814 to include the HTTP header "Youtubedl-no-compression", which will be
815 removed before making the real request.
817 Part of this code was copied from:
819 http://techknack.net/python-urllib2-handlers/
821 Andrew Rowls, the author of that code, agreed to release it to the
825 def __init__(self
, params
, *args
, **kwargs
):
826 compat_urllib_request
.HTTPHandler
.__init
__(self
, *args
, **kwargs
)
827 self
._params
= params
829 def http_open(self
, req
):
830 conn_class
= compat_http_client
.HTTPConnection
832 socks_proxy
= req
.headers
.get('Ytdl-socks-proxy')
834 conn_class
= make_socks_conn_class(conn_class
, socks_proxy
)
835 del req
.headers
['Ytdl-socks-proxy']
837 return self
.do_open(functools
.partial(
838 _create_http_connection
, self
, conn_class
, False),
844 return zlib
.decompress(data
, -zlib
.MAX_WBITS
)
846 return zlib
.decompress(data
)
849 def addinfourl_wrapper(stream
, headers
, url
, code
):
850 if hasattr(compat_urllib_request
.addinfourl
, 'getcode'):
851 return compat_urllib_request
.addinfourl(stream
, headers
, url
, code
)
852 ret
= compat_urllib_request
.addinfourl(stream
, headers
, url
)
856 def http_request(self
, req
):
857 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
858 # always respected by websites, some tend to give out URLs with non percent-encoded
859 # non-ASCII characters (see telemb.py, ard.py [#3412])
860 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
861 # To work around aforementioned issue we will replace request's original URL with
862 # percent-encoded one
863 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
864 # the code of this workaround has been moved here from YoutubeDL.urlopen()
865 url
= req
.get_full_url()
866 url_escaped
= escape_url(url
)
868 # Substitute URL if any change after escaping
869 if url
!= url_escaped
:
870 req
= update_Request(req
, url
=url_escaped
)
872 for h
, v
in std_headers
.items():
873 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
874 # The dict keys are capitalized because of this bug by urllib
875 if h
.capitalize() not in req
.headers
:
878 req
.headers
= handle_youtubedl_headers(req
.headers
)
880 if sys
.version_info
< (2, 7) and '#' in req
.get_full_url():
881 # Python 2.6 is brain-dead when it comes to fragments
882 req
._Request
__original
= req
._Request
__original
.partition('#')[0]
883 req
._Request
__r
_type
= req
._Request
__r
_type
.partition('#')[0]
887 def http_response(self
, req
, resp
):
890 if resp
.headers
.get('Content-encoding', '') == 'gzip':
891 content
= resp
.read()
892 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
), mode
='rb')
894 uncompressed
= io
.BytesIO(gz
.read())
895 except IOError as original_ioerror
:
896 # There may be junk add the end of the file
897 # See http://stackoverflow.com/q/4928560/35070 for details
898 for i
in range(1, 1024):
900 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
[:-i
]), mode
='rb')
901 uncompressed
= io
.BytesIO(gz
.read())
906 raise original_ioerror
907 resp
= self
.addinfourl_wrapper(uncompressed
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
908 resp
.msg
= old_resp
.msg
909 del resp
.headers
['Content-encoding']
911 if resp
.headers
.get('Content-encoding', '') == 'deflate':
912 gz
= io
.BytesIO(self
.deflate(resp
.read()))
913 resp
= self
.addinfourl_wrapper(gz
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
914 resp
.msg
= old_resp
.msg
915 del resp
.headers
['Content-encoding']
916 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
917 # https://github.com/rg3/youtube-dl/issues/6457).
918 if 300 <= resp
.code
< 400:
919 location
= resp
.headers
.get('Location')
921 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
922 if sys
.version_info
>= (3, 0):
923 location
= location
.encode('iso-8859-1').decode('utf-8')
925 location
= location
.decode('utf-8')
926 location_escaped
= escape_url(location
)
927 if location
!= location_escaped
:
928 del resp
.headers
['Location']
929 if sys
.version_info
< (3, 0):
930 location_escaped
= location_escaped
.encode('utf-8')
931 resp
.headers
['Location'] = location_escaped
934 https_request
= http_request
935 https_response
= http_response
938 def make_socks_conn_class(base_class
, socks_proxy
):
939 assert issubclass(base_class
, (
940 compat_http_client
.HTTPConnection
, compat_http_client
.HTTPSConnection
))
942 url_components
= compat_urlparse
.urlparse(socks_proxy
)
943 if url_components
.scheme
.lower() == 'socks5':
944 socks_type
= ProxyType
.SOCKS5
945 elif url_components
.scheme
.lower() in ('socks', 'socks4'):
946 socks_type
= ProxyType
.SOCKS4
947 elif url_components
.scheme
.lower() == 'socks4a':
948 socks_type
= ProxyType
.SOCKS4A
950 def unquote_if_non_empty(s
):
953 return compat_urllib_parse_unquote_plus(s
)
957 url_components
.hostname
, url_components
.port
or 1080,
959 unquote_if_non_empty(url_components
.username
),
960 unquote_if_non_empty(url_components
.password
),
963 class SocksConnection(base_class
):
965 self
.sock
= sockssocket()
966 self
.sock
.setproxy(*proxy_args
)
967 if type(self
.timeout
) in (int, float):
968 self
.sock
.settimeout(self
.timeout
)
969 self
.sock
.connect((self
.host
, self
.port
))
971 if isinstance(self
, compat_http_client
.HTTPSConnection
):
972 if hasattr(self
, '_context'): # Python > 2.6
973 self
.sock
= self
._context
.wrap_socket(
974 self
.sock
, server_hostname
=self
.host
)
976 self
.sock
= ssl
.wrap_socket(self
.sock
)
978 return SocksConnection
981 class YoutubeDLHTTPSHandler(compat_urllib_request
.HTTPSHandler
):
982 def __init__(self
, params
, https_conn_class
=None, *args
, **kwargs
):
983 compat_urllib_request
.HTTPSHandler
.__init
__(self
, *args
, **kwargs
)
984 self
._https
_conn
_class
= https_conn_class
or compat_http_client
.HTTPSConnection
985 self
._params
= params
987 def https_open(self
, req
):
989 conn_class
= self
._https
_conn
_class
991 if hasattr(self
, '_context'): # python > 2.6
992 kwargs
['context'] = self
._context
993 if hasattr(self
, '_check_hostname'): # python 3.x
994 kwargs
['check_hostname'] = self
._check
_hostname
996 socks_proxy
= req
.headers
.get('Ytdl-socks-proxy')
998 conn_class
= make_socks_conn_class(conn_class
, socks_proxy
)
999 del req
.headers
['Ytdl-socks-proxy']
1001 return self
.do_open(functools
.partial(
1002 _create_http_connection
, self
, conn_class
, True),
1006 class YoutubeDLCookieProcessor(compat_urllib_request
.HTTPCookieProcessor
):
1007 def __init__(self
, cookiejar
=None):
1008 compat_urllib_request
.HTTPCookieProcessor
.__init
__(self
, cookiejar
)
1010 def http_response(self
, request
, response
):
1011 # Python 2 will choke on next HTTP request in row if there are non-ASCII
1012 # characters in Set-Cookie HTTP header of last response (see
1013 # https://github.com/rg3/youtube-dl/issues/6769).
1014 # In order to at least prevent crashing we will percent encode Set-Cookie
1015 # header before HTTPCookieProcessor starts processing it.
1016 # if sys.version_info < (3, 0) and response.headers:
1017 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
1018 # set_cookie = response.headers.get(set_cookie_header)
1020 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
1021 # if set_cookie != set_cookie_escaped:
1022 # del response.headers[set_cookie_header]
1023 # response.headers[set_cookie_header] = set_cookie_escaped
1024 return compat_urllib_request
.HTTPCookieProcessor
.http_response(self
, request
, response
)
1026 https_request
= compat_urllib_request
.HTTPCookieProcessor
.http_request
1027 https_response
= http_response
1030 def extract_timezone(date_str
):
1032 r
'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
1035 timezone
= datetime
.timedelta()
1037 date_str
= date_str
[:-len(m
.group('tz'))]
1038 if not m
.group('sign'):
1039 timezone
= datetime
.timedelta()
1041 sign
= 1 if m
.group('sign') == '+' else -1
1042 timezone
= datetime
.timedelta(
1043 hours
=sign
* int(m
.group('hours')),
1044 minutes
=sign
* int(m
.group('minutes')))
1045 return timezone
, date_str
1048 def parse_iso8601(date_str
, delimiter
='T', timezone
=None):
1049 """ Return a UNIX timestamp from the given date """
1051 if date_str
is None:
1054 date_str
= re
.sub(r
'\.[0-9]+', '', date_str
)
1056 if timezone
is None:
1057 timezone
, date_str
= extract_timezone(date_str
)
1060 date_format
= '%Y-%m-%d{0}%H:%M:%S'.format(delimiter
)
1061 dt
= datetime
.datetime
.strptime(date_str
, date_format
) - timezone
1062 return calendar
.timegm(dt
.timetuple())
1067 def date_formats(day_first
=True):
1068 return DATE_FORMATS_DAY_FIRST
if day_first
else DATE_FORMATS_MONTH_FIRST
1071 def unified_strdate(date_str
, day_first
=True):
1072 """Return a string with the date in the format YYYYMMDD"""
1074 if date_str
is None:
1078 date_str
= date_str
.replace(',', ' ')
1079 # Remove AM/PM + timezone
1080 date_str
= re
.sub(r
'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str
)
1081 _
, date_str
= extract_timezone(date_str
)
1083 for expression
in date_formats(day_first
):
1085 upload_date
= datetime
.datetime
.strptime(date_str
, expression
).strftime('%Y%m%d')
1088 if upload_date
is None:
1089 timetuple
= email
.utils
.parsedate_tz(date_str
)
1092 upload_date
= datetime
.datetime(*timetuple
[:6]).strftime('%Y%m%d')
1095 if upload_date
is not None:
1096 return compat_str(upload_date
)
1099 def unified_timestamp(date_str
, day_first
=True):
1100 if date_str
is None:
1103 date_str
= date_str
.replace(',', ' ')
1105 pm_delta
= 12 if re
.search(r
'(?i)PM', date_str
) else 0
1106 timezone
, date_str
= extract_timezone(date_str
)
1108 # Remove AM/PM + timezone
1109 date_str
= re
.sub(r
'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str
)
1111 for expression
in date_formats(day_first
):
1113 dt
= datetime
.datetime
.strptime(date_str
, expression
) - timezone
+ datetime
.timedelta(hours
=pm_delta
)
1114 return calendar
.timegm(dt
.timetuple())
1117 timetuple
= email
.utils
.parsedate_tz(date_str
)
1119 return calendar
.timegm(timetuple
) + pm_delta
* 3600
1122 def determine_ext(url
, default_ext
='unknown_video'):
1125 guess
= url
.partition('?')[0].rpartition('.')[2]
1126 if re
.match(r
'^[A-Za-z0-9]+$', guess
):
1128 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
1129 elif guess
.rstrip('/') in KNOWN_EXTENSIONS
:
1130 return guess
.rstrip('/')
1135 def subtitles_filename(filename
, sub_lang
, sub_format
):
1136 return filename
.rsplit('.', 1)[0] + '.' + sub_lang
+ '.' + sub_format
1139 def date_from_str(date_str
):
1141 Return a datetime object from a string in the format YYYYMMDD or
1142 (now|today)[+-][0-9](day|week|month|year)(s)?"""
1143 today
= datetime
.date
.today()
1144 if date_str
in ('now', 'today'):
1146 if date_str
== 'yesterday':
1147 return today
- datetime
.timedelta(days
=1)
1148 match
= re
.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str
)
1149 if match
is not None:
1150 sign
= match
.group('sign')
1151 time
= int(match
.group('time'))
1154 unit
= match
.group('unit')
1155 # A bad approximation?
1159 elif unit
== 'year':
1163 delta
= datetime
.timedelta(**{unit: time}
)
1164 return today
+ delta
1165 return datetime
.datetime
.strptime(date_str
, '%Y%m%d').date()
1168 def hyphenate_date(date_str
):
1170 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1171 match
= re
.match(r
'^(\d\d\d\d)(\d\d)(\d\d)$', date_str
)
1172 if match
is not None:
1173 return '-'.join(match
.groups())
1178 class DateRange(object):
1179 """Represents a time interval between two dates"""
1181 def __init__(self
, start
=None, end
=None):
1182 """start and end must be strings in the format accepted by date"""
1183 if start
is not None:
1184 self
.start
= date_from_str(start
)
1186 self
.start
= datetime
.datetime
.min.date()
1188 self
.end
= date_from_str(end
)
1190 self
.end
= datetime
.datetime
.max.date()
1191 if self
.start
> self
.end
:
1192 raise ValueError('Date range: "%s" , the start date must be before the end date' % self
)
1196 """Returns a range that only contains the given day"""
1197 return cls(day
, day
)
1199 def __contains__(self
, date
):
1200 """Check if the date is in the range"""
1201 if not isinstance(date
, datetime
.date
):
1202 date
= date_from_str(date
)
1203 return self
.start
<= date
<= self
.end
1206 return '%s - %s' % (self
.start
.isoformat(), self
.end
.isoformat())
1209 def platform_name():
1210 """ Returns the platform name as a compat_str """
1211 res
= platform
.platform()
1212 if isinstance(res
, bytes):
1213 res
= res
.decode(preferredencoding())
1215 assert isinstance(res
, compat_str
)
1219 def _windows_write_string(s
, out
):
1220 """ Returns True if the string was written using special methods,
1221 False if it has yet to be written out."""
1222 # Adapted from http://stackoverflow.com/a/3259271/35070
1225 import ctypes
.wintypes
1233 fileno
= out
.fileno()
1234 except AttributeError:
1235 # If the output stream doesn't have a fileno, it's virtual
1237 except io
.UnsupportedOperation
:
1238 # Some strange Windows pseudo files?
1240 if fileno
not in WIN_OUTPUT_IDS
:
1243 GetStdHandle
= ctypes
.WINFUNCTYPE(
1244 ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.DWORD
)(
1245 (b
'GetStdHandle', ctypes
.windll
.kernel32
))
1246 h
= GetStdHandle(WIN_OUTPUT_IDS
[fileno
])
1248 WriteConsoleW
= ctypes
.WINFUNCTYPE(
1249 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.LPWSTR
,
1250 ctypes
.wintypes
.DWORD
, ctypes
.POINTER(ctypes
.wintypes
.DWORD
),
1251 ctypes
.wintypes
.LPVOID
)((b
'WriteConsoleW', ctypes
.windll
.kernel32
))
1252 written
= ctypes
.wintypes
.DWORD(0)
1254 GetFileType
= ctypes
.WINFUNCTYPE(ctypes
.wintypes
.DWORD
, ctypes
.wintypes
.DWORD
)((b
'GetFileType', ctypes
.windll
.kernel32
))
1255 FILE_TYPE_CHAR
= 0x0002
1256 FILE_TYPE_REMOTE
= 0x8000
1257 GetConsoleMode
= ctypes
.WINFUNCTYPE(
1258 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
,
1259 ctypes
.POINTER(ctypes
.wintypes
.DWORD
))(
1260 (b
'GetConsoleMode', ctypes
.windll
.kernel32
))
1261 INVALID_HANDLE_VALUE
= ctypes
.wintypes
.DWORD(-1).value
1263 def not_a_console(handle
):
1264 if handle
== INVALID_HANDLE_VALUE
or handle
is None:
1266 return ((GetFileType(handle
) & ~FILE_TYPE_REMOTE
) != FILE_TYPE_CHAR
or
1267 GetConsoleMode(handle
, ctypes
.byref(ctypes
.wintypes
.DWORD())) == 0)
1269 if not_a_console(h
):
1272 def next_nonbmp_pos(s
):
1274 return next(i
for i
, c
in enumerate(s
) if ord(c
) > 0xffff)
1275 except StopIteration:
1279 count
= min(next_nonbmp_pos(s
), 1024)
1281 ret
= WriteConsoleW(
1282 h
, s
, count
if count
else 2, ctypes
.byref(written
), None)
1284 raise OSError('Failed to write string')
1285 if not count
: # We just wrote a non-BMP character
1286 assert written
.value
== 2
1289 assert written
.value
> 0
1290 s
= s
[written
.value
:]
1294 def write_string(s
, out
=None, encoding
=None):
1297 assert type(s
) == compat_str
1299 if sys
.platform
== 'win32' and encoding
is None and hasattr(out
, 'fileno'):
1300 if _windows_write_string(s
, out
):
1303 if ('b' in getattr(out
, 'mode', '') or
1304 sys
.version_info
[0] < 3): # Python 2 lies about mode of sys.stderr
1305 byt
= s
.encode(encoding
or preferredencoding(), 'ignore')
1307 elif hasattr(out
, 'buffer'):
1308 enc
= encoding
or getattr(out
, 'encoding', None) or preferredencoding()
1309 byt
= s
.encode(enc
, 'ignore')
1310 out
.buffer.write(byt
)
1316 def bytes_to_intlist(bs
):
1319 if isinstance(bs
[0], int): # Python 3
1322 return [ord(c
) for c
in bs
]
1325 def intlist_to_bytes(xs
):
1328 return compat_struct_pack('%dB' % len(xs
), *xs
)
1331 # Cross-platform file locking
1332 if sys
.platform
== 'win32':
1333 import ctypes
.wintypes
1336 class OVERLAPPED(ctypes
.Structure
):
1338 ('Internal', ctypes
.wintypes
.LPVOID
),
1339 ('InternalHigh', ctypes
.wintypes
.LPVOID
),
1340 ('Offset', ctypes
.wintypes
.DWORD
),
1341 ('OffsetHigh', ctypes
.wintypes
.DWORD
),
1342 ('hEvent', ctypes
.wintypes
.HANDLE
),
1345 kernel32
= ctypes
.windll
.kernel32
1346 LockFileEx
= kernel32
.LockFileEx
1347 LockFileEx
.argtypes
= [
1348 ctypes
.wintypes
.HANDLE
, # hFile
1349 ctypes
.wintypes
.DWORD
, # dwFlags
1350 ctypes
.wintypes
.DWORD
, # dwReserved
1351 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
1352 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
1353 ctypes
.POINTER(OVERLAPPED
) # Overlapped
1355 LockFileEx
.restype
= ctypes
.wintypes
.BOOL
1356 UnlockFileEx
= kernel32
.UnlockFileEx
1357 UnlockFileEx
.argtypes
= [
1358 ctypes
.wintypes
.HANDLE
, # hFile
1359 ctypes
.wintypes
.DWORD
, # dwReserved
1360 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
1361 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
1362 ctypes
.POINTER(OVERLAPPED
) # Overlapped
1364 UnlockFileEx
.restype
= ctypes
.wintypes
.BOOL
1365 whole_low
= 0xffffffff
1366 whole_high
= 0x7fffffff
1368 def _lock_file(f
, exclusive
):
1369 overlapped
= OVERLAPPED()
1370 overlapped
.Offset
= 0
1371 overlapped
.OffsetHigh
= 0
1372 overlapped
.hEvent
= 0
1373 f
._lock
_file
_overlapped
_p
= ctypes
.pointer(overlapped
)
1374 handle
= msvcrt
.get_osfhandle(f
.fileno())
1375 if not LockFileEx(handle
, 0x2 if exclusive
else 0x0, 0,
1376 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
1377 raise OSError('Locking file failed: %r' % ctypes
.FormatError())
1379 def _unlock_file(f
):
1380 assert f
._lock
_file
_overlapped
_p
1381 handle
= msvcrt
.get_osfhandle(f
.fileno())
1382 if not UnlockFileEx(handle
, 0,
1383 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
1384 raise OSError('Unlocking file failed: %r' % ctypes
.FormatError())
1387 # Some platforms, such as Jython, is missing fcntl
1391 def _lock_file(f
, exclusive
):
1392 fcntl
.flock(f
, fcntl
.LOCK_EX
if exclusive
else fcntl
.LOCK_SH
)
1394 def _unlock_file(f
):
1395 fcntl
.flock(f
, fcntl
.LOCK_UN
)
1397 UNSUPPORTED_MSG
= 'file locking is not supported on this platform'
1399 def _lock_file(f
, exclusive
):
1400 raise IOError(UNSUPPORTED_MSG
)
1402 def _unlock_file(f
):
1403 raise IOError(UNSUPPORTED_MSG
)
1406 class locked_file(object):
1407 def __init__(self
, filename
, mode
, encoding
=None):
1408 assert mode
in ['r', 'a', 'w']
1409 self
.f
= io
.open(filename
, mode
, encoding
=encoding
)
1412 def __enter__(self
):
1413 exclusive
= self
.mode
!= 'r'
1415 _lock_file(self
.f
, exclusive
)
1421 def __exit__(self
, etype
, value
, traceback
):
1423 _unlock_file(self
.f
)
1430 def write(self
, *args
):
1431 return self
.f
.write(*args
)
1433 def read(self
, *args
):
1434 return self
.f
.read(*args
)
1437 def get_filesystem_encoding():
1438 encoding
= sys
.getfilesystemencoding()
1439 return encoding
if encoding
is not None else 'utf-8'
1442 def shell_quote(args
):
1444 encoding
= get_filesystem_encoding()
1446 if isinstance(a
, bytes):
1447 # We may get a filename encoded with 'encodeFilename'
1448 a
= a
.decode(encoding
)
1449 quoted_args
.append(pipes
.quote(a
))
1450 return ' '.join(quoted_args
)
1453 def smuggle_url(url
, data
):
1454 """ Pass additional data in a URL for internal use. """
1456 url
, idata
= unsmuggle_url(url
, {})
1458 sdata
= compat_urllib_parse_urlencode(
1459 {'__youtubedl_smuggle': json.dumps(data)}
)
1460 return url
+ '#' + sdata
1463 def unsmuggle_url(smug_url
, default
=None):
1464 if '#__youtubedl_smuggle' not in smug_url
:
1465 return smug_url
, default
1466 url
, _
, sdata
= smug_url
.rpartition('#')
1467 jsond
= compat_parse_qs(sdata
)['__youtubedl_smuggle'][0]
1468 data
= json
.loads(jsond
)
1472 def format_bytes(bytes):
1475 if type(bytes) is str:
1476 bytes = float(bytes)
1480 exponent
= int(math
.log(bytes, 1024.0))
1481 suffix
= ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent
]
1482 converted
= float(bytes) / float(1024 ** exponent
)
1483 return '%.2f%s' % (converted
, suffix
)
1486 def lookup_unit_table(unit_table
, s
):
1487 units_re
= '|'.join(re
.escape(u
) for u
in unit_table
)
1489 r
'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re
, s
)
1492 num_str
= m
.group('num').replace(',', '.')
1493 mult
= unit_table
[m
.group('unit')]
1494 return int(float(num_str
) * mult
)
1497 def parse_filesize(s
):
1501 # The lower-case forms are of course incorrect and unofficial,
1502 # but we support those too
1540 return lookup_unit_table(_UNIT_TABLE
, s
)
1549 if re
.match(r
'^[\d,.]+$', s
):
1550 return str_to_int(s
)
1561 return lookup_unit_table(_UNIT_TABLE
, s
)
1564 def month_by_name(name
):
1565 """ Return the number of a month by (locale-independently) English name """
1568 return ENGLISH_MONTH_NAMES
.index(name
) + 1
1573 def month_by_abbreviation(abbrev
):
1574 """ Return the number of a month by (locale-independently) English
1578 return [s
[:3] for s
in ENGLISH_MONTH_NAMES
].index(abbrev
) + 1
1583 def fix_xml_ampersands(xml_str
):
1584 """Replace all the '&' by '&' in XML"""
1586 r
'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1591 def setproctitle(title
):
1592 assert isinstance(title
, compat_str
)
1594 # ctypes in Jython is not complete
1595 # http://bugs.jython.org/issue2148
1596 if sys
.platform
.startswith('java'):
1600 libc
= ctypes
.cdll
.LoadLibrary('libc.so.6')
1603 title_bytes
= title
.encode('utf-8')
1604 buf
= ctypes
.create_string_buffer(len(title_bytes
))
1605 buf
.value
= title_bytes
1607 libc
.prctl(15, buf
, 0, 0, 0)
1608 except AttributeError:
1609 return # Strange libc, just skip this
1612 def remove_start(s
, start
):
1613 return s
[len(start
):] if s
is not None and s
.startswith(start
) else s
1616 def remove_end(s
, end
):
1617 return s
[:-len(end
)] if s
is not None and s
.endswith(end
) else s
1620 def remove_quotes(s
):
1621 if s
is None or len(s
) < 2:
1623 for quote
in ('"', "'", ):
1624 if s
[0] == quote
and s
[-1] == quote
:
1629 def url_basename(url
):
1630 path
= compat_urlparse
.urlparse(url
).path
1631 return path
.strip('/').split('/')[-1]
1634 class HEADRequest(compat_urllib_request
.Request
):
1635 def get_method(self
):
1639 class PUTRequest(compat_urllib_request
.Request
):
1640 def get_method(self
):
1644 def int_or_none(v
, scale
=1, default
=None, get_attr
=None, invscale
=1):
1647 v
= getattr(v
, get_attr
, None)
1653 return int(v
) * invscale
// scale
1658 def str_or_none(v
, default
=None):
1659 return default
if v
is None else compat_str(v
)
1662 def str_to_int(int_str
):
1663 """ A more relaxed version of int_or_none """
1666 int_str
= re
.sub(r
'[,\.\+]', '', int_str
)
1670 def float_or_none(v
, scale
=1, invscale
=1, default
=None):
1674 return float(v
) * invscale
/ scale
1679 def strip_or_none(v
):
1680 return None if v
is None else v
.strip()
1683 def parse_duration(s
):
1684 if not isinstance(s
, compat_basestring
):
1689 days
, hours
, mins
, secs
, ms
= [None] * 5
1690 m
= re
.match(r
'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s
)
1692 days
, hours
, mins
, secs
, ms
= m
.groups()
1697 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
1700 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
1703 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
1706 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
1709 days
, hours
, mins
, secs
, ms
= m
.groups()
1711 m
= re
.match(r
'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s
)
1713 hours
, mins
= m
.groups()
1719 duration
+= float(secs
)
1721 duration
+= float(mins
) * 60
1723 duration
+= float(hours
) * 60 * 60
1725 duration
+= float(days
) * 24 * 60 * 60
1727 duration
+= float(ms
)
1731 def prepend_extension(filename
, ext
, expected_real_ext
=None):
1732 name
, real_ext
= os
.path
.splitext(filename
)
1734 '{0}.{1}{2}'.format(name
, ext
, real_ext
)
1735 if not expected_real_ext
or real_ext
[1:] == expected_real_ext
1736 else '{0}.{1}'.format(filename
, ext
))
1739 def replace_extension(filename
, ext
, expected_real_ext
=None):
1740 name
, real_ext
= os
.path
.splitext(filename
)
1741 return '{0}.{1}'.format(
1742 name
if not expected_real_ext
or real_ext
[1:] == expected_real_ext
else filename
,
1746 def check_executable(exe
, args
=[]):
1747 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1748 args can be a list of arguments for a short output (like -version) """
1750 subprocess
.Popen([exe
] + args
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
).communicate()
1756 def get_exe_version(exe
, args
=['--version'],
1757 version_re
=None, unrecognized
='present'):
1758 """ Returns the version of the specified executable,
1759 or False if the executable is not present """
1761 out
, _
= subprocess
.Popen(
1762 [encodeArgument(exe
)] + args
,
1763 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
).communicate()
1766 if isinstance(out
, bytes): # Python 2.x
1767 out
= out
.decode('ascii', 'ignore')
1768 return detect_exe_version(out
, version_re
, unrecognized
)
1771 def detect_exe_version(output
, version_re
=None, unrecognized
='present'):
1772 assert isinstance(output
, compat_str
)
1773 if version_re
is None:
1774 version_re
= r
'version\s+([-0-9._a-zA-Z]+)'
1775 m
= re
.search(version_re
, output
)
1782 class PagedList(object):
1784 # This is only useful for tests
1785 return len(self
.getslice())
1788 class OnDemandPagedList(PagedList
):
1789 def __init__(self
, pagefunc
, pagesize
, use_cache
=False):
1790 self
._pagefunc
= pagefunc
1791 self
._pagesize
= pagesize
1792 self
._use
_cache
= use_cache
1796 def getslice(self
, start
=0, end
=None):
1798 for pagenum
in itertools
.count(start
// self
._pagesize
):
1799 firstid
= pagenum
* self
._pagesize
1800 nextfirstid
= pagenum
* self
._pagesize
+ self
._pagesize
1801 if start
>= nextfirstid
:
1806 page_results
= self
._cache
.get(pagenum
)
1807 if page_results
is None:
1808 page_results
= list(self
._pagefunc
(pagenum
))
1810 self
._cache
[pagenum
] = page_results
1813 start
% self
._pagesize
1814 if firstid
<= start
< nextfirstid
1818 ((end
- 1) % self
._pagesize
) + 1
1819 if (end
is not None and firstid
<= end
<= nextfirstid
)
1822 if startv
!= 0 or endv
is not None:
1823 page_results
= page_results
[startv
:endv
]
1824 res
.extend(page_results
)
1826 # A little optimization - if current page is not "full", ie. does
1827 # not contain page_size videos then we can assume that this page
1828 # is the last one - there are no more ids on further pages -
1829 # i.e. no need to query again.
1830 if len(page_results
) + startv
< self
._pagesize
:
1833 # If we got the whole page, but the next page is not interesting,
1834 # break out early as well
1835 if end
== nextfirstid
:
1840 class InAdvancePagedList(PagedList
):
1841 def __init__(self
, pagefunc
, pagecount
, pagesize
):
1842 self
._pagefunc
= pagefunc
1843 self
._pagecount
= pagecount
1844 self
._pagesize
= pagesize
1846 def getslice(self
, start
=0, end
=None):
1848 start_page
= start
// self
._pagesize
1850 self
._pagecount
if end
is None else (end
// self
._pagesize
+ 1))
1851 skip_elems
= start
- start_page
* self
._pagesize
1852 only_more
= None if end
is None else end
- start
1853 for pagenum
in range(start_page
, end_page
):
1854 page
= list(self
._pagefunc
(pagenum
))
1856 page
= page
[skip_elems
:]
1858 if only_more
is not None:
1859 if len(page
) < only_more
:
1860 only_more
-= len(page
)
1862 page
= page
[:only_more
]
1869 def uppercase_escape(s
):
1870 unicode_escape
= codecs
.getdecoder('unicode_escape')
1872 r
'\\U[0-9a-fA-F]{8}',
1873 lambda m
: unicode_escape(m
.group(0))[0],
1877 def lowercase_escape(s
):
1878 unicode_escape
= codecs
.getdecoder('unicode_escape')
1880 r
'\\u[0-9a-fA-F]{4}',
1881 lambda m
: unicode_escape(m
.group(0))[0],
1885 def escape_rfc3986(s
):
1886 """Escape non-ASCII characters as suggested by RFC 3986"""
1887 if sys
.version_info
< (3, 0) and isinstance(s
, compat_str
):
1888 s
= s
.encode('utf-8')
1889 return compat_urllib_parse
.quote(s
, b
"%/;:@&=+$,!~*'()?#[]")
1892 def escape_url(url
):
1893 """Escape URL as suggested by RFC 3986"""
1894 url_parsed
= compat_urllib_parse_urlparse(url
)
1895 return url_parsed
._replace
(
1896 netloc
=url_parsed
.netloc
.encode('idna').decode('ascii'),
1897 path
=escape_rfc3986(url_parsed
.path
),
1898 params
=escape_rfc3986(url_parsed
.params
),
1899 query
=escape_rfc3986(url_parsed
.query
),
1900 fragment
=escape_rfc3986(url_parsed
.fragment
)
1904 def read_batch_urls(batch_fd
):
1906 if not isinstance(url
, compat_str
):
1907 url
= url
.decode('utf-8', 'replace')
1908 BOM_UTF8
= '\xef\xbb\xbf'
1909 if url
.startswith(BOM_UTF8
):
1910 url
= url
[len(BOM_UTF8
):]
1912 if url
.startswith(('#', ';', ']')):
1916 with contextlib
.closing(batch_fd
) as fd
:
1917 return [url
for url
in map(fixup
, fd
) if url
]
1920 def urlencode_postdata(*args
, **kargs
):
1921 return compat_urllib_parse_urlencode(*args
, **kargs
).encode('ascii')
1924 def update_url_query(url
, query
):
1927 parsed_url
= compat_urlparse
.urlparse(url
)
1928 qs
= compat_parse_qs(parsed_url
.query
)
1930 return compat_urlparse
.urlunparse(parsed_url
._replace
(
1931 query
=compat_urllib_parse_urlencode(qs
, True)))
1934 def update_Request(req
, url
=None, data
=None, headers
={}, query={}
):
1935 req_headers
= req
.headers
.copy()
1936 req_headers
.update(headers
)
1937 req_data
= data
or req
.data
1938 req_url
= update_url_query(url
or req
.get_full_url(), query
)
1939 req_get_method
= req
.get_method()
1940 if req_get_method
== 'HEAD':
1941 req_type
= HEADRequest
1942 elif req_get_method
== 'PUT':
1943 req_type
= PUTRequest
1945 req_type
= compat_urllib_request
.Request
1947 req_url
, data
=req_data
, headers
=req_headers
,
1948 origin_req_host
=req
.origin_req_host
, unverifiable
=req
.unverifiable
)
1949 if hasattr(req
, 'timeout'):
1950 new_req
.timeout
= req
.timeout
1954 def dict_get(d
, key_or_keys
, default
=None, skip_false_values
=True):
1955 if isinstance(key_or_keys
, (list, tuple)):
1956 for key
in key_or_keys
:
1957 if key
not in d
or d
[key
] is None or skip_false_values
and not d
[key
]:
1961 return d
.get(key_or_keys
, default
)
1964 def try_get(src
, getter
, expected_type
=None):
1967 except (AttributeError, KeyError, TypeError, IndexError):
1970 if expected_type
is None or isinstance(v
, expected_type
):
1974 def encode_compat_str(string
, encoding
=preferredencoding(), errors
='strict'):
1975 return string
if isinstance(string
, compat_str
) else compat_str(string
, encoding
, errors
)
1987 def parse_age_limit(s
):
1990 m
= re
.match(r
'^(?P<age>\d{1,2})\+?$', s
)
1991 return int(m
.group('age')) if m
else US_RATINGS
.get(s
)
1994 def strip_jsonp(code
):
1996 r
'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r
'\1', code
)
1999 def js_to_json(code
):
2002 if v
in ('true', 'false', 'null'):
2004 elif v
.startswith('/*') or v
== ',':
2007 if v
[0] in ("'", '"'):
2008 v
= re
.sub(r
'(?s)\\.|"', lambda m
: {
2013 }.get(m
.group(0), m
.group(0)), v
[1:-1])
2016 (r
'^0[xX][0-9a-fA-F]+', 16),
2020 for regex
, base
in INTEGER_TABLE
:
2021 im
= re
.match(regex
, v
)
2023 i
= int(im
.group(0), base
)
2024 return '"%d":' % i
if v
.endswith(':') else '%d' % i
2028 return re
.sub(r
'''(?sx)
2029 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
2030 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
2031 /\*.*?\*/|,(?=\s*[\]}])|
2032 [a-zA-Z_][.a-zA-Z_0-9]*|
2033 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
2038 def qualities(quality_ids
):
2039 """ Get a numeric quality value out of a list of possible values """
2042 return quality_ids
.index(qid
)
2048 DEFAULT_OUTTMPL
= '%(title)s-%(id)s.%(ext)s'
2051 def limit_length(s
, length
):
2052 """ Add ellipses to overly long strings """
2057 return s
[:length
- len(ELLIPSES
)] + ELLIPSES
2061 def version_tuple(v
):
2062 return tuple(int(e
) for e
in re
.split(r
'[-.]', v
))
2065 def is_outdated_version(version
, limit
, assume_new
=True):
2067 return not assume_new
2069 return version_tuple(version
) < version_tuple(limit
)
2071 return not assume_new
2074 def ytdl_is_updateable():
2075 """ Returns if youtube-dl can be updated with -U """
2076 from zipimport
import zipimporter
2078 return isinstance(globals().get('__loader__'), zipimporter
) or hasattr(sys
, 'frozen')
2081 def args_to_str(args
):
2082 # Get a short string representation for a subprocess command
2083 return ' '.join(compat_shlex_quote(a
) for a
in args
)
2086 def error_to_compat_str(err
):
2088 # On python 2 error byte string must be decoded with proper
2089 # encoding rather than ascii
2090 if sys
.version_info
[0] < 3:
2091 err_str
= err_str
.decode(preferredencoding())
2095 def mimetype2ext(mt
):
2101 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
2102 # it's the most popular one
2103 'audio/mpeg': 'mp3',
2108 _
, _
, res
= mt
.rpartition('/')
2113 'smptett+xml': 'tt',
2119 'x-mp4-fragmented': 'mp4',
2122 'x-mpegurl': 'm3u8',
2123 'vnd.apple.mpegurl': 'm3u8',
2128 'vnd.ms-sstr+xml': 'ism',
2132 def parse_codecs(codecs_str
):
2133 # http://tools.ietf.org/html/rfc6381
2136 splited_codecs
= list(filter(None, map(
2137 lambda str: str.strip(), codecs_str
.strip().strip(',').split(','))))
2138 vcodec
, acodec
= None, None
2139 for full_codec
in splited_codecs
:
2140 codec
= full_codec
.split('.')[0]
2141 if codec
in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
2144 elif codec
in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac'):
2148 write_string('WARNING: Unknown codec %s' % full_codec
, sys
.stderr
)
2149 if not vcodec
and not acodec
:
2150 if len(splited_codecs
) == 2:
2155 elif len(splited_codecs
) == 1:
2162 'vcodec': vcodec
or 'none',
2163 'acodec': acodec
or 'none',
2168 def urlhandle_detect_ext(url_handle
):
2169 getheader
= url_handle
.headers
.get
2171 cd
= getheader('Content-Disposition')
2173 m
= re
.match(r
'attachment;\s*filename="(?P<filename>[^"]+)"', cd
)
2175 e
= determine_ext(m
.group('filename'), default_ext
=None)
2179 return mimetype2ext(getheader('Content-Type'))
2182 def encode_data_uri(data
, mime_type
):
2183 return 'data:%s;base64,%s' % (mime_type
, base64
.b64encode(data
).decode('ascii'))
2186 def age_restricted(content_limit
, age_limit
):
2187 """ Returns True iff the content should be blocked """
2189 if age_limit
is None: # No limit set
2191 if content_limit
is None:
2192 return False # Content available for everyone
2193 return age_limit
< content_limit
2196 def is_html(first_bytes
):
2197 """ Detect whether a file contains HTML by examining its first bytes. """
2200 (b
'\xef\xbb\xbf', 'utf-8'),
2201 (b
'\x00\x00\xfe\xff', 'utf-32-be'),
2202 (b
'\xff\xfe\x00\x00', 'utf-32-le'),
2203 (b
'\xff\xfe', 'utf-16-le'),
2204 (b
'\xfe\xff', 'utf-16-be'),
2206 for bom
, enc
in BOMS
:
2207 if first_bytes
.startswith(bom
):
2208 s
= first_bytes
[len(bom
):].decode(enc
, 'replace')
2211 s
= first_bytes
.decode('utf-8', 'replace')
2213 return re
.match(r
'^\s*<', s
)
2216 def determine_protocol(info_dict
):
2217 protocol
= info_dict
.get('protocol')
2218 if protocol
is not None:
2221 url
= info_dict
['url']
2222 if url
.startswith('rtmp'):
2224 elif url
.startswith('mms'):
2226 elif url
.startswith('rtsp'):
2229 ext
= determine_ext(url
)
2235 return compat_urllib_parse_urlparse(url
).scheme
2238 def render_table(header_row
, data
):
2239 """ Render a list of rows, each as a list of values """
2240 table
= [header_row
] + data
2241 max_lens
= [max(len(compat_str(v
)) for v
in col
) for col
in zip(*table
)]
2242 format_str
= ' '.join('%-' + compat_str(ml
+ 1) + 's' for ml
in max_lens
[:-1]) + '%s'
2243 return '\n'.join(format_str
% tuple(row
) for row
in table
)
2246 def _match_one(filter_part
, dct
):
2247 COMPARISON_OPERATORS
= {
2255 operator_rex
= re
.compile(r
'''(?x)\s*
2257 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
2259 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
2260 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
2263 ''' % '|'.join(map(re
.escape
, COMPARISON_OPERATORS
.keys())))
2264 m
= operator_rex
.search(filter_part
)
2266 op
= COMPARISON_OPERATORS
[m
.group('op')]
2267 if m
.group('strval') is not None:
2268 if m
.group('op') not in ('=', '!='):
2270 'Operator %s does not support string values!' % m
.group('op'))
2271 comparison_value
= m
.group('strval')
2274 comparison_value
= int(m
.group('intval'))
2276 comparison_value
= parse_filesize(m
.group('intval'))
2277 if comparison_value
is None:
2278 comparison_value
= parse_filesize(m
.group('intval') + 'B')
2279 if comparison_value
is None:
2281 'Invalid integer value %r in filter part %r' % (
2282 m
.group('intval'), filter_part
))
2283 actual_value
= dct
.get(m
.group('key'))
2284 if actual_value
is None:
2285 return m
.group('none_inclusive')
2286 return op(actual_value
, comparison_value
)
2289 '': lambda v
: v
is not None,
2290 '!': lambda v
: v
is None,
2292 operator_rex
= re
.compile(r
'''(?x)\s*
2293 (?P<op>%s)\s*(?P<key>[a-z_]+)
2295 ''' % '|'.join(map(re
.escape
, UNARY_OPERATORS
.keys())))
2296 m
= operator_rex
.search(filter_part
)
2298 op
= UNARY_OPERATORS
[m
.group('op')]
2299 actual_value
= dct
.get(m
.group('key'))
2300 return op(actual_value
)
2302 raise ValueError('Invalid filter part %r' % filter_part
)
2305 def match_str(filter_str
, dct
):
2306 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2309 _match_one(filter_part
, dct
) for filter_part
in filter_str
.split('&'))
2312 def match_filter_func(filter_str
):
2313 def _match_func(info_dict
):
2314 if match_str(filter_str
, info_dict
):
2317 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
2318 return '%s does not pass filter %s, skipping ..' % (video_title
, filter_str
)
2322 def parse_dfxp_time_expr(time_expr
):
2326 mobj
= re
.match(r
'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr
)
2328 return float(mobj
.group('time_offset'))
2330 mobj
= re
.match(r
'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr
)
2332 return 3600 * int(mobj
.group(1)) + 60 * int(mobj
.group(2)) + float(mobj
.group(3).replace(':', '.'))
2335 def srt_subtitles_timecode(seconds
):
2336 return '%02d:%02d:%02d,%03d' % (seconds
/ 3600, (seconds
% 3600) / 60, seconds
% 60, (seconds
% 1) * 1000)
2339 def dfxp2srt(dfxp_data
):
2340 _x
= functools
.partial(xpath_with_ns
, ns_map
={
2341 'ttml': 'http://www.w3.org/ns/ttml',
2342 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2343 'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
2346 class TTMLPElementParser(object):
2349 def start(self
, tag
, attrib
):
2350 if tag
in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2356 def data(self
, data
):
2360 return self
.out
.strip()
2362 def parse_node(node
):
2363 target
= TTMLPElementParser()
2364 parser
= xml
.etree
.ElementTree
.XMLParser(target
=target
)
2365 parser
.feed(xml
.etree
.ElementTree
.tostring(node
))
2366 return parser
.close()
2368 dfxp
= compat_etree_fromstring(dfxp_data
.encode('utf-8'))
2370 paras
= dfxp
.findall(_x('.//ttml:p')) or dfxp
.findall(_x('.//ttaf1:p')) or dfxp
.findall(_x('.//ttaf1_0604:p')) or dfxp
.findall('.//p')
2373 raise ValueError('Invalid dfxp/TTML subtitle')
2375 for para
, index
in zip(paras
, itertools
.count(1)):
2376 begin_time
= parse_dfxp_time_expr(para
.attrib
.get('begin'))
2377 end_time
= parse_dfxp_time_expr(para
.attrib
.get('end'))
2378 dur
= parse_dfxp_time_expr(para
.attrib
.get('dur'))
2379 if begin_time
is None:
2384 end_time
= begin_time
+ dur
2385 out
.append('%d\n%s --> %s\n%s\n\n' % (
2387 srt_subtitles_timecode(begin_time
),
2388 srt_subtitles_timecode(end_time
),
2394 def cli_option(params
, command_option
, param
):
2395 param
= params
.get(param
)
2396 return [command_option
, param
] if param
is not None else []
2399 def cli_bool_option(params
, command_option
, param
, true_value
='true', false_value
='false', separator
=None):
2400 param
= params
.get(param
)
2401 assert isinstance(param
, bool)
2403 return [command_option
+ separator
+ (true_value
if param
else false_value
)]
2404 return [command_option
, true_value
if param
else false_value
]
2407 def cli_valueless_option(params
, command_option
, param
, expected_value
=True):
2408 param
= params
.get(param
)
2409 return [command_option
] if param
== expected_value
else []
2412 def cli_configuration_args(params
, param
, default
=[]):
2413 ex_args
= params
.get(param
)
2416 assert isinstance(ex_args
, list)
2420 class ISO639Utils(object):
2421 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2610 def short2long(cls
, code
):
2611 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2612 return cls
._lang
_map
.get(code
[:2])
2615 def long2short(cls
, code
):
2616 """Convert language code from ISO 639-2/T to ISO 639-1"""
2617 for short_name
, long_name
in cls
._lang
_map
.items():
2618 if long_name
== code
:
2622 class ISO3166Utils(object):
2623 # From http://data.okfn.org/data/core/country-list
2625 'AF': 'Afghanistan',
2626 'AX': 'Åland Islands',
2629 'AS': 'American Samoa',
2634 'AG': 'Antigua and Barbuda',
2651 'BO': 'Bolivia, Plurinational State of',
2652 'BQ': 'Bonaire, Sint Eustatius and Saba',
2653 'BA': 'Bosnia and Herzegovina',
2655 'BV': 'Bouvet Island',
2657 'IO': 'British Indian Ocean Territory',
2658 'BN': 'Brunei Darussalam',
2660 'BF': 'Burkina Faso',
2666 'KY': 'Cayman Islands',
2667 'CF': 'Central African Republic',
2671 'CX': 'Christmas Island',
2672 'CC': 'Cocos (Keeling) Islands',
2676 'CD': 'Congo, the Democratic Republic of the',
2677 'CK': 'Cook Islands',
2679 'CI': 'Côte d\'Ivoire',
2684 'CZ': 'Czech Republic',
2688 'DO': 'Dominican Republic',
2691 'SV': 'El Salvador',
2692 'GQ': 'Equatorial Guinea',
2696 'FK': 'Falkland Islands (Malvinas)',
2697 'FO': 'Faroe Islands',
2701 'GF': 'French Guiana',
2702 'PF': 'French Polynesia',
2703 'TF': 'French Southern Territories',
2718 'GW': 'Guinea-Bissau',
2721 'HM': 'Heard Island and McDonald Islands',
2722 'VA': 'Holy See (Vatican City State)',
2729 'IR': 'Iran, Islamic Republic of',
2732 'IM': 'Isle of Man',
2742 'KP': 'Korea, Democratic People\'s Republic of',
2743 'KR': 'Korea, Republic of',
2746 'LA': 'Lao People\'s Democratic Republic',
2752 'LI': 'Liechtenstein',
2756 'MK': 'Macedonia, the Former Yugoslav Republic of',
2763 'MH': 'Marshall Islands',
2769 'FM': 'Micronesia, Federated States of',
2770 'MD': 'Moldova, Republic of',
2781 'NL': 'Netherlands',
2782 'NC': 'New Caledonia',
2783 'NZ': 'New Zealand',
2788 'NF': 'Norfolk Island',
2789 'MP': 'Northern Mariana Islands',
2794 'PS': 'Palestine, State of',
2796 'PG': 'Papua New Guinea',
2799 'PH': 'Philippines',
2803 'PR': 'Puerto Rico',
2807 'RU': 'Russian Federation',
2809 'BL': 'Saint Barthélemy',
2810 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2811 'KN': 'Saint Kitts and Nevis',
2812 'LC': 'Saint Lucia',
2813 'MF': 'Saint Martin (French part)',
2814 'PM': 'Saint Pierre and Miquelon',
2815 'VC': 'Saint Vincent and the Grenadines',
2818 'ST': 'Sao Tome and Principe',
2819 'SA': 'Saudi Arabia',
2823 'SL': 'Sierra Leone',
2825 'SX': 'Sint Maarten (Dutch part)',
2828 'SB': 'Solomon Islands',
2830 'ZA': 'South Africa',
2831 'GS': 'South Georgia and the South Sandwich Islands',
2832 'SS': 'South Sudan',
2837 'SJ': 'Svalbard and Jan Mayen',
2840 'CH': 'Switzerland',
2841 'SY': 'Syrian Arab Republic',
2842 'TW': 'Taiwan, Province of China',
2844 'TZ': 'Tanzania, United Republic of',
2846 'TL': 'Timor-Leste',
2850 'TT': 'Trinidad and Tobago',
2853 'TM': 'Turkmenistan',
2854 'TC': 'Turks and Caicos Islands',
2858 'AE': 'United Arab Emirates',
2859 'GB': 'United Kingdom',
2860 'US': 'United States',
2861 'UM': 'United States Minor Outlying Islands',
2865 'VE': 'Venezuela, Bolivarian Republic of',
2867 'VG': 'Virgin Islands, British',
2868 'VI': 'Virgin Islands, U.S.',
2869 'WF': 'Wallis and Futuna',
2870 'EH': 'Western Sahara',
2877 def short2full(cls
, code
):
2878 """Convert an ISO 3166-2 country code to the corresponding full name"""
2879 return cls
._country
_map
.get(code
.upper())
2882 class PerRequestProxyHandler(compat_urllib_request
.ProxyHandler
):
2883 def __init__(self
, proxies
=None):
2884 # Set default handlers
2885 for type in ('http', 'https'):
2886 setattr(self
, '%s_open' % type,
2887 lambda r
, proxy
='__noproxy__', type=type, meth
=self
.proxy_open
:
2888 meth(r
, proxy
, type))
2889 return compat_urllib_request
.ProxyHandler
.__init
__(self
, proxies
)
2891 def proxy_open(self
, req
, proxy
, type):
2892 req_proxy
= req
.headers
.get('Ytdl-request-proxy')
2893 if req_proxy
is not None:
2895 del req
.headers
['Ytdl-request-proxy']
2897 if proxy
== '__noproxy__':
2898 return None # No Proxy
2899 if compat_urlparse
.urlparse(proxy
).scheme
.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
2900 req
.add_header('Ytdl-socks-proxy', proxy
)
2901 # youtube-dl's http/https handlers do wrapping the socket with socks
2903 return compat_urllib_request
.ProxyHandler
.proxy_open(
2904 self
, req
, proxy
, type)
2907 def ohdave_rsa_encrypt(data
, exponent
, modulus
):
2909 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
2912 data: data to encrypt, bytes-like object
2913 exponent, modulus: parameter e and N of RSA algorithm, both integer
2914 Output: hex string of encrypted data
2916 Limitation: supports one block encryption only
2919 payload
= int(binascii
.hexlify(data
[::-1]), 16)
2920 encrypted
= pow(payload
, exponent
, modulus
)
2921 return '%x' % encrypted
2924 def encode_base_n(num
, n
, table
=None):
2925 FULL_TABLE
= '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
2927 table
= FULL_TABLE
[:n
]
2930 raise ValueError('base %d exceeds table length %d' % (n
, len(table
)))
2937 ret
= table
[num
% n
] + ret
2942 def decode_packed_codes(code
):
2944 r
"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
2946 obfucasted_code
, base
, count
, symbols
= mobj
.groups()
2949 symbols
= symbols
.split('|')
2954 base_n_count
= encode_base_n(count
, base
)
2955 symbol_table
[base_n_count
] = symbols
[count
] or base_n_count
2958 r
'\b(\w+)\b', lambda mobj
: symbol_table
[mobj
.group(0)],
2962 def parse_m3u8_attributes(attrib
):
2964 for (key
, val
) in re
.findall(r
'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib
):
2965 if val
.startswith('"'):
2971 def urshift(val
, n
):
2972 return val
>> n
if val
>= 0 else (val
+ 0x100000000) >> n
2975 # Based on png2str() written by @gdkchan and improved by @yokrysty
2976 # Originally posted at https://github.com/rg3/youtube-dl/issues/9706
2977 def decode_png(png_data
):
2978 # Reference: https://www.w3.org/TR/PNG/
2979 header
= png_data
[8:]
2981 if png_data
[:8] != b
'\x89PNG\x0d\x0a\x1a\x0a' or header
[4:8] != b
'IHDR':
2982 raise IOError('Not a valid PNG file.')
2984 int_map
= {1: '>B', 2: '>H', 4: '>I'}
2985 unpack_integer
= lambda x
: compat_struct_unpack(int_map
[len(x
)], x
)[0]
2990 length
= unpack_integer(header
[:4])
2993 chunk_type
= header
[:4]
2996 chunk_data
= header
[:length
]
2997 header
= header
[length
:]
2999 header
= header
[4:] # Skip CRC
3007 ihdr
= chunks
[0]['data']
3009 width
= unpack_integer(ihdr
[:4])
3010 height
= unpack_integer(ihdr
[4:8])
3014 for chunk
in chunks
:
3015 if chunk
['type'] == b
'IDAT':
3016 idat
+= chunk
['data']
3019 raise IOError('Unable to read PNG data.')
3021 decompressed_data
= bytearray(zlib
.decompress(idat
))
3026 def _get_pixel(idx
):
3031 for y
in range(height
):
3032 basePos
= y
* (1 + stride
)
3033 filter_type
= decompressed_data
[basePos
]
3037 pixels
.append(current_row
)
3039 for x
in range(stride
):
3040 color
= decompressed_data
[1 + basePos
+ x
]
3041 basex
= y
* stride
+ x
3046 left
= _get_pixel(basex
- 3)
3048 up
= _get_pixel(basex
- stride
)
3050 if filter_type
== 1: # Sub
3051 color
= (color
+ left
) & 0xff
3052 elif filter_type
== 2: # Up
3053 color
= (color
+ up
) & 0xff
3054 elif filter_type
== 3: # Average
3055 color
= (color
+ ((left
+ up
) >> 1)) & 0xff
3056 elif filter_type
== 4: # Paeth
3062 c
= _get_pixel(basex
- stride
- 3)
3070 if pa
<= pb
and pa
<= pc
:
3071 color
= (color
+ a
) & 0xff
3073 color
= (color
+ b
) & 0xff
3075 color
= (color
+ c
) & 0xff
3077 current_row
.append(color
)
3079 return width
, height
, pixels