youtube_dl/utils.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import unicode_literals
   5
   6 import base64
   7 import calendar
   8 import codecs
   9 import contextlib
  10 import ctypes
  11 import datetime
  12 import email.utils
  13 import errno
  14 import functools
  15 import gzip
  16 import itertools
  17 import io
  18 import json
  19 import locale
  20 import math
  21 import operator
  22 import os
  23 import pipes
  24 import platform
  25 import re
  26 import ssl
  27 import socket
  28 import struct
  29 import subprocess
  30 import sys
  31 import tempfile
  32 import traceback
  33 import xml.etree.ElementTree
  34 import zlib
  35
  36 from .compat import (
  37     compat_basestring,
  38     compat_chr,
  39     compat_etree_fromstring,
  40     compat_html_entities,
  41     compat_http_client,
  42     compat_kwargs,
  43     compat_parse_qs,
  44     compat_socket_create_connection,
  45     compat_str,
  46     compat_urllib_error,
  47     compat_urllib_parse,
  48     compat_urllib_parse_urlparse,
  49     compat_urllib_request,
  50     compat_urlparse,
  51     shlex_quote,
  52 )
  53
  54
  55 # This is not clearly defined otherwise
  56 compiled_regex_type = type(re.compile(''))
  57
  58 std_headers = {
  59     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
  60     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  61     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  62     'Accept-Encoding': 'gzip, deflate',
  63     'Accept-Language': 'en-us,en;q=0.5',
  64 }
  65
  66
  67 NO_DEFAULT = object()
  68
  69 ENGLISH_MONTH_NAMES = [
  70     'January', 'February', 'March', 'April', 'May', 'June',
  71     'July', 'August', 'September', 'October', 'November', 'December']
  72
  73
  74 def preferredencoding():
  75     """Get preferred encoding.
  76
  77     Returns the best encoding scheme for the system, based on
  78     locale.getpreferredencoding() and some further tweaks.
  79     """
  80     try:
  81         pref = locale.getpreferredencoding()
  82         'TEST'.encode(pref)
  83     except Exception:
  84         pref = 'UTF-8'
  85
  86     return pref
  87
  88
  89 def write_json_file(obj, fn):
  90     """ Encode obj as JSON and write it to fn, atomically if possible """
  91
  92     fn = encodeFilename(fn)
  93     if sys.version_info < (3, 0) and sys.platform != 'win32':
  94         encoding = get_filesystem_encoding()
  95         # os.path.basename returns a bytes object, but NamedTemporaryFile
  96         # will fail if the filename contains non ascii characters unless we
  97         # use a unicode object
  98         path_basename = lambda f: os.path.basename(fn).decode(encoding)
  99         # the same for os.path.dirname
 100         path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
 101     else:
 102         path_basename = os.path.basename
 103         path_dirname = os.path.dirname
 104
 105     args = {
 106         'suffix': '.tmp',
 107         'prefix': path_basename(fn) + '.',
 108         'dir': path_dirname(fn),
 109         'delete': False,
 110     }
 111
 112     # In Python 2.x, json.dump expects a bytestream.
 113     # In Python 3.x, it writes to a character stream
 114     if sys.version_info < (3, 0):
 115         args['mode'] = 'wb'
 116     else:
 117         args.update({
 118             'mode': 'w',
 119             'encoding': 'utf-8',
 120         })
 121
 122     tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
 123
 124     try:
 125         with tf:
 126             json.dump(obj, tf)
 127         if sys.platform == 'win32':
 128             # Need to remove existing file on Windows, else os.rename raises
 129             # WindowsError or FileExistsError.
 130             try:
 131                 os.unlink(fn)
 132             except OSError:
 133                 pass
 134         os.rename(tf.name, fn)
 135     except Exception:
 136         try:
 137             os.remove(tf.name)
 138         except OSError:
 139             pass
 140         raise
 141
 142
 143 if sys.version_info >= (2, 7):
 144     def find_xpath_attr(node, xpath, key, val=None):
 145         """ Find the xpath xpath[@key=val] """
 146         assert re.match(r'^[a-zA-Z_-]+$', key)
 147         if val:
 148             assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
 149         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
 150         return node.find(expr)
 151 else:
 152     def find_xpath_attr(node, xpath, key, val=None):
 153         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
 154         # .//node does not match if a node is a direct child of . !
 155         if isinstance(xpath, compat_str):
 156             xpath = xpath.encode('ascii')
 157
 158         for f in node.findall(xpath):
 159             if key not in f.attrib:
 160                 continue
 161             if val is None or f.attrib.get(key) == val:
 162                 return f
 163         return None
 164
 165 # On python2.6 the xml.etree.ElementTree.Element methods don't support
 166 # the namespace parameter
 167
 168
 169 def xpath_with_ns(path, ns_map):
 170     components = [c.split(':') for c in path.split('/')]
 171     replaced = []
 172     for c in components:
 173         if len(c) == 1:
 174             replaced.append(c[0])
 175         else:
 176             ns, tag = c
 177             replaced.append('{%s}%s' % (ns_map[ns], tag))
 178     return '/'.join(replaced)
 179
 180
 181 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
 182     def _find_xpath(xpath):
 183         if sys.version_info < (2, 7):  # Crazy 2.6
 184             xpath = xpath.encode('ascii')
 185         return node.find(xpath)
 186
 187     if isinstance(xpath, (str, compat_str)):
 188         n = _find_xpath(xpath)
 189     else:
 190         for xp in xpath:
 191             n = _find_xpath(xp)
 192             if n is not None:
 193                 break
 194
 195     if n is None:
 196         if default is not NO_DEFAULT:
 197             return default
 198         elif fatal:
 199             name = xpath if name is None else name
 200             raise ExtractorError('Could not find XML element %s' % name)
 201         else:
 202             return None
 203     return n
 204
 205
 206 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
 207     n = xpath_element(node, xpath, name, fatal=fatal, default=default)
 208     if n is None or n == default:
 209         return n
 210     if n.text is None:
 211         if default is not NO_DEFAULT:
 212             return default
 213         elif fatal:
 214             name = xpath if name is None else name
 215             raise ExtractorError('Could not find XML element\'s text %s' % name)
 216         else:
 217             return None
 218     return n.text
 219
 220
 221 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
 222     n = find_xpath_attr(node, xpath, key)
 223     if n is None:
 224         if default is not NO_DEFAULT:
 225             return default
 226         elif fatal:
 227             name = '%s[@%s]' % (xpath, key) if name is None else name
 228             raise ExtractorError('Could not find XML attribute %s' % name)
 229         else:
 230             return None
 231     return n.attrib[key]
 232
 233
 234 def get_element_by_id(id, html):
 235     """Return the content of the tag with the specified ID in the passed HTML document"""
 236     return get_element_by_attribute("id", id, html)
 237
 238
 239 def get_element_by_attribute(attribute, value, html):
 240     """Return the content of the tag with the specified attribute in the passed HTML document"""
 241
 242     m = re.search(r'''(?xs)
 243         <([a-zA-Z0-9:._-]+)
 244          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
 245          \s+%s=['"]?%s['"]?
 246          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
 247         \s*>
 248         (?P<content>.*?)
 249         </\1>
 250     ''' % (re.escape(attribute), re.escape(value)), html)
 251
 252     if not m:
 253         return None
 254     res = m.group('content')
 255
 256     if res.startswith('"') or res.startswith("'"):
 257         res = res[1:-1]
 258
 259     return unescapeHTML(res)
 260
 261
 262 def clean_html(html):
 263     """Clean an HTML snippet into a readable string"""
 264
 265     if html is None:  # Convenience for sanitizing descriptions etc.
 266         return html
 267
 268     # Newline vs <br />
 269     html = html.replace('\n', ' ')
 270     html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
 271     html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
 272     # Strip html tags
 273     html = re.sub('<.*?>', '', html)
 274     # Replace html entities
 275     html = unescapeHTML(html)
 276     return html.strip()
 277
 278
 279 def sanitize_open(filename, open_mode):
 280     """Try to open the given filename, and slightly tweak it if this fails.
 281
 282     Attempts to open the given filename. If this fails, it tries to change
 283     the filename slightly, step by step, until it's either able to open it
 284     or it fails and raises a final exception, like the standard open()
 285     function.
 286
 287     It returns the tuple (stream, definitive_file_name).
 288     """
 289     try:
 290         if filename == '-':
 291             if sys.platform == 'win32':
 292                 import msvcrt
 293                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 294             return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
 295         stream = open(encodeFilename(filename), open_mode)
 296         return (stream, filename)
 297     except (IOError, OSError) as err:
 298         if err.errno in (errno.EACCES,):
 299             raise
 300
 301         # In case of error, try to remove win32 forbidden chars
 302         alt_filename = sanitize_path(filename)
 303         if alt_filename == filename:
 304             raise
 305         else:
 306             # An exception here should be caught in the caller
 307             stream = open(encodeFilename(alt_filename), open_mode)
 308             return (stream, alt_filename)
 309
 310
 311 def timeconvert(timestr):
 312     """Convert RFC 2822 defined time string into system timestamp"""
 313     timestamp = None
 314     timetuple = email.utils.parsedate_tz(timestr)
 315     if timetuple is not None:
 316         timestamp = email.utils.mktime_tz(timetuple)
 317     return timestamp
 318
 319
 320 def sanitize_filename(s, restricted=False, is_id=False):
 321     """Sanitizes a string so it could be used as part of a filename.
 322     If restricted is set, use a stricter subset of allowed characters.
 323     Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
 324     """
 325     def replace_insane(char):
 326         if char == '?' or ord(char) < 32 or ord(char) == 127:
 327             return ''
 328         elif char == '"':
 329             return '' if restricted else '\''
 330         elif char == ':':
 331             return '_-' if restricted else ' -'
 332         elif char in '\\/|*<>':
 333             return '_'
 334         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
 335             return '_'
 336         if restricted and ord(char) > 127:
 337             return '_'
 338         return char
 339
 340     # Handle timestamps
 341     s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
 342     result = ''.join(map(replace_insane, s))
 343     if not is_id:
 344         while '__' in result:
 345             result = result.replace('__', '_')
 346         result = result.strip('_')
 347         # Common case of "Foreign band name - English song title"
 348         if restricted and result.startswith('-_'):
 349             result = result[2:]
 350         if result.startswith('-'):
 351             result = '_' + result[len('-'):]
 352         result = result.lstrip('.')
 353         if not result:
 354             result = '_'
 355     return result
 356
 357
 358 def sanitize_path(s):
 359     """Sanitizes and normalizes path on Windows"""
 360     if sys.platform != 'win32':
 361         return s
 362     drive_or_unc, _ = os.path.splitdrive(s)
 363     if sys.version_info < (2, 7) and not drive_or_unc:
 364         drive_or_unc, _ = os.path.splitunc(s)
 365     norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
 366     if drive_or_unc:
 367         norm_path.pop(0)
 368     sanitized_path = [
 369         path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
 370         for path_part in norm_path]
 371     if drive_or_unc:
 372         sanitized_path.insert(0, drive_or_unc + os.path.sep)
 373     return os.path.join(*sanitized_path)
 374
 375
 376 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
 377 # unwanted failures due to missing protocol
 378 def sanitized_Request(url, *args, **kwargs):
 379     return compat_urllib_request.Request(
 380         'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
 381
 382
 383 def orderedSet(iterable):
 384     """ Remove all duplicates from the input iterable """
 385     res = []
 386     for el in iterable:
 387         if el not in res:
 388             res.append(el)
 389     return res
 390
 391
 392 def _htmlentity_transform(entity):
 393     """Transforms an HTML entity to a character."""
 394     # Known non-numeric HTML entity
 395     if entity in compat_html_entities.name2codepoint:
 396         return compat_chr(compat_html_entities.name2codepoint[entity])
 397
 398     mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
 399     if mobj is not None:
 400         numstr = mobj.group(1)
 401         if numstr.startswith('x'):
 402             base = 16
 403             numstr = '0%s' % numstr
 404         else:
 405             base = 10
 406         # See https://github.com/rg3/youtube-dl/issues/7518
 407         try:
 408             return compat_chr(int(numstr, base))
 409         except ValueError:
 410             pass
 411
 412     # Unknown entity in name, return its literal representation
 413     return '&%s;' % entity
 414
 415
 416 def unescapeHTML(s):
 417     if s is None:
 418         return None
 419     assert type(s) == compat_str
 420
 421     return re.sub(
 422         r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
 423
 424
 425 def get_subprocess_encoding():
 426     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
 427         # For subprocess calls, encode with locale encoding
 428         # Refer to http://stackoverflow.com/a/9951851/35070
 429         encoding = preferredencoding()
 430     else:
 431         encoding = sys.getfilesystemencoding()
 432     if encoding is None:
 433         encoding = 'utf-8'
 434     return encoding
 435
 436
 437 def encodeFilename(s, for_subprocess=False):
 438     """
 439     @param s The name of the file
 440     """
 441
 442     assert type(s) == compat_str
 443
 444     # Python 3 has a Unicode API
 445     if sys.version_info >= (3, 0):
 446         return s
 447
 448     # Pass '' directly to use Unicode APIs on Windows 2000 and up
 449     # (Detecting Windows NT 4 is tricky because 'major >= 4' would
 450     # match Windows 9x series as well. Besides, NT 4 is obsolete.)
 451     if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
 452         return s
 453
 454     return s.encode(get_subprocess_encoding(), 'ignore')
 455
 456
 457 def decodeFilename(b, for_subprocess=False):
 458
 459     if sys.version_info >= (3, 0):
 460         return b
 461
 462     if not isinstance(b, bytes):
 463         return b
 464
 465     return b.decode(get_subprocess_encoding(), 'ignore')
 466
 467
 468 def encodeArgument(s):
 469     if not isinstance(s, compat_str):
 470         # Legacy code that uses byte strings
 471         # Uncomment the following line after fixing all post processors
 472         # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
 473         s = s.decode('ascii')
 474     return encodeFilename(s, True)
 475
 476
 477 def decodeArgument(b):
 478     return decodeFilename(b, True)
 479
 480
 481 def decodeOption(optval):
 482     if optval is None:
 483         return optval
 484     if isinstance(optval, bytes):
 485         optval = optval.decode(preferredencoding())
 486
 487     assert isinstance(optval, compat_str)
 488     return optval
 489
 490
 491 def formatSeconds(secs):
 492     if secs > 3600:
 493         return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
 494     elif secs > 60:
 495         return '%d:%02d' % (secs // 60, secs % 60)
 496     else:
 497         return '%d' % secs
 498
 499
 500 def make_HTTPS_handler(params, **kwargs):
 501     opts_no_check_certificate = params.get('nocheckcertificate', False)
 502     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
 503         context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
 504         if opts_no_check_certificate:
 505             context.check_hostname = False
 506             context.verify_mode = ssl.CERT_NONE
 507         try:
 508             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 509         except TypeError:
 510             # Python 2.7.8
 511             # (create_default_context present but HTTPSHandler has no context=)
 512             pass
 513
 514     if sys.version_info < (3, 2):
 515         return YoutubeDLHTTPSHandler(params, **kwargs)
 516     else:  # Python < 3.4
 517         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
 518         context.verify_mode = (ssl.CERT_NONE
 519                                if opts_no_check_certificate
 520                                else ssl.CERT_REQUIRED)
 521         context.set_default_verify_paths()
 522         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 523
 524
 525 def bug_reports_message():
 526     if ytdl_is_updateable():
 527         update_cmd = 'type  youtube-dl -U  to update'
 528     else:
 529         update_cmd = 'see  https://yt-dl.org/update  on how to update'
 530     msg = '; please report this issue on https://yt-dl.org/bug .'
 531     msg += ' Make sure you are using the latest version; %s.' % update_cmd
 532     msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
 533     return msg
 534
 535
 536 class ExtractorError(Exception):
 537     """Error during info extraction."""
 538
 539     def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
 540         """ tb, if given, is the original traceback (so that it can be printed out).
 541         If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
 542         """
 543
 544         if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
 545             expected = True
 546         if video_id is not None:
 547             msg = video_id + ': ' + msg
 548         if cause:
 549             msg += ' (caused by %r)' % cause
 550         if not expected:
 551             msg += bug_reports_message()
 552         super(ExtractorError, self).__init__(msg)
 553
 554         self.traceback = tb
 555         self.exc_info = sys.exc_info()  # preserve original exception
 556         self.cause = cause
 557         self.video_id = video_id
 558
 559     def format_traceback(self):
 560         if self.traceback is None:
 561             return None
 562         return ''.join(traceback.format_tb(self.traceback))
 563
 564
 565 class UnsupportedError(ExtractorError):
 566     def __init__(self, url):
 567         super(UnsupportedError, self).__init__(
 568             'Unsupported URL: %s' % url, expected=True)
 569         self.url = url
 570
 571
 572 class RegexNotFoundError(ExtractorError):
 573     """Error when a regex didn't match"""
 574     pass
 575
 576
 577 class DownloadError(Exception):
 578     """Download Error exception.
 579
 580     This exception may be thrown by FileDownloader objects if they are not
 581     configured to continue on errors. They will contain the appropriate
 582     error message.
 583     """
 584
 585     def __init__(self, msg, exc_info=None):
 586         """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
 587         super(DownloadError, self).__init__(msg)
 588         self.exc_info = exc_info
 589
 590
 591 class SameFileError(Exception):
 592     """Same File exception.
 593
 594     This exception will be thrown by FileDownloader objects if they detect
 595     multiple files would have to be downloaded to the same file on disk.
 596     """
 597     pass
 598
 599
 600 class PostProcessingError(Exception):
 601     """Post Processing exception.
 602
 603     This exception may be raised by PostProcessor's .run() method to
 604     indicate an error in the postprocessing task.
 605     """
 606
 607     def __init__(self, msg):
 608         self.msg = msg
 609
 610
 611 class MaxDownloadsReached(Exception):
 612     """ --max-downloads limit has been reached. """
 613     pass
 614
 615
 616 class UnavailableVideoError(Exception):
 617     """Unavailable Format exception.
 618
 619     This exception will be thrown when a video is requested
 620     in a format that is not available for that video.
 621     """
 622     pass
 623
 624
 625 class ContentTooShortError(Exception):
 626     """Content Too Short exception.
 627
 628     This exception may be raised by FileDownloader objects when a file they
 629     download is too small for what the server announced first, indicating
 630     the connection was probably interrupted.
 631     """
 632
 633     def __init__(self, downloaded, expected):
 634         # Both in bytes
 635         self.downloaded = downloaded
 636         self.expected = expected
 637
 638
 639 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
 640     # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
 641     # expected HTTP responses to meet HTTP/1.0 or later (see also
 642     # https://github.com/rg3/youtube-dl/issues/6727)
 643     if sys.version_info < (3, 0):
 644         kwargs[b'strict'] = True
 645     hc = http_class(*args, **kwargs)
 646     source_address = ydl_handler._params.get('source_address')
 647     if source_address is not None:
 648         sa = (source_address, 0)
 649         if hasattr(hc, 'source_address'):  # Python 2.7+
 650             hc.source_address = sa
 651         else:  # Python 2.6
 652             def _hc_connect(self, *args, **kwargs):
 653                 sock = compat_socket_create_connection(
 654                     (self.host, self.port), self.timeout, sa)
 655                 if is_https:
 656                     self.sock = ssl.wrap_socket(
 657                         sock, self.key_file, self.cert_file,
 658                         ssl_version=ssl.PROTOCOL_TLSv1)
 659                 else:
 660                     self.sock = sock
 661             hc.connect = functools.partial(_hc_connect, hc)
 662
 663     return hc
 664
 665
 666 def handle_youtubedl_headers(headers):
 667     filtered_headers = headers
 668
 669     if 'Youtubedl-no-compression' in filtered_headers:
 670         filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
 671         del filtered_headers['Youtubedl-no-compression']
 672
 673     return filtered_headers
 674
 675
 676 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 677     """Handler for HTTP requests and responses.
 678
 679     This class, when installed with an OpenerDirector, automatically adds
 680     the standard headers to every HTTP request and handles gzipped and
 681     deflated responses from web servers. If compression is to be avoided in
 682     a particular request, the original request in the program code only has
 683     to include the HTTP header "Youtubedl-no-compression", which will be
 684     removed before making the real request.
 685
 686     Part of this code was copied from:
 687
 688     http://techknack.net/python-urllib2-handlers/
 689
 690     Andrew Rowls, the author of that code, agreed to release it to the
 691     public domain.
 692     """
 693
 694     def __init__(self, params, *args, **kwargs):
 695         compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
 696         self._params = params
 697
 698     def http_open(self, req):
 699         return self.do_open(functools.partial(
 700             _create_http_connection, self, compat_http_client.HTTPConnection, False),
 701             req)
 702
 703     @staticmethod
 704     def deflate(data):
 705         try:
 706             return zlib.decompress(data, -zlib.MAX_WBITS)
 707         except zlib.error:
 708             return zlib.decompress(data)
 709
 710     @staticmethod
 711     def addinfourl_wrapper(stream, headers, url, code):
 712         if hasattr(compat_urllib_request.addinfourl, 'getcode'):
 713             return compat_urllib_request.addinfourl(stream, headers, url, code)
 714         ret = compat_urllib_request.addinfourl(stream, headers, url)
 715         ret.code = code
 716         return ret
 717
 718     def http_request(self, req):
 719         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
 720         # always respected by websites, some tend to give out URLs with non percent-encoded
 721         # non-ASCII characters (see telemb.py, ard.py [#3412])
 722         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
 723         # To work around aforementioned issue we will replace request's original URL with
 724         # percent-encoded one
 725         # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
 726         # the code of this workaround has been moved here from YoutubeDL.urlopen()
 727         url = req.get_full_url()
 728         url_escaped = escape_url(url)
 729
 730         # Substitute URL if any change after escaping
 731         if url != url_escaped:
 732             req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
 733             new_req = req_type(
 734                 url_escaped, data=req.data, headers=req.headers,
 735                 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
 736             new_req.timeout = req.timeout
 737             req = new_req
 738
 739         for h, v in std_headers.items():
 740             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
 741             # The dict keys are capitalized because of this bug by urllib
 742             if h.capitalize() not in req.headers:
 743                 req.add_header(h, v)
 744
 745         req.headers = handle_youtubedl_headers(req.headers)
 746
 747         if sys.version_info < (2, 7) and '#' in req.get_full_url():
 748             # Python 2.6 is brain-dead when it comes to fragments
 749             req._Request__original = req._Request__original.partition('#')[0]
 750             req._Request__r_type = req._Request__r_type.partition('#')[0]
 751
 752         return req
 753
 754     def http_response(self, req, resp):
 755         old_resp = resp
 756         # gzip
 757         if resp.headers.get('Content-encoding', '') == 'gzip':
 758             content = resp.read()
 759             gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
 760             try:
 761                 uncompressed = io.BytesIO(gz.read())
 762             except IOError as original_ioerror:
 763                 # There may be junk add the end of the file
 764                 # See http://stackoverflow.com/q/4928560/35070 for details
 765                 for i in range(1, 1024):
 766                     try:
 767                         gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
 768                         uncompressed = io.BytesIO(gz.read())
 769                     except IOError:
 770                         continue
 771                     break
 772                 else:
 773                     raise original_ioerror
 774             resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
 775             resp.msg = old_resp.msg
 776         # deflate
 777         if resp.headers.get('Content-encoding', '') == 'deflate':
 778             gz = io.BytesIO(self.deflate(resp.read()))
 779             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 780             resp.msg = old_resp.msg
 781         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
 782         # https://github.com/rg3/youtube-dl/issues/6457).
 783         if 300 <= resp.code < 400:
 784             location = resp.headers.get('Location')
 785             if location:
 786                 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
 787                 if sys.version_info >= (3, 0):
 788                     location = location.encode('iso-8859-1').decode('utf-8')
 789                 location_escaped = escape_url(location)
 790                 if location != location_escaped:
 791                     del resp.headers['Location']
 792                     resp.headers['Location'] = location_escaped
 793         return resp
 794
 795     https_request = http_request
 796     https_response = http_response
 797
 798
 799 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
 800     def __init__(self, params, https_conn_class=None, *args, **kwargs):
 801         compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
 802         self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
 803         self._params = params
 804
 805     def https_open(self, req):
 806         kwargs = {}
 807         if hasattr(self, '_context'):  # python > 2.6
 808             kwargs['context'] = self._context
 809         if hasattr(self, '_check_hostname'):  # python 3.x
 810             kwargs['check_hostname'] = self._check_hostname
 811         return self.do_open(functools.partial(
 812             _create_http_connection, self, self._https_conn_class, True),
 813             req, **kwargs)
 814
 815
 816 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
 817     def __init__(self, cookiejar=None):
 818         compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
 819
 820     def http_response(self, request, response):
 821         # Python 2 will choke on next HTTP request in row if there are non-ASCII
 822         # characters in Set-Cookie HTTP header of last response (see
 823         # https://github.com/rg3/youtube-dl/issues/6769).
 824         # In order to at least prevent crashing we will percent encode Set-Cookie
 825         # header before HTTPCookieProcessor starts processing it.
 826         # if sys.version_info < (3, 0) and response.headers:
 827         #     for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
 828         #         set_cookie = response.headers.get(set_cookie_header)
 829         #         if set_cookie:
 830         #             set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
 831         #             if set_cookie != set_cookie_escaped:
 832         #                 del response.headers[set_cookie_header]
 833         #                 response.headers[set_cookie_header] = set_cookie_escaped
 834         return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
 835
 836     https_request = compat_urllib_request.HTTPCookieProcessor.http_request
 837     https_response = http_response
 838
 839
 840 def parse_iso8601(date_str, delimiter='T', timezone=None):
 841     """ Return a UNIX timestamp from the given date """
 842
 843     if date_str is None:
 844         return None
 845
 846     date_str = re.sub(r'\.[0-9]+', '', date_str)
 847
 848     if timezone is None:
 849         m = re.search(
 850             r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
 851             date_str)
 852         if not m:
 853             timezone = datetime.timedelta()
 854         else:
 855             date_str = date_str[:-len(m.group(0))]
 856             if not m.group('sign'):
 857                 timezone = datetime.timedelta()
 858             else:
 859                 sign = 1 if m.group('sign') == '+' else -1
 860                 timezone = datetime.timedelta(
 861                     hours=sign * int(m.group('hours')),
 862                     minutes=sign * int(m.group('minutes')))
 863     try:
 864         date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
 865         dt = datetime.datetime.strptime(date_str, date_format) - timezone
 866         return calendar.timegm(dt.timetuple())
 867     except ValueError:
 868         pass
 869
 870
 871 def unified_strdate(date_str, day_first=True):
 872     """Return a string with the date in the format YYYYMMDD"""
 873
 874     if date_str is None:
 875         return None
 876     upload_date = None
 877     # Replace commas
 878     date_str = date_str.replace(',', ' ')
 879     # %z (UTC offset) is only supported in python>=3.2
 880     if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
 881         date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
 882     # Remove AM/PM + timezone
 883     date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 884
 885     format_expressions = [
 886         '%d %B %Y',
 887         '%d %b %Y',
 888         '%B %d %Y',
 889         '%b %d %Y',
 890         '%b %dst %Y %I:%M%p',
 891         '%b %dnd %Y %I:%M%p',
 892         '%b %dth %Y %I:%M%p',
 893         '%Y %m %d',
 894         '%Y-%m-%d',
 895         '%Y/%m/%d',
 896         '%Y/%m/%d %H:%M:%S',
 897         '%Y-%m-%d %H:%M:%S',
 898         '%Y-%m-%d %H:%M:%S.%f',
 899         '%d.%m.%Y %H:%M',
 900         '%d.%m.%Y %H.%M',
 901         '%Y-%m-%dT%H:%M:%SZ',
 902         '%Y-%m-%dT%H:%M:%S.%fZ',
 903         '%Y-%m-%dT%H:%M:%S.%f0Z',
 904         '%Y-%m-%dT%H:%M:%S',
 905         '%Y-%m-%dT%H:%M:%S.%f',
 906         '%Y-%m-%dT%H:%M',
 907     ]
 908     if day_first:
 909         format_expressions.extend([
 910             '%d-%m-%Y',
 911             '%d.%m.%Y',
 912             '%d/%m/%Y',
 913             '%d/%m/%y',
 914             '%d/%m/%Y %H:%M:%S',
 915         ])
 916     else:
 917         format_expressions.extend([
 918             '%m-%d-%Y',
 919             '%m.%d.%Y',
 920             '%m/%d/%Y',
 921             '%m/%d/%y',
 922             '%m/%d/%Y %H:%M:%S',
 923         ])
 924     for expression in format_expressions:
 925         try:
 926             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
 927         except ValueError:
 928             pass
 929     if upload_date is None:
 930         timetuple = email.utils.parsedate_tz(date_str)
 931         if timetuple:
 932             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
 933     if upload_date is not None:
 934         return compat_str(upload_date)
 935
 936
 937 def determine_ext(url, default_ext='unknown_video'):
 938     if url is None:
 939         return default_ext
 940     guess = url.partition('?')[0].rpartition('.')[2]
 941     if re.match(r'^[A-Za-z0-9]+$', guess):
 942         return guess
 943     elif guess.rstrip('/') in (
 944             'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
 945             'flv', 'f4v', 'f4a', 'f4b',
 946             'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
 947             'mkv', 'mka', 'mk3d',
 948             'avi', 'divx',
 949             'mov',
 950             'asf', 'wmv', 'wma',
 951             '3gp', '3g2',
 952             'mp3',
 953             'flac',
 954             'ape',
 955             'wav',
 956             'f4f', 'f4m', 'm3u8', 'smil'):
 957         return guess.rstrip('/')
 958     else:
 959         return default_ext
 960
 961
 962 def subtitles_filename(filename, sub_lang, sub_format):
 963     return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
 964
 965
 966 def date_from_str(date_str):
 967     """
 968     Return a datetime object from a string in the format YYYYMMDD or
 969     (now|today)[+-][0-9](day|week|month|year)(s)?"""
 970     today = datetime.date.today()
 971     if date_str in ('now', 'today'):
 972         return today
 973     if date_str == 'yesterday':
 974         return today - datetime.timedelta(days=1)
 975     match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
 976     if match is not None:
 977         sign = match.group('sign')
 978         time = int(match.group('time'))
 979         if sign == '-':
 980             time = -time
 981         unit = match.group('unit')
 982         # A bad aproximation?
 983         if unit == 'month':
 984             unit = 'day'
 985             time *= 30
 986         elif unit == 'year':
 987             unit = 'day'
 988             time *= 365
 989         unit += 's'
 990         delta = datetime.timedelta(**{unit: time})
 991         return today + delta
 992     return datetime.datetime.strptime(date_str, "%Y%m%d").date()
 993
 994
 995 def hyphenate_date(date_str):
 996     """
 997     Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
 998     match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
 999     if match is not None:
1000         return '-'.join(match.groups())
1001     else:
1002         return date_str
1003
1004
1005 class DateRange(object):
1006     """Represents a time interval between two dates"""
1007
1008     def __init__(self, start=None, end=None):
1009         """start and end must be strings in the format accepted by date"""
1010         if start is not None:
1011             self.start = date_from_str(start)
1012         else:
1013             self.start = datetime.datetime.min.date()
1014         if end is not None:
1015             self.end = date_from_str(end)
1016         else:
1017             self.end = datetime.datetime.max.date()
1018         if self.start > self.end:
1019             raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1020
1021     @classmethod
1022     def day(cls, day):
1023         """Returns a range that only contains the given day"""
1024         return cls(day, day)
1025
1026     def __contains__(self, date):
1027         """Check if the date is in the range"""
1028         if not isinstance(date, datetime.date):
1029             date = date_from_str(date)
1030         return self.start <= date <= self.end
1031
1032     def __str__(self):
1033         return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1034
1035
1036 def platform_name():
1037     """ Returns the platform name as a compat_str """
1038     res = platform.platform()
1039     if isinstance(res, bytes):
1040         res = res.decode(preferredencoding())
1041
1042     assert isinstance(res, compat_str)
1043     return res
1044
1045
1046 def _windows_write_string(s, out):
1047     """ Returns True if the string was written using special methods,
1048     False if it has yet to be written out."""
1049     # Adapted from http://stackoverflow.com/a/3259271/35070
1050
1051     import ctypes
1052     import ctypes.wintypes
1053
1054     WIN_OUTPUT_IDS = {
1055         1: -11,
1056         2: -12,
1057     }
1058
1059     try:
1060         fileno = out.fileno()
1061     except AttributeError:
1062         # If the output stream doesn't have a fileno, it's virtual
1063         return False
1064     except io.UnsupportedOperation:
1065         # Some strange Windows pseudo files?
1066         return False
1067     if fileno not in WIN_OUTPUT_IDS:
1068         return False
1069
1070     GetStdHandle = ctypes.WINFUNCTYPE(
1071         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1072         (b"GetStdHandle", ctypes.windll.kernel32))
1073     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1074
1075     WriteConsoleW = ctypes.WINFUNCTYPE(
1076         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1077         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1078         ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
1079     written = ctypes.wintypes.DWORD(0)
1080
1081     GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
1082     FILE_TYPE_CHAR = 0x0002
1083     FILE_TYPE_REMOTE = 0x8000
1084     GetConsoleMode = ctypes.WINFUNCTYPE(
1085         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1086         ctypes.POINTER(ctypes.wintypes.DWORD))(
1087         (b"GetConsoleMode", ctypes.windll.kernel32))
1088     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1089
1090     def not_a_console(handle):
1091         if handle == INVALID_HANDLE_VALUE or handle is None:
1092             return True
1093         return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1094                 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1095
1096     if not_a_console(h):
1097         return False
1098
1099     def next_nonbmp_pos(s):
1100         try:
1101             return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1102         except StopIteration:
1103             return len(s)
1104
1105     while s:
1106         count = min(next_nonbmp_pos(s), 1024)
1107
1108         ret = WriteConsoleW(
1109             h, s, count if count else 2, ctypes.byref(written), None)
1110         if ret == 0:
1111             raise OSError('Failed to write string')
1112         if not count:  # We just wrote a non-BMP character
1113             assert written.value == 2
1114             s = s[1:]
1115         else:
1116             assert written.value > 0
1117             s = s[written.value:]
1118     return True
1119
1120
1121 def write_string(s, out=None, encoding=None):
1122     if out is None:
1123         out = sys.stderr
1124     assert type(s) == compat_str
1125
1126     if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1127         if _windows_write_string(s, out):
1128             return
1129
1130     if ('b' in getattr(out, 'mode', '') or
1131             sys.version_info[0] < 3):  # Python 2 lies about mode of sys.stderr
1132         byt = s.encode(encoding or preferredencoding(), 'ignore')
1133         out.write(byt)
1134     elif hasattr(out, 'buffer'):
1135         enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1136         byt = s.encode(enc, 'ignore')
1137         out.buffer.write(byt)
1138     else:
1139         out.write(s)
1140     out.flush()
1141
1142
1143 def bytes_to_intlist(bs):
1144     if not bs:
1145         return []
1146     if isinstance(bs[0], int):  # Python 3
1147         return list(bs)
1148     else:
1149         return [ord(c) for c in bs]
1150
1151
1152 def intlist_to_bytes(xs):
1153     if not xs:
1154         return b''
1155     return struct_pack('%dB' % len(xs), *xs)
1156
1157
1158 # Cross-platform file locking
1159 if sys.platform == 'win32':
1160     import ctypes.wintypes
1161     import msvcrt
1162
1163     class OVERLAPPED(ctypes.Structure):
1164         _fields_ = [
1165             ('Internal', ctypes.wintypes.LPVOID),
1166             ('InternalHigh', ctypes.wintypes.LPVOID),
1167             ('Offset', ctypes.wintypes.DWORD),
1168             ('OffsetHigh', ctypes.wintypes.DWORD),
1169             ('hEvent', ctypes.wintypes.HANDLE),
1170         ]
1171
1172     kernel32 = ctypes.windll.kernel32
1173     LockFileEx = kernel32.LockFileEx
1174     LockFileEx.argtypes = [
1175         ctypes.wintypes.HANDLE,     # hFile
1176         ctypes.wintypes.DWORD,      # dwFlags
1177         ctypes.wintypes.DWORD,      # dwReserved
1178         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
1179         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
1180         ctypes.POINTER(OVERLAPPED)  # Overlapped
1181     ]
1182     LockFileEx.restype = ctypes.wintypes.BOOL
1183     UnlockFileEx = kernel32.UnlockFileEx
1184     UnlockFileEx.argtypes = [
1185         ctypes.wintypes.HANDLE,     # hFile
1186         ctypes.wintypes.DWORD,      # dwReserved
1187         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
1188         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
1189         ctypes.POINTER(OVERLAPPED)  # Overlapped
1190     ]
1191     UnlockFileEx.restype = ctypes.wintypes.BOOL
1192     whole_low = 0xffffffff
1193     whole_high = 0x7fffffff
1194
1195     def _lock_file(f, exclusive):
1196         overlapped = OVERLAPPED()
1197         overlapped.Offset = 0
1198         overlapped.OffsetHigh = 0
1199         overlapped.hEvent = 0
1200         f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1201         handle = msvcrt.get_osfhandle(f.fileno())
1202         if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1203                           whole_low, whole_high, f._lock_file_overlapped_p):
1204             raise OSError('Locking file failed: %r' % ctypes.FormatError())
1205
1206     def _unlock_file(f):
1207         assert f._lock_file_overlapped_p
1208         handle = msvcrt.get_osfhandle(f.fileno())
1209         if not UnlockFileEx(handle, 0,
1210                             whole_low, whole_high, f._lock_file_overlapped_p):
1211             raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1212
1213 else:
1214     import fcntl
1215
1216     def _lock_file(f, exclusive):
1217         fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1218
1219     def _unlock_file(f):
1220         fcntl.flock(f, fcntl.LOCK_UN)
1221
1222
1223 class locked_file(object):
1224     def __init__(self, filename, mode, encoding=None):
1225         assert mode in ['r', 'a', 'w']
1226         self.f = io.open(filename, mode, encoding=encoding)
1227         self.mode = mode
1228
1229     def __enter__(self):
1230         exclusive = self.mode != 'r'
1231         try:
1232             _lock_file(self.f, exclusive)
1233         except IOError:
1234             self.f.close()
1235             raise
1236         return self
1237
1238     def __exit__(self, etype, value, traceback):
1239         try:
1240             _unlock_file(self.f)
1241         finally:
1242             self.f.close()
1243
1244     def __iter__(self):
1245         return iter(self.f)
1246
1247     def write(self, *args):
1248         return self.f.write(*args)
1249
1250     def read(self, *args):
1251         return self.f.read(*args)
1252
1253
1254 def get_filesystem_encoding():
1255     encoding = sys.getfilesystemencoding()
1256     return encoding if encoding is not None else 'utf-8'
1257
1258
1259 def shell_quote(args):
1260     quoted_args = []
1261     encoding = get_filesystem_encoding()
1262     for a in args:
1263         if isinstance(a, bytes):
1264             # We may get a filename encoded with 'encodeFilename'
1265             a = a.decode(encoding)
1266         quoted_args.append(pipes.quote(a))
1267     return ' '.join(quoted_args)
1268
1269
1270 def smuggle_url(url, data):
1271     """ Pass additional data in a URL for internal use. """
1272
1273     sdata = compat_urllib_parse.urlencode(
1274         {'__youtubedl_smuggle': json.dumps(data)})
1275     return url + '#' + sdata
1276
1277
1278 def unsmuggle_url(smug_url, default=None):
1279     if '#__youtubedl_smuggle' not in smug_url:
1280         return smug_url, default
1281     url, _, sdata = smug_url.rpartition('#')
1282     jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1283     data = json.loads(jsond)
1284     return url, data
1285
1286
1287 def format_bytes(bytes):
1288     if bytes is None:
1289         return 'N/A'
1290     if type(bytes) is str:
1291         bytes = float(bytes)
1292     if bytes == 0.0:
1293         exponent = 0
1294     else:
1295         exponent = int(math.log(bytes, 1024.0))
1296     suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1297     converted = float(bytes) / float(1024 ** exponent)
1298     return '%.2f%s' % (converted, suffix)
1299
1300
1301 def parse_filesize(s):
1302     if s is None:
1303         return None
1304
1305     # The lower-case forms are of course incorrect and inofficial,
1306     # but we support those too
1307     _UNIT_TABLE = {
1308         'B': 1,
1309         'b': 1,
1310         'KiB': 1024,
1311         'KB': 1000,
1312         'kB': 1024,
1313         'Kb': 1000,
1314         'MiB': 1024 ** 2,
1315         'MB': 1000 ** 2,
1316         'mB': 1024 ** 2,
1317         'Mb': 1000 ** 2,
1318         'GiB': 1024 ** 3,
1319         'GB': 1000 ** 3,
1320         'gB': 1024 ** 3,
1321         'Gb': 1000 ** 3,
1322         'TiB': 1024 ** 4,
1323         'TB': 1000 ** 4,
1324         'tB': 1024 ** 4,
1325         'Tb': 1000 ** 4,
1326         'PiB': 1024 ** 5,
1327         'PB': 1000 ** 5,
1328         'pB': 1024 ** 5,
1329         'Pb': 1000 ** 5,
1330         'EiB': 1024 ** 6,
1331         'EB': 1000 ** 6,
1332         'eB': 1024 ** 6,
1333         'Eb': 1000 ** 6,
1334         'ZiB': 1024 ** 7,
1335         'ZB': 1000 ** 7,
1336         'zB': 1024 ** 7,
1337         'Zb': 1000 ** 7,
1338         'YiB': 1024 ** 8,
1339         'YB': 1000 ** 8,
1340         'yB': 1024 ** 8,
1341         'Yb': 1000 ** 8,
1342     }
1343
1344     units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1345     m = re.match(
1346         r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1347     if not m:
1348         return None
1349
1350     num_str = m.group('num').replace(',', '.')
1351     mult = _UNIT_TABLE[m.group('unit')]
1352     return int(float(num_str) * mult)
1353
1354
1355 def month_by_name(name):
1356     """ Return the number of a month by (locale-independently) English name """
1357
1358     try:
1359         return ENGLISH_MONTH_NAMES.index(name) + 1
1360     except ValueError:
1361         return None
1362
1363
1364 def month_by_abbreviation(abbrev):
1365     """ Return the number of a month by (locale-independently) English
1366         abbreviations """
1367
1368     try:
1369         return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1370     except ValueError:
1371         return None
1372
1373
1374 def fix_xml_ampersands(xml_str):
1375     """Replace all the '&' by '&amp;' in XML"""
1376     return re.sub(
1377         r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1378         '&amp;',
1379         xml_str)
1380
1381
1382 def setproctitle(title):
1383     assert isinstance(title, compat_str)
1384     try:
1385         libc = ctypes.cdll.LoadLibrary("libc.so.6")
1386     except OSError:
1387         return
1388     title_bytes = title.encode('utf-8')
1389     buf = ctypes.create_string_buffer(len(title_bytes))
1390     buf.value = title_bytes
1391     try:
1392         libc.prctl(15, buf, 0, 0, 0)
1393     except AttributeError:
1394         return  # Strange libc, just skip this
1395
1396
1397 def remove_start(s, start):
1398     if s.startswith(start):
1399         return s[len(start):]
1400     return s
1401
1402
1403 def remove_end(s, end):
1404     if s.endswith(end):
1405         return s[:-len(end)]
1406     return s
1407
1408
1409 def remove_quotes(s):
1410     if s is None or len(s) < 2:
1411         return s
1412     for quote in ('"', "'", ):
1413         if s[0] == quote and s[-1] == quote:
1414             return s[1:-1]
1415     return s
1416
1417
1418 def url_basename(url):
1419     path = compat_urlparse.urlparse(url).path
1420     return path.strip('/').split('/')[-1]
1421
1422
1423 class HEADRequest(compat_urllib_request.Request):
1424     def get_method(self):
1425         return "HEAD"
1426
1427
1428 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1429     if get_attr:
1430         if v is not None:
1431             v = getattr(v, get_attr, None)
1432     if v == '':
1433         v = None
1434     if v is None:
1435         return default
1436     try:
1437         return int(v) * invscale // scale
1438     except ValueError:
1439         return default
1440
1441
1442 def str_or_none(v, default=None):
1443     return default if v is None else compat_str(v)
1444
1445
1446 def str_to_int(int_str):
1447     """ A more relaxed version of int_or_none """
1448     if int_str is None:
1449         return None
1450     int_str = re.sub(r'[,\.\+]', '', int_str)
1451     return int(int_str)
1452
1453
1454 def float_or_none(v, scale=1, invscale=1, default=None):
1455     if v is None:
1456         return default
1457     try:
1458         return float(v) * invscale / scale
1459     except ValueError:
1460         return default
1461
1462
1463 def parse_duration(s):
1464     if not isinstance(s, compat_basestring):
1465         return None
1466
1467     s = s.strip()
1468
1469     m = re.match(
1470         r'''(?ix)(?:P?T)?
1471         (?:
1472             (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1473             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1474
1475             \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1476             (?:
1477                 (?:
1478                     (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1479                     (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1480                 )?
1481                 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1482             )?
1483             (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1484         )$''', s)
1485     if not m:
1486         return None
1487     res = 0
1488     if m.group('only_mins'):
1489         return float_or_none(m.group('only_mins'), invscale=60)
1490     if m.group('only_hours'):
1491         return float_or_none(m.group('only_hours'), invscale=60 * 60)
1492     if m.group('secs'):
1493         res += int(m.group('secs'))
1494     if m.group('mins_reversed'):
1495         res += int(m.group('mins_reversed')) * 60
1496     if m.group('mins'):
1497         res += int(m.group('mins')) * 60
1498     if m.group('hours'):
1499         res += int(m.group('hours')) * 60 * 60
1500     if m.group('hours_reversed'):
1501         res += int(m.group('hours_reversed')) * 60 * 60
1502     if m.group('days'):
1503         res += int(m.group('days')) * 24 * 60 * 60
1504     if m.group('ms'):
1505         res += float(m.group('ms'))
1506     return res
1507
1508
1509 def prepend_extension(filename, ext, expected_real_ext=None):
1510     name, real_ext = os.path.splitext(filename)
1511     return (
1512         '{0}.{1}{2}'.format(name, ext, real_ext)
1513         if not expected_real_ext or real_ext[1:] == expected_real_ext
1514         else '{0}.{1}'.format(filename, ext))
1515
1516
1517 def replace_extension(filename, ext, expected_real_ext=None):
1518     name, real_ext = os.path.splitext(filename)
1519     return '{0}.{1}'.format(
1520         name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1521         ext)
1522
1523
1524 def check_executable(exe, args=[]):
1525     """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1526     args can be a list of arguments for a short output (like -version) """
1527     try:
1528         subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1529     except OSError:
1530         return False
1531     return exe
1532
1533
1534 def get_exe_version(exe, args=['--version'],
1535                     version_re=None, unrecognized='present'):
1536     """ Returns the version of the specified executable,
1537     or False if the executable is not present """
1538     try:
1539         out, _ = subprocess.Popen(
1540             [encodeArgument(exe)] + args,
1541             stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1542     except OSError:
1543         return False
1544     if isinstance(out, bytes):  # Python 2.x
1545         out = out.decode('ascii', 'ignore')
1546     return detect_exe_version(out, version_re, unrecognized)
1547
1548
1549 def detect_exe_version(output, version_re=None, unrecognized='present'):
1550     assert isinstance(output, compat_str)
1551     if version_re is None:
1552         version_re = r'version\s+([-0-9._a-zA-Z]+)'
1553     m = re.search(version_re, output)
1554     if m:
1555         return m.group(1)
1556     else:
1557         return unrecognized
1558
1559
1560 class PagedList(object):
1561     def __len__(self):
1562         # This is only useful for tests
1563         return len(self.getslice())
1564
1565
1566 class OnDemandPagedList(PagedList):
1567     def __init__(self, pagefunc, pagesize):
1568         self._pagefunc = pagefunc
1569         self._pagesize = pagesize
1570
1571     def getslice(self, start=0, end=None):
1572         res = []
1573         for pagenum in itertools.count(start // self._pagesize):
1574             firstid = pagenum * self._pagesize
1575             nextfirstid = pagenum * self._pagesize + self._pagesize
1576             if start >= nextfirstid:
1577                 continue
1578
1579             page_results = list(self._pagefunc(pagenum))
1580
1581             startv = (
1582                 start % self._pagesize
1583                 if firstid <= start < nextfirstid
1584                 else 0)
1585
1586             endv = (
1587                 ((end - 1) % self._pagesize) + 1
1588                 if (end is not None and firstid <= end <= nextfirstid)
1589                 else None)
1590
1591             if startv != 0 or endv is not None:
1592                 page_results = page_results[startv:endv]
1593             res.extend(page_results)
1594
1595             # A little optimization - if current page is not "full", ie. does
1596             # not contain page_size videos then we can assume that this page
1597             # is the last one - there are no more ids on further pages -
1598             # i.e. no need to query again.
1599             if len(page_results) + startv < self._pagesize:
1600                 break
1601
1602             # If we got the whole page, but the next page is not interesting,
1603             # break out early as well
1604             if end == nextfirstid:
1605                 break
1606         return res
1607
1608
1609 class InAdvancePagedList(PagedList):
1610     def __init__(self, pagefunc, pagecount, pagesize):
1611         self._pagefunc = pagefunc
1612         self._pagecount = pagecount
1613         self._pagesize = pagesize
1614
1615     def getslice(self, start=0, end=None):
1616         res = []
1617         start_page = start // self._pagesize
1618         end_page = (
1619             self._pagecount if end is None else (end // self._pagesize + 1))
1620         skip_elems = start - start_page * self._pagesize
1621         only_more = None if end is None else end - start
1622         for pagenum in range(start_page, end_page):
1623             page = list(self._pagefunc(pagenum))
1624             if skip_elems:
1625                 page = page[skip_elems:]
1626                 skip_elems = None
1627             if only_more is not None:
1628                 if len(page) < only_more:
1629                     only_more -= len(page)
1630                 else:
1631                     page = page[:only_more]
1632                     res.extend(page)
1633                     break
1634             res.extend(page)
1635         return res
1636
1637
1638 def uppercase_escape(s):
1639     unicode_escape = codecs.getdecoder('unicode_escape')
1640     return re.sub(
1641         r'\\U[0-9a-fA-F]{8}',
1642         lambda m: unicode_escape(m.group(0))[0],
1643         s)
1644
1645
1646 def lowercase_escape(s):
1647     unicode_escape = codecs.getdecoder('unicode_escape')
1648     return re.sub(
1649         r'\\u[0-9a-fA-F]{4}',
1650         lambda m: unicode_escape(m.group(0))[0],
1651         s)
1652
1653
1654 def escape_rfc3986(s):
1655     """Escape non-ASCII characters as suggested by RFC 3986"""
1656     if sys.version_info < (3, 0) and isinstance(s, compat_str):
1657         s = s.encode('utf-8')
1658     return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1659
1660
1661 def escape_url(url):
1662     """Escape URL as suggested by RFC 3986"""
1663     url_parsed = compat_urllib_parse_urlparse(url)
1664     return url_parsed._replace(
1665         path=escape_rfc3986(url_parsed.path),
1666         params=escape_rfc3986(url_parsed.params),
1667         query=escape_rfc3986(url_parsed.query),
1668         fragment=escape_rfc3986(url_parsed.fragment)
1669     ).geturl()
1670
1671 try:
1672     struct.pack('!I', 0)
1673 except TypeError:
1674     # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1675     def struct_pack(spec, *args):
1676         if isinstance(spec, compat_str):
1677             spec = spec.encode('ascii')
1678         return struct.pack(spec, *args)
1679
1680     def struct_unpack(spec, *args):
1681         if isinstance(spec, compat_str):
1682             spec = spec.encode('ascii')
1683         return struct.unpack(spec, *args)
1684 else:
1685     struct_pack = struct.pack
1686     struct_unpack = struct.unpack
1687
1688
1689 def read_batch_urls(batch_fd):
1690     def fixup(url):
1691         if not isinstance(url, compat_str):
1692             url = url.decode('utf-8', 'replace')
1693         BOM_UTF8 = '\xef\xbb\xbf'
1694         if url.startswith(BOM_UTF8):
1695             url = url[len(BOM_UTF8):]
1696         url = url.strip()
1697         if url.startswith(('#', ';', ']')):
1698             return False
1699         return url
1700
1701     with contextlib.closing(batch_fd) as fd:
1702         return [url for url in map(fixup, fd) if url]
1703
1704
1705 def urlencode_postdata(*args, **kargs):
1706     return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1707
1708
1709 def encode_dict(d, encoding='utf-8'):
1710     def encode(v):
1711         return v.encode(encoding) if isinstance(v, compat_basestring) else v
1712     return dict((encode(k), encode(v)) for k, v in d.items())
1713
1714
1715 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
1716     return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
1717
1718
1719 US_RATINGS = {
1720     'G': 0,
1721     'PG': 10,
1722     'PG-13': 13,
1723     'R': 16,
1724     'NC': 18,
1725 }
1726
1727
1728 def parse_age_limit(s):
1729     if s is None:
1730         return None
1731     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1732     return int(m.group('age')) if m else US_RATINGS.get(s, None)
1733
1734
1735 def strip_jsonp(code):
1736     return re.sub(
1737         r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1738
1739
1740 def js_to_json(code):
1741     def fix_kv(m):
1742         v = m.group(0)
1743         if v in ('true', 'false', 'null'):
1744             return v
1745         if v.startswith('"'):
1746             v = re.sub(r"\\'", "'", v[1:-1])
1747         elif v.startswith("'"):
1748             v = v[1:-1]
1749             v = re.sub(r"\\\\|\\'|\"", lambda m: {
1750                 '\\\\': '\\\\',
1751                 "\\'": "'",
1752                 '"': '\\"',
1753             }[m.group(0)], v)
1754         return '"%s"' % v
1755
1756     res = re.sub(r'''(?x)
1757         "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1758         '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1759         [a-zA-Z_][.a-zA-Z_0-9]*
1760         ''', fix_kv, code)
1761     res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1762     return res
1763
1764
1765 def qualities(quality_ids):
1766     """ Get a numeric quality value out of a list of possible values """
1767     def q(qid):
1768         try:
1769             return quality_ids.index(qid)
1770         except ValueError:
1771             return -1
1772     return q
1773
1774
1775 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1776
1777
1778 def limit_length(s, length):
1779     """ Add ellipses to overly long strings """
1780     if s is None:
1781         return None
1782     ELLIPSES = '...'
1783     if len(s) > length:
1784         return s[:length - len(ELLIPSES)] + ELLIPSES
1785     return s
1786
1787
1788 def version_tuple(v):
1789     return tuple(int(e) for e in re.split(r'[-.]', v))
1790
1791
1792 def is_outdated_version(version, limit, assume_new=True):
1793     if not version:
1794         return not assume_new
1795     try:
1796         return version_tuple(version) < version_tuple(limit)
1797     except ValueError:
1798         return not assume_new
1799
1800
1801 def ytdl_is_updateable():
1802     """ Returns if youtube-dl can be updated with -U """
1803     from zipimport import zipimporter
1804
1805     return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1806
1807
1808 def args_to_str(args):
1809     # Get a short string representation for a subprocess command
1810     return ' '.join(shlex_quote(a) for a in args)
1811
1812
1813 def error_to_compat_str(err):
1814     err_str = str(err)
1815     # On python 2 error byte string must be decoded with proper
1816     # encoding rather than ascii
1817     if sys.version_info[0] < 3:
1818         err_str = err_str.decode(preferredencoding())
1819     return err_str
1820
1821
1822 def mimetype2ext(mt):
1823     _, _, res = mt.rpartition('/')
1824
1825     return {
1826         'x-ms-wmv': 'wmv',
1827         'x-mp4-fragmented': 'mp4',
1828         'ttml+xml': 'ttml',
1829     }.get(res, res)
1830
1831
1832 def urlhandle_detect_ext(url_handle):
1833     try:
1834         url_handle.headers
1835         getheader = lambda h: url_handle.headers[h]
1836     except AttributeError:  # Python < 3
1837         getheader = url_handle.info().getheader
1838
1839     cd = getheader('Content-Disposition')
1840     if cd:
1841         m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1842         if m:
1843             e = determine_ext(m.group('filename'), default_ext=None)
1844             if e:
1845                 return e
1846
1847     return mimetype2ext(getheader('Content-Type'))
1848
1849
1850 def encode_data_uri(data, mime_type):
1851     return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1852
1853
1854 def age_restricted(content_limit, age_limit):
1855     """ Returns True iff the content should be blocked """
1856
1857     if age_limit is None:  # No limit set
1858         return False
1859     if content_limit is None:
1860         return False  # Content available for everyone
1861     return age_limit < content_limit
1862
1863
1864 def is_html(first_bytes):
1865     """ Detect whether a file contains HTML by examining its first bytes. """
1866
1867     BOMS = [
1868         (b'\xef\xbb\xbf', 'utf-8'),
1869         (b'\x00\x00\xfe\xff', 'utf-32-be'),
1870         (b'\xff\xfe\x00\x00', 'utf-32-le'),
1871         (b'\xff\xfe', 'utf-16-le'),
1872         (b'\xfe\xff', 'utf-16-be'),
1873     ]
1874     for bom, enc in BOMS:
1875         if first_bytes.startswith(bom):
1876             s = first_bytes[len(bom):].decode(enc, 'replace')
1877             break
1878     else:
1879         s = first_bytes.decode('utf-8', 'replace')
1880
1881     return re.match(r'^\s*<', s)
1882
1883
1884 def determine_protocol(info_dict):
1885     protocol = info_dict.get('protocol')
1886     if protocol is not None:
1887         return protocol
1888
1889     url = info_dict['url']
1890     if url.startswith('rtmp'):
1891         return 'rtmp'
1892     elif url.startswith('mms'):
1893         return 'mms'
1894     elif url.startswith('rtsp'):
1895         return 'rtsp'
1896
1897     ext = determine_ext(url)
1898     if ext == 'm3u8':
1899         return 'm3u8'
1900     elif ext == 'f4m':
1901         return 'f4m'
1902
1903     return compat_urllib_parse_urlparse(url).scheme
1904
1905
1906 def render_table(header_row, data):
1907     """ Render a list of rows, each as a list of values """
1908     table = [header_row] + data
1909     max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1910     format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1911     return '\n'.join(format_str % tuple(row) for row in table)
1912
1913
1914 def _match_one(filter_part, dct):
1915     COMPARISON_OPERATORS = {
1916         '<': operator.lt,
1917         '<=': operator.le,
1918         '>': operator.gt,
1919         '>=': operator.ge,
1920         '=': operator.eq,
1921         '!=': operator.ne,
1922     }
1923     operator_rex = re.compile(r'''(?x)\s*
1924         (?P<key>[a-z_]+)
1925         \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1926         (?:
1927             (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1928             (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1929         )
1930         \s*$
1931         ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1932     m = operator_rex.search(filter_part)
1933     if m:
1934         op = COMPARISON_OPERATORS[m.group('op')]
1935         if m.group('strval') is not None:
1936             if m.group('op') not in ('=', '!='):
1937                 raise ValueError(
1938                     'Operator %s does not support string values!' % m.group('op'))
1939             comparison_value = m.group('strval')
1940         else:
1941             try:
1942                 comparison_value = int(m.group('intval'))
1943             except ValueError:
1944                 comparison_value = parse_filesize(m.group('intval'))
1945                 if comparison_value is None:
1946                     comparison_value = parse_filesize(m.group('intval') + 'B')
1947                 if comparison_value is None:
1948                     raise ValueError(
1949                         'Invalid integer value %r in filter part %r' % (
1950                             m.group('intval'), filter_part))
1951         actual_value = dct.get(m.group('key'))
1952         if actual_value is None:
1953             return m.group('none_inclusive')
1954         return op(actual_value, comparison_value)
1955
1956     UNARY_OPERATORS = {
1957         '': lambda v: v is not None,
1958         '!': lambda v: v is None,
1959     }
1960     operator_rex = re.compile(r'''(?x)\s*
1961         (?P<op>%s)\s*(?P<key>[a-z_]+)
1962         \s*$
1963         ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1964     m = operator_rex.search(filter_part)
1965     if m:
1966         op = UNARY_OPERATORS[m.group('op')]
1967         actual_value = dct.get(m.group('key'))
1968         return op(actual_value)
1969
1970     raise ValueError('Invalid filter part %r' % filter_part)
1971
1972
1973 def match_str(filter_str, dct):
1974     """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1975
1976     return all(
1977         _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1978
1979
1980 def match_filter_func(filter_str):
1981     def _match_func(info_dict):
1982         if match_str(filter_str, info_dict):
1983             return None
1984         else:
1985             video_title = info_dict.get('title', info_dict.get('id', 'video'))
1986             return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1987     return _match_func
1988
1989
1990 def parse_dfxp_time_expr(time_expr):
1991     if not time_expr:
1992         return
1993
1994     mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1995     if mobj:
1996         return float(mobj.group('time_offset'))
1997
1998     mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
1999     if mobj:
2000         return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2001
2002
2003 def srt_subtitles_timecode(seconds):
2004     return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2005
2006
2007 def dfxp2srt(dfxp_data):
2008     _x = functools.partial(xpath_with_ns, ns_map={
2009         'ttml': 'http://www.w3.org/ns/ttml',
2010         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2011     })
2012
2013     def parse_node(node):
2014         str_or_empty = functools.partial(str_or_none, default='')
2015
2016         out = str_or_empty(node.text)
2017
2018         for child in node:
2019             if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2020                 out += '\n' + str_or_empty(child.tail)
2021             elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
2022                 out += str_or_empty(parse_node(child))
2023             else:
2024                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
2025
2026         return out
2027
2028     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
2029     out = []
2030     paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
2031
2032     if not paras:
2033         raise ValueError('Invalid dfxp/TTML subtitle')
2034
2035     for para, index in zip(paras, itertools.count(1)):
2036         begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2037         end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2038         dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2039         if begin_time is None:
2040             continue
2041         if not end_time:
2042             if not dur:
2043                 continue
2044             end_time = begin_time + dur
2045         out.append('%d\n%s --> %s\n%s\n\n' % (
2046             index,
2047             srt_subtitles_timecode(begin_time),
2048             srt_subtitles_timecode(end_time),
2049             parse_node(para)))
2050
2051     return ''.join(out)
2052
2053
2054 def cli_option(params, command_option, param):
2055     param = params.get(param)
2056     return [command_option, param] if param is not None else []
2057
2058
2059 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2060     param = params.get(param)
2061     assert isinstance(param, bool)
2062     if separator:
2063         return [command_option + separator + (true_value if param else false_value)]
2064     return [command_option, true_value if param else false_value]
2065
2066
2067 def cli_valueless_option(params, command_option, param, expected_value=True):
2068     param = params.get(param)
2069     return [command_option] if param == expected_value else []
2070
2071
2072 def cli_configuration_args(params, param, default=[]):
2073     ex_args = params.get(param)
2074     if ex_args is None:
2075         return default
2076     assert isinstance(ex_args, list)
2077     return ex_args
2078
2079
2080 class ISO639Utils(object):
2081     # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2082     _lang_map = {
2083         'aa': 'aar',
2084         'ab': 'abk',
2085         'ae': 'ave',
2086         'af': 'afr',
2087         'ak': 'aka',
2088         'am': 'amh',
2089         'an': 'arg',
2090         'ar': 'ara',
2091         'as': 'asm',
2092         'av': 'ava',
2093         'ay': 'aym',
2094         'az': 'aze',
2095         'ba': 'bak',
2096         'be': 'bel',
2097         'bg': 'bul',
2098         'bh': 'bih',
2099         'bi': 'bis',
2100         'bm': 'bam',
2101         'bn': 'ben',
2102         'bo': 'bod',
2103         'br': 'bre',
2104         'bs': 'bos',
2105         'ca': 'cat',
2106         'ce': 'che',
2107         'ch': 'cha',
2108         'co': 'cos',
2109         'cr': 'cre',
2110         'cs': 'ces',
2111         'cu': 'chu',
2112         'cv': 'chv',
2113         'cy': 'cym',
2114         'da': 'dan',
2115         'de': 'deu',
2116         'dv': 'div',
2117         'dz': 'dzo',
2118         'ee': 'ewe',
2119         'el': 'ell',
2120         'en': 'eng',
2121         'eo': 'epo',
2122         'es': 'spa',
2123         'et': 'est',
2124         'eu': 'eus',
2125         'fa': 'fas',
2126         'ff': 'ful',
2127         'fi': 'fin',
2128         'fj': 'fij',
2129         'fo': 'fao',
2130         'fr': 'fra',
2131         'fy': 'fry',
2132         'ga': 'gle',
2133         'gd': 'gla',
2134         'gl': 'glg',
2135         'gn': 'grn',
2136         'gu': 'guj',
2137         'gv': 'glv',
2138         'ha': 'hau',
2139         'he': 'heb',
2140         'hi': 'hin',
2141         'ho': 'hmo',
2142         'hr': 'hrv',
2143         'ht': 'hat',
2144         'hu': 'hun',
2145         'hy': 'hye',
2146         'hz': 'her',
2147         'ia': 'ina',
2148         'id': 'ind',
2149         'ie': 'ile',
2150         'ig': 'ibo',
2151         'ii': 'iii',
2152         'ik': 'ipk',
2153         'io': 'ido',
2154         'is': 'isl',
2155         'it': 'ita',
2156         'iu': 'iku',
2157         'ja': 'jpn',
2158         'jv': 'jav',
2159         'ka': 'kat',
2160         'kg': 'kon',
2161         'ki': 'kik',
2162         'kj': 'kua',
2163         'kk': 'kaz',
2164         'kl': 'kal',
2165         'km': 'khm',
2166         'kn': 'kan',
2167         'ko': 'kor',
2168         'kr': 'kau',
2169         'ks': 'kas',
2170         'ku': 'kur',
2171         'kv': 'kom',
2172         'kw': 'cor',
2173         'ky': 'kir',
2174         'la': 'lat',
2175         'lb': 'ltz',
2176         'lg': 'lug',
2177         'li': 'lim',
2178         'ln': 'lin',
2179         'lo': 'lao',
2180         'lt': 'lit',
2181         'lu': 'lub',
2182         'lv': 'lav',
2183         'mg': 'mlg',
2184         'mh': 'mah',
2185         'mi': 'mri',
2186         'mk': 'mkd',
2187         'ml': 'mal',
2188         'mn': 'mon',
2189         'mr': 'mar',
2190         'ms': 'msa',
2191         'mt': 'mlt',
2192         'my': 'mya',
2193         'na': 'nau',
2194         'nb': 'nob',
2195         'nd': 'nde',
2196         'ne': 'nep',
2197         'ng': 'ndo',
2198         'nl': 'nld',
2199         'nn': 'nno',
2200         'no': 'nor',
2201         'nr': 'nbl',
2202         'nv': 'nav',
2203         'ny': 'nya',
2204         'oc': 'oci',
2205         'oj': 'oji',
2206         'om': 'orm',
2207         'or': 'ori',
2208         'os': 'oss',
2209         'pa': 'pan',
2210         'pi': 'pli',
2211         'pl': 'pol',
2212         'ps': 'pus',
2213         'pt': 'por',
2214         'qu': 'que',
2215         'rm': 'roh',
2216         'rn': 'run',
2217         'ro': 'ron',
2218         'ru': 'rus',
2219         'rw': 'kin',
2220         'sa': 'san',
2221         'sc': 'srd',
2222         'sd': 'snd',
2223         'se': 'sme',
2224         'sg': 'sag',
2225         'si': 'sin',
2226         'sk': 'slk',
2227         'sl': 'slv',
2228         'sm': 'smo',
2229         'sn': 'sna',
2230         'so': 'som',
2231         'sq': 'sqi',
2232         'sr': 'srp',
2233         'ss': 'ssw',
2234         'st': 'sot',
2235         'su': 'sun',
2236         'sv': 'swe',
2237         'sw': 'swa',
2238         'ta': 'tam',
2239         'te': 'tel',
2240         'tg': 'tgk',
2241         'th': 'tha',
2242         'ti': 'tir',
2243         'tk': 'tuk',
2244         'tl': 'tgl',
2245         'tn': 'tsn',
2246         'to': 'ton',
2247         'tr': 'tur',
2248         'ts': 'tso',
2249         'tt': 'tat',
2250         'tw': 'twi',
2251         'ty': 'tah',
2252         'ug': 'uig',
2253         'uk': 'ukr',
2254         'ur': 'urd',
2255         'uz': 'uzb',
2256         've': 'ven',
2257         'vi': 'vie',
2258         'vo': 'vol',
2259         'wa': 'wln',
2260         'wo': 'wol',
2261         'xh': 'xho',
2262         'yi': 'yid',
2263         'yo': 'yor',
2264         'za': 'zha',
2265         'zh': 'zho',
2266         'zu': 'zul',
2267     }
2268
2269     @classmethod
2270     def short2long(cls, code):
2271         """Convert language code from ISO 639-1 to ISO 639-2/T"""
2272         return cls._lang_map.get(code[:2])
2273
2274     @classmethod
2275     def long2short(cls, code):
2276         """Convert language code from ISO 639-2/T to ISO 639-1"""
2277         for short_name, long_name in cls._lang_map.items():
2278             if long_name == code:
2279                 return short_name
2280
2281
2282 class ISO3166Utils(object):
2283     # From http://data.okfn.org/data/core/country-list
2284     _country_map = {
2285         'AF': 'Afghanistan',
2286         'AX': 'Åland Islands',
2287         'AL': 'Albania',
2288         'DZ': 'Algeria',
2289         'AS': 'American Samoa',
2290         'AD': 'Andorra',
2291         'AO': 'Angola',
2292         'AI': 'Anguilla',
2293         'AQ': 'Antarctica',
2294         'AG': 'Antigua and Barbuda',
2295         'AR': 'Argentina',
2296         'AM': 'Armenia',
2297         'AW': 'Aruba',
2298         'AU': 'Australia',
2299         'AT': 'Austria',
2300         'AZ': 'Azerbaijan',
2301         'BS': 'Bahamas',
2302         'BH': 'Bahrain',
2303         'BD': 'Bangladesh',
2304         'BB': 'Barbados',
2305         'BY': 'Belarus',
2306         'BE': 'Belgium',
2307         'BZ': 'Belize',
2308         'BJ': 'Benin',
2309         'BM': 'Bermuda',
2310         'BT': 'Bhutan',
2311         'BO': 'Bolivia, Plurinational State of',
2312         'BQ': 'Bonaire, Sint Eustatius and Saba',
2313         'BA': 'Bosnia and Herzegovina',
2314         'BW': 'Botswana',
2315         'BV': 'Bouvet Island',
2316         'BR': 'Brazil',
2317         'IO': 'British Indian Ocean Territory',
2318         'BN': 'Brunei Darussalam',
2319         'BG': 'Bulgaria',
2320         'BF': 'Burkina Faso',
2321         'BI': 'Burundi',
2322         'KH': 'Cambodia',
2323         'CM': 'Cameroon',
2324         'CA': 'Canada',
2325         'CV': 'Cape Verde',
2326         'KY': 'Cayman Islands',
2327         'CF': 'Central African Republic',
2328         'TD': 'Chad',
2329         'CL': 'Chile',
2330         'CN': 'China',
2331         'CX': 'Christmas Island',
2332         'CC': 'Cocos (Keeling) Islands',
2333         'CO': 'Colombia',
2334         'KM': 'Comoros',
2335         'CG': 'Congo',
2336         'CD': 'Congo, the Democratic Republic of the',
2337         'CK': 'Cook Islands',
2338         'CR': 'Costa Rica',
2339         'CI': 'Côte d\'Ivoire',
2340         'HR': 'Croatia',
2341         'CU': 'Cuba',
2342         'CW': 'Curaçao',
2343         'CY': 'Cyprus',
2344         'CZ': 'Czech Republic',
2345         'DK': 'Denmark',
2346         'DJ': 'Djibouti',
2347         'DM': 'Dominica',
2348         'DO': 'Dominican Republic',
2349         'EC': 'Ecuador',
2350         'EG': 'Egypt',
2351         'SV': 'El Salvador',
2352         'GQ': 'Equatorial Guinea',
2353         'ER': 'Eritrea',
2354         'EE': 'Estonia',
2355         'ET': 'Ethiopia',
2356         'FK': 'Falkland Islands (Malvinas)',
2357         'FO': 'Faroe Islands',
2358         'FJ': 'Fiji',
2359         'FI': 'Finland',
2360         'FR': 'France',
2361         'GF': 'French Guiana',
2362         'PF': 'French Polynesia',
2363         'TF': 'French Southern Territories',
2364         'GA': 'Gabon',
2365         'GM': 'Gambia',
2366         'GE': 'Georgia',
2367         'DE': 'Germany',
2368         'GH': 'Ghana',
2369         'GI': 'Gibraltar',
2370         'GR': 'Greece',
2371         'GL': 'Greenland',
2372         'GD': 'Grenada',
2373         'GP': 'Guadeloupe',
2374         'GU': 'Guam',
2375         'GT': 'Guatemala',
2376         'GG': 'Guernsey',
2377         'GN': 'Guinea',
2378         'GW': 'Guinea-Bissau',
2379         'GY': 'Guyana',
2380         'HT': 'Haiti',
2381         'HM': 'Heard Island and McDonald Islands',
2382         'VA': 'Holy See (Vatican City State)',
2383         'HN': 'Honduras',
2384         'HK': 'Hong Kong',
2385         'HU': 'Hungary',
2386         'IS': 'Iceland',
2387         'IN': 'India',
2388         'ID': 'Indonesia',
2389         'IR': 'Iran, Islamic Republic of',
2390         'IQ': 'Iraq',
2391         'IE': 'Ireland',
2392         'IM': 'Isle of Man',
2393         'IL': 'Israel',
2394         'IT': 'Italy',
2395         'JM': 'Jamaica',
2396         'JP': 'Japan',
2397         'JE': 'Jersey',
2398         'JO': 'Jordan',
2399         'KZ': 'Kazakhstan',
2400         'KE': 'Kenya',
2401         'KI': 'Kiribati',
2402         'KP': 'Korea, Democratic People\'s Republic of',
2403         'KR': 'Korea, Republic of',
2404         'KW': 'Kuwait',
2405         'KG': 'Kyrgyzstan',
2406         'LA': 'Lao People\'s Democratic Republic',
2407         'LV': 'Latvia',
2408         'LB': 'Lebanon',
2409         'LS': 'Lesotho',
2410         'LR': 'Liberia',
2411         'LY': 'Libya',
2412         'LI': 'Liechtenstein',
2413         'LT': 'Lithuania',
2414         'LU': 'Luxembourg',
2415         'MO': 'Macao',
2416         'MK': 'Macedonia, the Former Yugoslav Republic of',
2417         'MG': 'Madagascar',
2418         'MW': 'Malawi',
2419         'MY': 'Malaysia',
2420         'MV': 'Maldives',
2421         'ML': 'Mali',
2422         'MT': 'Malta',
2423         'MH': 'Marshall Islands',
2424         'MQ': 'Martinique',
2425         'MR': 'Mauritania',
2426         'MU': 'Mauritius',
2427         'YT': 'Mayotte',
2428         'MX': 'Mexico',
2429         'FM': 'Micronesia, Federated States of',
2430         'MD': 'Moldova, Republic of',
2431         'MC': 'Monaco',
2432         'MN': 'Mongolia',
2433         'ME': 'Montenegro',
2434         'MS': 'Montserrat',
2435         'MA': 'Morocco',
2436         'MZ': 'Mozambique',
2437         'MM': 'Myanmar',
2438         'NA': 'Namibia',
2439         'NR': 'Nauru',
2440         'NP': 'Nepal',
2441         'NL': 'Netherlands',
2442         'NC': 'New Caledonia',
2443         'NZ': 'New Zealand',
2444         'NI': 'Nicaragua',
2445         'NE': 'Niger',
2446         'NG': 'Nigeria',
2447         'NU': 'Niue',
2448         'NF': 'Norfolk Island',
2449         'MP': 'Northern Mariana Islands',
2450         'NO': 'Norway',
2451         'OM': 'Oman',
2452         'PK': 'Pakistan',
2453         'PW': 'Palau',
2454         'PS': 'Palestine, State of',
2455         'PA': 'Panama',
2456         'PG': 'Papua New Guinea',
2457         'PY': 'Paraguay',
2458         'PE': 'Peru',
2459         'PH': 'Philippines',
2460         'PN': 'Pitcairn',
2461         'PL': 'Poland',
2462         'PT': 'Portugal',
2463         'PR': 'Puerto Rico',
2464         'QA': 'Qatar',
2465         'RE': 'Réunion',
2466         'RO': 'Romania',
2467         'RU': 'Russian Federation',
2468         'RW': 'Rwanda',
2469         'BL': 'Saint Barthélemy',
2470         'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2471         'KN': 'Saint Kitts and Nevis',
2472         'LC': 'Saint Lucia',
2473         'MF': 'Saint Martin (French part)',
2474         'PM': 'Saint Pierre and Miquelon',
2475         'VC': 'Saint Vincent and the Grenadines',
2476         'WS': 'Samoa',
2477         'SM': 'San Marino',
2478         'ST': 'Sao Tome and Principe',
2479         'SA': 'Saudi Arabia',
2480         'SN': 'Senegal',
2481         'RS': 'Serbia',
2482         'SC': 'Seychelles',
2483         'SL': 'Sierra Leone',
2484         'SG': 'Singapore',
2485         'SX': 'Sint Maarten (Dutch part)',
2486         'SK': 'Slovakia',
2487         'SI': 'Slovenia',
2488         'SB': 'Solomon Islands',
2489         'SO': 'Somalia',
2490         'ZA': 'South Africa',
2491         'GS': 'South Georgia and the South Sandwich Islands',
2492         'SS': 'South Sudan',
2493         'ES': 'Spain',
2494         'LK': 'Sri Lanka',
2495         'SD': 'Sudan',
2496         'SR': 'Suriname',
2497         'SJ': 'Svalbard and Jan Mayen',
2498         'SZ': 'Swaziland',
2499         'SE': 'Sweden',
2500         'CH': 'Switzerland',
2501         'SY': 'Syrian Arab Republic',
2502         'TW': 'Taiwan, Province of China',
2503         'TJ': 'Tajikistan',
2504         'TZ': 'Tanzania, United Republic of',
2505         'TH': 'Thailand',
2506         'TL': 'Timor-Leste',
2507         'TG': 'Togo',
2508         'TK': 'Tokelau',
2509         'TO': 'Tonga',
2510         'TT': 'Trinidad and Tobago',
2511         'TN': 'Tunisia',
2512         'TR': 'Turkey',
2513         'TM': 'Turkmenistan',
2514         'TC': 'Turks and Caicos Islands',
2515         'TV': 'Tuvalu',
2516         'UG': 'Uganda',
2517         'UA': 'Ukraine',
2518         'AE': 'United Arab Emirates',
2519         'GB': 'United Kingdom',
2520         'US': 'United States',
2521         'UM': 'United States Minor Outlying Islands',
2522         'UY': 'Uruguay',
2523         'UZ': 'Uzbekistan',
2524         'VU': 'Vanuatu',
2525         'VE': 'Venezuela, Bolivarian Republic of',
2526         'VN': 'Viet Nam',
2527         'VG': 'Virgin Islands, British',
2528         'VI': 'Virgin Islands, U.S.',
2529         'WF': 'Wallis and Futuna',
2530         'EH': 'Western Sahara',
2531         'YE': 'Yemen',
2532         'ZM': 'Zambia',
2533         'ZW': 'Zimbabwe',
2534     }
2535
2536     @classmethod
2537     def short2full(cls, code):
2538         """Convert an ISO 3166-2 country code to the corresponding full name"""
2539         return cls._country_map.get(code.upper())
2540
2541
2542 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2543     def __init__(self, proxies=None):
2544         # Set default handlers
2545         for type in ('http', 'https'):
2546             setattr(self, '%s_open' % type,
2547                     lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2548                         meth(r, proxy, type))
2549         return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2550
2551     def proxy_open(self, req, proxy, type):
2552         req_proxy = req.headers.get('Ytdl-request-proxy')
2553         if req_proxy is not None:
2554             proxy = req_proxy
2555             del req.headers['Ytdl-request-proxy']
2556
2557         if proxy == '__noproxy__':
2558             return None  # No Proxy
2559         return compat_urllib_request.ProxyHandler.proxy_open(
2560             self, req, proxy, type)