youtube_dl/utils.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import unicode_literals
   5
   6 import calendar
   7 import codecs
   8 import contextlib
   9 import ctypes
  10 import datetime
  11 import email.utils
  12 import errno
  13 import functools
  14 import gzip
  15 import itertools
  16 import io
  17 import json
  18 import locale
  19 import math
  20 import operator
  21 import os
  22 import pipes
  23 import platform
  24 import re
  25 import ssl
  26 import socket
  27 import struct
  28 import subprocess
  29 import sys
  30 import tempfile
  31 import traceback
  32 import xml.etree.ElementTree
  33 import zlib
  34
  35 from .compat import (
  36     compat_basestring,
  37     compat_chr,
  38     compat_html_entities,
  39     compat_http_client,
  40     compat_kwargs,
  41     compat_parse_qs,
  42     compat_socket_create_connection,
  43     compat_str,
  44     compat_urllib_error,
  45     compat_urllib_parse,
  46     compat_urllib_parse_urlparse,
  47     compat_urllib_request,
  48     compat_urlparse,
  49     shlex_quote,
  50 )
  51
  52
  53 # This is not clearly defined otherwise
  54 compiled_regex_type = type(re.compile(''))
  55
  56 std_headers = {
  57     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
  58     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  59     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  60     'Accept-Encoding': 'gzip, deflate',
  61     'Accept-Language': 'en-us,en;q=0.5',
  62 }
  63
  64
  65 NO_DEFAULT = object()
  66
  67 ENGLISH_MONTH_NAMES = [
  68     'January', 'February', 'March', 'April', 'May', 'June',
  69     'July', 'August', 'September', 'October', 'November', 'December']
  70
  71
  72 def preferredencoding():
  73     """Get preferred encoding.
  74
  75     Returns the best encoding scheme for the system, based on
  76     locale.getpreferredencoding() and some further tweaks.
  77     """
  78     try:
  79         pref = locale.getpreferredencoding()
  80         'TEST'.encode(pref)
  81     except Exception:
  82         pref = 'UTF-8'
  83
  84     return pref
  85
  86
  87 def write_json_file(obj, fn):
  88     """ Encode obj as JSON and write it to fn, atomically if possible """
  89
  90     fn = encodeFilename(fn)
  91     if sys.version_info < (3, 0) and sys.platform != 'win32':
  92         encoding = get_filesystem_encoding()
  93         # os.path.basename returns a bytes object, but NamedTemporaryFile
  94         # will fail if the filename contains non ascii characters unless we
  95         # use a unicode object
  96         path_basename = lambda f: os.path.basename(fn).decode(encoding)
  97         # the same for os.path.dirname
  98         path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
  99     else:
 100         path_basename = os.path.basename
 101         path_dirname = os.path.dirname
 102
 103     args = {
 104         'suffix': '.tmp',
 105         'prefix': path_basename(fn) + '.',
 106         'dir': path_dirname(fn),
 107         'delete': False,
 108     }
 109
 110     # In Python 2.x, json.dump expects a bytestream.
 111     # In Python 3.x, it writes to a character stream
 112     if sys.version_info < (3, 0):
 113         args['mode'] = 'wb'
 114     else:
 115         args.update({
 116             'mode': 'w',
 117             'encoding': 'utf-8',
 118         })
 119
 120     tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
 121
 122     try:
 123         with tf:
 124             json.dump(obj, tf)
 125         if sys.platform == 'win32':
 126             # Need to remove existing file on Windows, else os.rename raises
 127             # WindowsError or FileExistsError.
 128             try:
 129                 os.unlink(fn)
 130             except OSError:
 131                 pass
 132         os.rename(tf.name, fn)
 133     except Exception:
 134         try:
 135             os.remove(tf.name)
 136         except OSError:
 137             pass
 138         raise
 139
 140
 141 if sys.version_info >= (2, 7):
 142     def find_xpath_attr(node, xpath, key, val=None):
 143         """ Find the xpath xpath[@key=val] """
 144         assert re.match(r'^[a-zA-Z_-]+$', key)
 145         if val:
 146             assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
 147         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
 148         return node.find(expr)
 149 else:
 150     def find_xpath_attr(node, xpath, key, val=None):
 151         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
 152         # .//node does not match if a node is a direct child of . !
 153         if isinstance(xpath, compat_str):
 154             xpath = xpath.encode('ascii')
 155
 156         for f in node.findall(xpath):
 157             if key not in f.attrib:
 158                 continue
 159             if val is None or f.attrib.get(key) == val:
 160                 return f
 161         return None
 162
 163 # On python2.6 the xml.etree.ElementTree.Element methods don't support
 164 # the namespace parameter
 165
 166
 167 def xpath_with_ns(path, ns_map):
 168     components = [c.split(':') for c in path.split('/')]
 169     replaced = []
 170     for c in components:
 171         if len(c) == 1:
 172             replaced.append(c[0])
 173         else:
 174             ns, tag = c
 175             replaced.append('{%s}%s' % (ns_map[ns], tag))
 176     return '/'.join(replaced)
 177
 178
 179 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
 180     if sys.version_info < (2, 7):  # Crazy 2.6
 181         xpath = xpath.encode('ascii')
 182
 183     n = node.find(xpath)
 184     if n is None:
 185         if default is not NO_DEFAULT:
 186             return default
 187         elif fatal:
 188             name = xpath if name is None else name
 189             raise ExtractorError('Could not find XML element %s' % name)
 190         else:
 191             return None
 192     return n
 193
 194
 195 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
 196     n = xpath_element(node, xpath, name, fatal=fatal, default=default)
 197     if n is None or n == default:
 198         return n
 199     if n.text is None:
 200         if default is not NO_DEFAULT:
 201             return default
 202         elif fatal:
 203             name = xpath if name is None else name
 204             raise ExtractorError('Could not find XML element\'s text %s' % name)
 205         else:
 206             return None
 207     return n.text
 208
 209
 210 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
 211     n = find_xpath_attr(node, xpath, key)
 212     if n is None:
 213         if default is not NO_DEFAULT:
 214             return default
 215         elif fatal:
 216             name = '%s[@%s]' % (xpath, key) if name is None else name
 217             raise ExtractorError('Could not find XML attribute %s' % name)
 218         else:
 219             return None
 220     return n.attrib[key]
 221
 222
 223 def get_element_by_id(id, html):
 224     """Return the content of the tag with the specified ID in the passed HTML document"""
 225     return get_element_by_attribute("id", id, html)
 226
 227
 228 def get_element_by_attribute(attribute, value, html):
 229     """Return the content of the tag with the specified attribute in the passed HTML document"""
 230
 231     m = re.search(r'''(?xs)
 232         <([a-zA-Z0-9:._-]+)
 233          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
 234          \s+%s=['"]?%s['"]?
 235          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
 236         \s*>
 237         (?P<content>.*?)
 238         </\1>
 239     ''' % (re.escape(attribute), re.escape(value)), html)
 240
 241     if not m:
 242         return None
 243     res = m.group('content')
 244
 245     if res.startswith('"') or res.startswith("'"):
 246         res = res[1:-1]
 247
 248     return unescapeHTML(res)
 249
 250
 251 def clean_html(html):
 252     """Clean an HTML snippet into a readable string"""
 253
 254     if html is None:  # Convenience for sanitizing descriptions etc.
 255         return html
 256
 257     # Newline vs <br />
 258     html = html.replace('\n', ' ')
 259     html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
 260     html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
 261     # Strip html tags
 262     html = re.sub('<.*?>', '', html)
 263     # Replace html entities
 264     html = unescapeHTML(html)
 265     return html.strip()
 266
 267
 268 def sanitize_open(filename, open_mode):
 269     """Try to open the given filename, and slightly tweak it if this fails.
 270
 271     Attempts to open the given filename. If this fails, it tries to change
 272     the filename slightly, step by step, until it's either able to open it
 273     or it fails and raises a final exception, like the standard open()
 274     function.
 275
 276     It returns the tuple (stream, definitive_file_name).
 277     """
 278     try:
 279         if filename == '-':
 280             if sys.platform == 'win32':
 281                 import msvcrt
 282                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 283             return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
 284         stream = open(encodeFilename(filename), open_mode)
 285         return (stream, filename)
 286     except (IOError, OSError) as err:
 287         if err.errno in (errno.EACCES,):
 288             raise
 289
 290         # In case of error, try to remove win32 forbidden chars
 291         alt_filename = sanitize_path(filename)
 292         if alt_filename == filename:
 293             raise
 294         else:
 295             # An exception here should be caught in the caller
 296             stream = open(encodeFilename(alt_filename), open_mode)
 297             return (stream, alt_filename)
 298
 299
 300 def timeconvert(timestr):
 301     """Convert RFC 2822 defined time string into system timestamp"""
 302     timestamp = None
 303     timetuple = email.utils.parsedate_tz(timestr)
 304     if timetuple is not None:
 305         timestamp = email.utils.mktime_tz(timetuple)
 306     return timestamp
 307
 308
 309 def sanitize_filename(s, restricted=False, is_id=False):
 310     """Sanitizes a string so it could be used as part of a filename.
 311     If restricted is set, use a stricter subset of allowed characters.
 312     Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
 313     """
 314     def replace_insane(char):
 315         if char == '?' or ord(char) < 32 or ord(char) == 127:
 316             return ''
 317         elif char == '"':
 318             return '' if restricted else '\''
 319         elif char == ':':
 320             return '_-' if restricted else ' -'
 321         elif char in '\\/|*<>':
 322             return '_'
 323         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
 324             return '_'
 325         if restricted and ord(char) > 127:
 326             return '_'
 327         return char
 328
 329     # Handle timestamps
 330     s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
 331     result = ''.join(map(replace_insane, s))
 332     if not is_id:
 333         while '__' in result:
 334             result = result.replace('__', '_')
 335         result = result.strip('_')
 336         # Common case of "Foreign band name - English song title"
 337         if restricted and result.startswith('-_'):
 338             result = result[2:]
 339         if result.startswith('-'):
 340             result = '_' + result[len('-'):]
 341         result = result.lstrip('.')
 342         if not result:
 343             result = '_'
 344     return result
 345
 346
 347 def sanitize_path(s):
 348     """Sanitizes and normalizes path on Windows"""
 349     if sys.platform != 'win32':
 350         return s
 351     drive_or_unc, _ = os.path.splitdrive(s)
 352     if sys.version_info < (2, 7) and not drive_or_unc:
 353         drive_or_unc, _ = os.path.splitunc(s)
 354     norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
 355     if drive_or_unc:
 356         norm_path.pop(0)
 357     sanitized_path = [
 358         path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
 359         for path_part in norm_path]
 360     if drive_or_unc:
 361         sanitized_path.insert(0, drive_or_unc + os.path.sep)
 362     return os.path.join(*sanitized_path)
 363
 364
 365 def orderedSet(iterable):
 366     """ Remove all duplicates from the input iterable """
 367     res = []
 368     for el in iterable:
 369         if el not in res:
 370             res.append(el)
 371     return res
 372
 373
 374 def _htmlentity_transform(entity):
 375     """Transforms an HTML entity to a character."""
 376     # Known non-numeric HTML entity
 377     if entity in compat_html_entities.name2codepoint:
 378         return compat_chr(compat_html_entities.name2codepoint[entity])
 379
 380     mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
 381     if mobj is not None:
 382         numstr = mobj.group(1)
 383         if numstr.startswith('x'):
 384             base = 16
 385             numstr = '0%s' % numstr
 386         else:
 387             base = 10
 388         return compat_chr(int(numstr, base))
 389
 390     # Unknown entity in name, return its literal representation
 391     return ('&%s;' % entity)
 392
 393
 394 def unescapeHTML(s):
 395     if s is None:
 396         return None
 397     assert type(s) == compat_str
 398
 399     return re.sub(
 400         r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
 401
 402
 403 def get_subprocess_encoding():
 404     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
 405         # For subprocess calls, encode with locale encoding
 406         # Refer to http://stackoverflow.com/a/9951851/35070
 407         encoding = preferredencoding()
 408     else:
 409         encoding = sys.getfilesystemencoding()
 410     if encoding is None:
 411         encoding = 'utf-8'
 412     return encoding
 413
 414
 415 def encodeFilename(s, for_subprocess=False):
 416     """
 417     @param s The name of the file
 418     """
 419
 420     assert type(s) == compat_str
 421
 422     # Python 3 has a Unicode API
 423     if sys.version_info >= (3, 0):
 424         return s
 425
 426     # Pass '' directly to use Unicode APIs on Windows 2000 and up
 427     # (Detecting Windows NT 4 is tricky because 'major >= 4' would
 428     # match Windows 9x series as well. Besides, NT 4 is obsolete.)
 429     if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
 430         return s
 431
 432     return s.encode(get_subprocess_encoding(), 'ignore')
 433
 434
 435 def decodeFilename(b, for_subprocess=False):
 436
 437     if sys.version_info >= (3, 0):
 438         return b
 439
 440     if not isinstance(b, bytes):
 441         return b
 442
 443     return b.decode(get_subprocess_encoding(), 'ignore')
 444
 445
 446 def encodeArgument(s):
 447     if not isinstance(s, compat_str):
 448         # Legacy code that uses byte strings
 449         # Uncomment the following line after fixing all post processors
 450         # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
 451         s = s.decode('ascii')
 452     return encodeFilename(s, True)
 453
 454
 455 def decodeArgument(b):
 456     return decodeFilename(b, True)
 457
 458
 459 def decodeOption(optval):
 460     if optval is None:
 461         return optval
 462     if isinstance(optval, bytes):
 463         optval = optval.decode(preferredencoding())
 464
 465     assert isinstance(optval, compat_str)
 466     return optval
 467
 468
 469 def formatSeconds(secs):
 470     if secs > 3600:
 471         return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
 472     elif secs > 60:
 473         return '%d:%02d' % (secs // 60, secs % 60)
 474     else:
 475         return '%d' % secs
 476
 477
 478 def make_HTTPS_handler(params, **kwargs):
 479     opts_no_check_certificate = params.get('nocheckcertificate', False)
 480     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
 481         context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
 482         if opts_no_check_certificate:
 483             context.check_hostname = False
 484             context.verify_mode = ssl.CERT_NONE
 485         try:
 486             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 487         except TypeError:
 488             # Python 2.7.8
 489             # (create_default_context present but HTTPSHandler has no context=)
 490             pass
 491
 492     if sys.version_info < (3, 2):
 493         return YoutubeDLHTTPSHandler(params, **kwargs)
 494     else:  # Python < 3.4
 495         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
 496         context.verify_mode = (ssl.CERT_NONE
 497                                if opts_no_check_certificate
 498                                else ssl.CERT_REQUIRED)
 499         context.set_default_verify_paths()
 500         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 501
 502
 503 def bug_reports_message():
 504     if ytdl_is_updateable():
 505         update_cmd = 'type  youtube-dl -U  to update'
 506     else:
 507         update_cmd = 'see  https://yt-dl.org/update  on how to update'
 508     msg = '; please report this issue on https://yt-dl.org/bug .'
 509     msg += ' Make sure you are using the latest version; %s.' % update_cmd
 510     msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
 511     return msg
 512
 513
 514 class ExtractorError(Exception):
 515     """Error during info extraction."""
 516
 517     def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
 518         """ tb, if given, is the original traceback (so that it can be printed out).
 519         If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
 520         """
 521
 522         if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
 523             expected = True
 524         if video_id is not None:
 525             msg = video_id + ': ' + msg
 526         if cause:
 527             msg += ' (caused by %r)' % cause
 528         if not expected:
 529             msg += bug_reports_message()
 530         super(ExtractorError, self).__init__(msg)
 531
 532         self.traceback = tb
 533         self.exc_info = sys.exc_info()  # preserve original exception
 534         self.cause = cause
 535         self.video_id = video_id
 536
 537     def format_traceback(self):
 538         if self.traceback is None:
 539             return None
 540         return ''.join(traceback.format_tb(self.traceback))
 541
 542
 543 class UnsupportedError(ExtractorError):
 544     def __init__(self, url):
 545         super(UnsupportedError, self).__init__(
 546             'Unsupported URL: %s' % url, expected=True)
 547         self.url = url
 548
 549
 550 class RegexNotFoundError(ExtractorError):
 551     """Error when a regex didn't match"""
 552     pass
 553
 554
 555 class DownloadError(Exception):
 556     """Download Error exception.
 557
 558     This exception may be thrown by FileDownloader objects if they are not
 559     configured to continue on errors. They will contain the appropriate
 560     error message.
 561     """
 562
 563     def __init__(self, msg, exc_info=None):
 564         """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
 565         super(DownloadError, self).__init__(msg)
 566         self.exc_info = exc_info
 567
 568
 569 class SameFileError(Exception):
 570     """Same File exception.
 571
 572     This exception will be thrown by FileDownloader objects if they detect
 573     multiple files would have to be downloaded to the same file on disk.
 574     """
 575     pass
 576
 577
 578 class PostProcessingError(Exception):
 579     """Post Processing exception.
 580
 581     This exception may be raised by PostProcessor's .run() method to
 582     indicate an error in the postprocessing task.
 583     """
 584
 585     def __init__(self, msg):
 586         self.msg = msg
 587
 588
 589 class MaxDownloadsReached(Exception):
 590     """ --max-downloads limit has been reached. """
 591     pass
 592
 593
 594 class UnavailableVideoError(Exception):
 595     """Unavailable Format exception.
 596
 597     This exception will be thrown when a video is requested
 598     in a format that is not available for that video.
 599     """
 600     pass
 601
 602
 603 class ContentTooShortError(Exception):
 604     """Content Too Short exception.
 605
 606     This exception may be raised by FileDownloader objects when a file they
 607     download is too small for what the server announced first, indicating
 608     the connection was probably interrupted.
 609     """
 610
 611     def __init__(self, downloaded, expected):
 612         # Both in bytes
 613         self.downloaded = downloaded
 614         self.expected = expected
 615
 616
 617 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
 618     # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
 619     # expected HTTP responses to meet HTTP/1.0 or later (see also
 620     # https://github.com/rg3/youtube-dl/issues/6727)
 621     if sys.version_info < (3, 0):
 622         kwargs['strict'] = True
 623     hc = http_class(*args, **kwargs)
 624     source_address = ydl_handler._params.get('source_address')
 625     if source_address is not None:
 626         sa = (source_address, 0)
 627         if hasattr(hc, 'source_address'):  # Python 2.7+
 628             hc.source_address = sa
 629         else:  # Python 2.6
 630             def _hc_connect(self, *args, **kwargs):
 631                 sock = compat_socket_create_connection(
 632                     (self.host, self.port), self.timeout, sa)
 633                 if is_https:
 634                     self.sock = ssl.wrap_socket(
 635                         sock, self.key_file, self.cert_file,
 636                         ssl_version=ssl.PROTOCOL_TLSv1)
 637                 else:
 638                     self.sock = sock
 639             hc.connect = functools.partial(_hc_connect, hc)
 640
 641     return hc
 642
 643
 644 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 645     """Handler for HTTP requests and responses.
 646
 647     This class, when installed with an OpenerDirector, automatically adds
 648     the standard headers to every HTTP request and handles gzipped and
 649     deflated responses from web servers. If compression is to be avoided in
 650     a particular request, the original request in the program code only has
 651     to include the HTTP header "Youtubedl-No-Compression", which will be
 652     removed before making the real request.
 653
 654     Part of this code was copied from:
 655
 656     http://techknack.net/python-urllib2-handlers/
 657
 658     Andrew Rowls, the author of that code, agreed to release it to the
 659     public domain.
 660     """
 661
 662     def __init__(self, params, *args, **kwargs):
 663         compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
 664         self._params = params
 665
 666     def http_open(self, req):
 667         return self.do_open(functools.partial(
 668             _create_http_connection, self, compat_http_client.HTTPConnection, False),
 669             req)
 670
 671     @staticmethod
 672     def deflate(data):
 673         try:
 674             return zlib.decompress(data, -zlib.MAX_WBITS)
 675         except zlib.error:
 676             return zlib.decompress(data)
 677
 678     @staticmethod
 679     def addinfourl_wrapper(stream, headers, url, code):
 680         if hasattr(compat_urllib_request.addinfourl, 'getcode'):
 681             return compat_urllib_request.addinfourl(stream, headers, url, code)
 682         ret = compat_urllib_request.addinfourl(stream, headers, url)
 683         ret.code = code
 684         return ret
 685
 686     def http_request(self, req):
 687         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
 688         # always respected by websites, some tend to give out URLs with non percent-encoded
 689         # non-ASCII characters (see telemb.py, ard.py [#3412])
 690         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
 691         # To work around aforementioned issue we will replace request's original URL with
 692         # percent-encoded one
 693         # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
 694         # the code of this workaround has been moved here from YoutubeDL.urlopen()
 695         url = req.get_full_url()
 696         url_escaped = escape_url(url)
 697
 698         # Substitute URL if any change after escaping
 699         if url != url_escaped:
 700             req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
 701             new_req = req_type(
 702                 url_escaped, data=req.data, headers=req.headers,
 703                 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
 704             new_req.timeout = req.timeout
 705             req = new_req
 706
 707         for h, v in std_headers.items():
 708             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
 709             # The dict keys are capitalized because of this bug by urllib
 710             if h.capitalize() not in req.headers:
 711                 req.add_header(h, v)
 712         if 'Youtubedl-no-compression' in req.headers:
 713             if 'Accept-encoding' in req.headers:
 714                 del req.headers['Accept-encoding']
 715             del req.headers['Youtubedl-no-compression']
 716
 717         if sys.version_info < (2, 7) and '#' in req.get_full_url():
 718             # Python 2.6 is brain-dead when it comes to fragments
 719             req._Request__original = req._Request__original.partition('#')[0]
 720             req._Request__r_type = req._Request__r_type.partition('#')[0]
 721
 722         return req
 723
 724     def http_response(self, req, resp):
 725         old_resp = resp
 726         # gzip
 727         if resp.headers.get('Content-encoding', '') == 'gzip':
 728             content = resp.read()
 729             gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
 730             try:
 731                 uncompressed = io.BytesIO(gz.read())
 732             except IOError as original_ioerror:
 733                 # There may be junk add the end of the file
 734                 # See http://stackoverflow.com/q/4928560/35070 for details
 735                 for i in range(1, 1024):
 736                     try:
 737                         gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
 738                         uncompressed = io.BytesIO(gz.read())
 739                     except IOError:
 740                         continue
 741                     break
 742                 else:
 743                     raise original_ioerror
 744             resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
 745             resp.msg = old_resp.msg
 746         # deflate
 747         if resp.headers.get('Content-encoding', '') == 'deflate':
 748             gz = io.BytesIO(self.deflate(resp.read()))
 749             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 750             resp.msg = old_resp.msg
 751         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
 752         if 300 <= resp.code < 400:
 753             location = resp.headers.get('Location')
 754             if location:
 755                 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
 756                 if sys.version_info >= (3, 0):
 757                     location = location.encode('iso-8859-1').decode('utf-8')
 758                 location_escaped = escape_url(location)
 759                 if location != location_escaped:
 760                     del resp.headers['Location']
 761                     resp.headers['Location'] = location_escaped
 762         return resp
 763
 764     https_request = http_request
 765     https_response = http_response
 766
 767
 768 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
 769     def __init__(self, params, https_conn_class=None, *args, **kwargs):
 770         compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
 771         self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
 772         self._params = params
 773
 774     def https_open(self, req):
 775         kwargs = {}
 776         if hasattr(self, '_context'):  # python > 2.6
 777             kwargs['context'] = self._context
 778         if hasattr(self, '_check_hostname'):  # python 3.x
 779             kwargs['check_hostname'] = self._check_hostname
 780         return self.do_open(functools.partial(
 781             _create_http_connection, self, self._https_conn_class, True),
 782             req, **kwargs)
 783
 784
 785 def parse_iso8601(date_str, delimiter='T', timezone=None):
 786     """ Return a UNIX timestamp from the given date """
 787
 788     if date_str is None:
 789         return None
 790
 791     if timezone is None:
 792         m = re.search(
 793             r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
 794             date_str)
 795         if not m:
 796             timezone = datetime.timedelta()
 797         else:
 798             date_str = date_str[:-len(m.group(0))]
 799             if not m.group('sign'):
 800                 timezone = datetime.timedelta()
 801             else:
 802                 sign = 1 if m.group('sign') == '+' else -1
 803                 timezone = datetime.timedelta(
 804                     hours=sign * int(m.group('hours')),
 805                     minutes=sign * int(m.group('minutes')))
 806     date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
 807     dt = datetime.datetime.strptime(date_str, date_format) - timezone
 808     return calendar.timegm(dt.timetuple())
 809
 810
 811 def unified_strdate(date_str, day_first=True):
 812     """Return a string with the date in the format YYYYMMDD"""
 813
 814     if date_str is None:
 815         return None
 816     upload_date = None
 817     # Replace commas
 818     date_str = date_str.replace(',', ' ')
 819     # %z (UTC offset) is only supported in python>=3.2
 820     if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
 821         date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
 822     # Remove AM/PM + timezone
 823     date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 824
 825     format_expressions = [
 826         '%d %B %Y',
 827         '%d %b %Y',
 828         '%B %d %Y',
 829         '%b %d %Y',
 830         '%b %dst %Y %I:%M%p',
 831         '%b %dnd %Y %I:%M%p',
 832         '%b %dth %Y %I:%M%p',
 833         '%Y %m %d',
 834         '%Y-%m-%d',
 835         '%Y/%m/%d',
 836         '%Y/%m/%d %H:%M:%S',
 837         '%Y-%m-%d %H:%M:%S',
 838         '%Y-%m-%d %H:%M:%S.%f',
 839         '%d.%m.%Y %H:%M',
 840         '%d.%m.%Y %H.%M',
 841         '%Y-%m-%dT%H:%M:%SZ',
 842         '%Y-%m-%dT%H:%M:%S.%fZ',
 843         '%Y-%m-%dT%H:%M:%S.%f0Z',
 844         '%Y-%m-%dT%H:%M:%S',
 845         '%Y-%m-%dT%H:%M:%S.%f',
 846         '%Y-%m-%dT%H:%M',
 847     ]
 848     if day_first:
 849         format_expressions.extend([
 850             '%d-%m-%Y',
 851             '%d.%m.%Y',
 852             '%d/%m/%Y',
 853             '%d/%m/%y',
 854             '%d/%m/%Y %H:%M:%S',
 855         ])
 856     else:
 857         format_expressions.extend([
 858             '%m-%d-%Y',
 859             '%m.%d.%Y',
 860             '%m/%d/%Y',
 861             '%m/%d/%y',
 862             '%m/%d/%Y %H:%M:%S',
 863         ])
 864     for expression in format_expressions:
 865         try:
 866             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
 867         except ValueError:
 868             pass
 869     if upload_date is None:
 870         timetuple = email.utils.parsedate_tz(date_str)
 871         if timetuple:
 872             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
 873     return upload_date
 874
 875
 876 def determine_ext(url, default_ext='unknown_video'):
 877     if url is None:
 878         return default_ext
 879     guess = url.partition('?')[0].rpartition('.')[2]
 880     if re.match(r'^[A-Za-z0-9]+$', guess):
 881         return guess
 882     else:
 883         return default_ext
 884
 885
 886 def subtitles_filename(filename, sub_lang, sub_format):
 887     return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
 888
 889
 890 def date_from_str(date_str):
 891     """
 892     Return a datetime object from a string in the format YYYYMMDD or
 893     (now|today)[+-][0-9](day|week|month|year)(s)?"""
 894     today = datetime.date.today()
 895     if date_str in ('now', 'today'):
 896         return today
 897     if date_str == 'yesterday':
 898         return today - datetime.timedelta(days=1)
 899     match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
 900     if match is not None:
 901         sign = match.group('sign')
 902         time = int(match.group('time'))
 903         if sign == '-':
 904             time = -time
 905         unit = match.group('unit')
 906         # A bad aproximation?
 907         if unit == 'month':
 908             unit = 'day'
 909             time *= 30
 910         elif unit == 'year':
 911             unit = 'day'
 912             time *= 365
 913         unit += 's'
 914         delta = datetime.timedelta(**{unit: time})
 915         return today + delta
 916     return datetime.datetime.strptime(date_str, "%Y%m%d").date()
 917
 918
 919 def hyphenate_date(date_str):
 920     """
 921     Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
 922     match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
 923     if match is not None:
 924         return '-'.join(match.groups())
 925     else:
 926         return date_str
 927
 928
 929 class DateRange(object):
 930     """Represents a time interval between two dates"""
 931
 932     def __init__(self, start=None, end=None):
 933         """start and end must be strings in the format accepted by date"""
 934         if start is not None:
 935             self.start = date_from_str(start)
 936         else:
 937             self.start = datetime.datetime.min.date()
 938         if end is not None:
 939             self.end = date_from_str(end)
 940         else:
 941             self.end = datetime.datetime.max.date()
 942         if self.start > self.end:
 943             raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
 944
 945     @classmethod
 946     def day(cls, day):
 947         """Returns a range that only contains the given day"""
 948         return cls(day, day)
 949
 950     def __contains__(self, date):
 951         """Check if the date is in the range"""
 952         if not isinstance(date, datetime.date):
 953             date = date_from_str(date)
 954         return self.start <= date <= self.end
 955
 956     def __str__(self):
 957         return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
 958
 959
 960 def platform_name():
 961     """ Returns the platform name as a compat_str """
 962     res = platform.platform()
 963     if isinstance(res, bytes):
 964         res = res.decode(preferredencoding())
 965
 966     assert isinstance(res, compat_str)
 967     return res
 968
 969
 970 def _windows_write_string(s, out):
 971     """ Returns True if the string was written using special methods,
 972     False if it has yet to be written out."""
 973     # Adapted from http://stackoverflow.com/a/3259271/35070
 974
 975     import ctypes
 976     import ctypes.wintypes
 977
 978     WIN_OUTPUT_IDS = {
 979         1: -11,
 980         2: -12,
 981     }
 982
 983     try:
 984         fileno = out.fileno()
 985     except AttributeError:
 986         # If the output stream doesn't have a fileno, it's virtual
 987         return False
 988     except io.UnsupportedOperation:
 989         # Some strange Windows pseudo files?
 990         return False
 991     if fileno not in WIN_OUTPUT_IDS:
 992         return False
 993
 994     GetStdHandle = ctypes.WINFUNCTYPE(
 995         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
 996         (b"GetStdHandle", ctypes.windll.kernel32))
 997     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 998
 999     WriteConsoleW = ctypes.WINFUNCTYPE(
1000         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1001         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1002         ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
1003     written = ctypes.wintypes.DWORD(0)
1004
1005     GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
1006     FILE_TYPE_CHAR = 0x0002
1007     FILE_TYPE_REMOTE = 0x8000
1008     GetConsoleMode = ctypes.WINFUNCTYPE(
1009         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1010         ctypes.POINTER(ctypes.wintypes.DWORD))(
1011         (b"GetConsoleMode", ctypes.windll.kernel32))
1012     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1013
1014     def not_a_console(handle):
1015         if handle == INVALID_HANDLE_VALUE or handle is None:
1016             return True
1017         return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1018                 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1019
1020     if not_a_console(h):
1021         return False
1022
1023     def next_nonbmp_pos(s):
1024         try:
1025             return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1026         except StopIteration:
1027             return len(s)
1028
1029     while s:
1030         count = min(next_nonbmp_pos(s), 1024)
1031
1032         ret = WriteConsoleW(
1033             h, s, count if count else 2, ctypes.byref(written), None)
1034         if ret == 0:
1035             raise OSError('Failed to write string')
1036         if not count:  # We just wrote a non-BMP character
1037             assert written.value == 2
1038             s = s[1:]
1039         else:
1040             assert written.value > 0
1041             s = s[written.value:]
1042     return True
1043
1044
1045 def write_string(s, out=None, encoding=None):
1046     if out is None:
1047         out = sys.stderr
1048     assert type(s) == compat_str
1049
1050     if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1051         if _windows_write_string(s, out):
1052             return
1053
1054     if ('b' in getattr(out, 'mode', '') or
1055             sys.version_info[0] < 3):  # Python 2 lies about mode of sys.stderr
1056         byt = s.encode(encoding or preferredencoding(), 'ignore')
1057         out.write(byt)
1058     elif hasattr(out, 'buffer'):
1059         enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1060         byt = s.encode(enc, 'ignore')
1061         out.buffer.write(byt)
1062     else:
1063         out.write(s)
1064     out.flush()
1065
1066
1067 def bytes_to_intlist(bs):
1068     if not bs:
1069         return []
1070     if isinstance(bs[0], int):  # Python 3
1071         return list(bs)
1072     else:
1073         return [ord(c) for c in bs]
1074
1075
1076 def intlist_to_bytes(xs):
1077     if not xs:
1078         return b''
1079     return struct_pack('%dB' % len(xs), *xs)
1080
1081
1082 # Cross-platform file locking
1083 if sys.platform == 'win32':
1084     import ctypes.wintypes
1085     import msvcrt
1086
1087     class OVERLAPPED(ctypes.Structure):
1088         _fields_ = [
1089             ('Internal', ctypes.wintypes.LPVOID),
1090             ('InternalHigh', ctypes.wintypes.LPVOID),
1091             ('Offset', ctypes.wintypes.DWORD),
1092             ('OffsetHigh', ctypes.wintypes.DWORD),
1093             ('hEvent', ctypes.wintypes.HANDLE),
1094         ]
1095
1096     kernel32 = ctypes.windll.kernel32
1097     LockFileEx = kernel32.LockFileEx
1098     LockFileEx.argtypes = [
1099         ctypes.wintypes.HANDLE,     # hFile
1100         ctypes.wintypes.DWORD,      # dwFlags
1101         ctypes.wintypes.DWORD,      # dwReserved
1102         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
1103         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
1104         ctypes.POINTER(OVERLAPPED)  # Overlapped
1105     ]
1106     LockFileEx.restype = ctypes.wintypes.BOOL
1107     UnlockFileEx = kernel32.UnlockFileEx
1108     UnlockFileEx.argtypes = [
1109         ctypes.wintypes.HANDLE,     # hFile
1110         ctypes.wintypes.DWORD,      # dwReserved
1111         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
1112         ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
1113         ctypes.POINTER(OVERLAPPED)  # Overlapped
1114     ]
1115     UnlockFileEx.restype = ctypes.wintypes.BOOL
1116     whole_low = 0xffffffff
1117     whole_high = 0x7fffffff
1118
1119     def _lock_file(f, exclusive):
1120         overlapped = OVERLAPPED()
1121         overlapped.Offset = 0
1122         overlapped.OffsetHigh = 0
1123         overlapped.hEvent = 0
1124         f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1125         handle = msvcrt.get_osfhandle(f.fileno())
1126         if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1127                           whole_low, whole_high, f._lock_file_overlapped_p):
1128             raise OSError('Locking file failed: %r' % ctypes.FormatError())
1129
1130     def _unlock_file(f):
1131         assert f._lock_file_overlapped_p
1132         handle = msvcrt.get_osfhandle(f.fileno())
1133         if not UnlockFileEx(handle, 0,
1134                             whole_low, whole_high, f._lock_file_overlapped_p):
1135             raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1136
1137 else:
1138     import fcntl
1139
1140     def _lock_file(f, exclusive):
1141         fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1142
1143     def _unlock_file(f):
1144         fcntl.flock(f, fcntl.LOCK_UN)
1145
1146
1147 class locked_file(object):
1148     def __init__(self, filename, mode, encoding=None):
1149         assert mode in ['r', 'a', 'w']
1150         self.f = io.open(filename, mode, encoding=encoding)
1151         self.mode = mode
1152
1153     def __enter__(self):
1154         exclusive = self.mode != 'r'
1155         try:
1156             _lock_file(self.f, exclusive)
1157         except IOError:
1158             self.f.close()
1159             raise
1160         return self
1161
1162     def __exit__(self, etype, value, traceback):
1163         try:
1164             _unlock_file(self.f)
1165         finally:
1166             self.f.close()
1167
1168     def __iter__(self):
1169         return iter(self.f)
1170
1171     def write(self, *args):
1172         return self.f.write(*args)
1173
1174     def read(self, *args):
1175         return self.f.read(*args)
1176
1177
1178 def get_filesystem_encoding():
1179     encoding = sys.getfilesystemencoding()
1180     return encoding if encoding is not None else 'utf-8'
1181
1182
1183 def shell_quote(args):
1184     quoted_args = []
1185     encoding = get_filesystem_encoding()
1186     for a in args:
1187         if isinstance(a, bytes):
1188             # We may get a filename encoded with 'encodeFilename'
1189             a = a.decode(encoding)
1190         quoted_args.append(pipes.quote(a))
1191     return ' '.join(quoted_args)
1192
1193
1194 def smuggle_url(url, data):
1195     """ Pass additional data in a URL for internal use. """
1196
1197     sdata = compat_urllib_parse.urlencode(
1198         {'__youtubedl_smuggle': json.dumps(data)})
1199     return url + '#' + sdata
1200
1201
1202 def unsmuggle_url(smug_url, default=None):
1203     if '#__youtubedl_smuggle' not in smug_url:
1204         return smug_url, default
1205     url, _, sdata = smug_url.rpartition('#')
1206     jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1207     data = json.loads(jsond)
1208     return url, data
1209
1210
1211 def format_bytes(bytes):
1212     if bytes is None:
1213         return 'N/A'
1214     if type(bytes) is str:
1215         bytes = float(bytes)
1216     if bytes == 0.0:
1217         exponent = 0
1218     else:
1219         exponent = int(math.log(bytes, 1024.0))
1220     suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1221     converted = float(bytes) / float(1024 ** exponent)
1222     return '%.2f%s' % (converted, suffix)
1223
1224
1225 def parse_filesize(s):
1226     if s is None:
1227         return None
1228
1229     # The lower-case forms are of course incorrect and inofficial,
1230     # but we support those too
1231     _UNIT_TABLE = {
1232         'B': 1,
1233         'b': 1,
1234         'KiB': 1024,
1235         'KB': 1000,
1236         'kB': 1024,
1237         'Kb': 1000,
1238         'MiB': 1024 ** 2,
1239         'MB': 1000 ** 2,
1240         'mB': 1024 ** 2,
1241         'Mb': 1000 ** 2,
1242         'GiB': 1024 ** 3,
1243         'GB': 1000 ** 3,
1244         'gB': 1024 ** 3,
1245         'Gb': 1000 ** 3,
1246         'TiB': 1024 ** 4,
1247         'TB': 1000 ** 4,
1248         'tB': 1024 ** 4,
1249         'Tb': 1000 ** 4,
1250         'PiB': 1024 ** 5,
1251         'PB': 1000 ** 5,
1252         'pB': 1024 ** 5,
1253         'Pb': 1000 ** 5,
1254         'EiB': 1024 ** 6,
1255         'EB': 1000 ** 6,
1256         'eB': 1024 ** 6,
1257         'Eb': 1000 ** 6,
1258         'ZiB': 1024 ** 7,
1259         'ZB': 1000 ** 7,
1260         'zB': 1024 ** 7,
1261         'Zb': 1000 ** 7,
1262         'YiB': 1024 ** 8,
1263         'YB': 1000 ** 8,
1264         'yB': 1024 ** 8,
1265         'Yb': 1000 ** 8,
1266     }
1267
1268     units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1269     m = re.match(
1270         r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1271     if not m:
1272         return None
1273
1274     num_str = m.group('num').replace(',', '.')
1275     mult = _UNIT_TABLE[m.group('unit')]
1276     return int(float(num_str) * mult)
1277
1278
1279 def month_by_name(name):
1280     """ Return the number of a month by (locale-independently) English name """
1281
1282     try:
1283         return ENGLISH_MONTH_NAMES.index(name) + 1
1284     except ValueError:
1285         return None
1286
1287
1288 def month_by_abbreviation(abbrev):
1289     """ Return the number of a month by (locale-independently) English
1290         abbreviations """
1291
1292     try:
1293         return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1294     except ValueError:
1295         return None
1296
1297
1298 def fix_xml_ampersands(xml_str):
1299     """Replace all the '&' by '&amp;' in XML"""
1300     return re.sub(
1301         r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1302         '&amp;',
1303         xml_str)
1304
1305
1306 def setproctitle(title):
1307     assert isinstance(title, compat_str)
1308     try:
1309         libc = ctypes.cdll.LoadLibrary("libc.so.6")
1310     except OSError:
1311         return
1312     title_bytes = title.encode('utf-8')
1313     buf = ctypes.create_string_buffer(len(title_bytes))
1314     buf.value = title_bytes
1315     try:
1316         libc.prctl(15, buf, 0, 0, 0)
1317     except AttributeError:
1318         return  # Strange libc, just skip this
1319
1320
1321 def remove_start(s, start):
1322     if s.startswith(start):
1323         return s[len(start):]
1324     return s
1325
1326
1327 def remove_end(s, end):
1328     if s.endswith(end):
1329         return s[:-len(end)]
1330     return s
1331
1332
1333 def url_basename(url):
1334     path = compat_urlparse.urlparse(url).path
1335     return path.strip('/').split('/')[-1]
1336
1337
1338 class HEADRequest(compat_urllib_request.Request):
1339     def get_method(self):
1340         return "HEAD"
1341
1342
1343 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1344     if get_attr:
1345         if v is not None:
1346             v = getattr(v, get_attr, None)
1347     if v == '':
1348         v = None
1349     return default if v is None else (int(v) * invscale // scale)
1350
1351
1352 def str_or_none(v, default=None):
1353     return default if v is None else compat_str(v)
1354
1355
1356 def str_to_int(int_str):
1357     """ A more relaxed version of int_or_none """
1358     if int_str is None:
1359         return None
1360     int_str = re.sub(r'[,\.\+]', '', int_str)
1361     return int(int_str)
1362
1363
1364 def float_or_none(v, scale=1, invscale=1, default=None):
1365     return default if v is None else (float(v) * invscale / scale)
1366
1367
1368 def parse_duration(s):
1369     if not isinstance(s, compat_basestring):
1370         return None
1371
1372     s = s.strip()
1373
1374     m = re.match(
1375         r'''(?ix)(?:P?T)?
1376         (?:
1377             (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1378             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1379
1380             \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1381             (?:
1382                 (?:
1383                     (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1384                     (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1385                 )?
1386                 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1387             )?
1388             (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1389         )$''', s)
1390     if not m:
1391         return None
1392     res = 0
1393     if m.group('only_mins'):
1394         return float_or_none(m.group('only_mins'), invscale=60)
1395     if m.group('only_hours'):
1396         return float_or_none(m.group('only_hours'), invscale=60 * 60)
1397     if m.group('secs'):
1398         res += int(m.group('secs'))
1399     if m.group('mins_reversed'):
1400         res += int(m.group('mins_reversed')) * 60
1401     if m.group('mins'):
1402         res += int(m.group('mins')) * 60
1403     if m.group('hours'):
1404         res += int(m.group('hours')) * 60 * 60
1405     if m.group('hours_reversed'):
1406         res += int(m.group('hours_reversed')) * 60 * 60
1407     if m.group('days'):
1408         res += int(m.group('days')) * 24 * 60 * 60
1409     if m.group('ms'):
1410         res += float(m.group('ms'))
1411     return res
1412
1413
1414 def prepend_extension(filename, ext, expected_real_ext=None):
1415     name, real_ext = os.path.splitext(filename)
1416     return (
1417         '{0}.{1}{2}'.format(name, ext, real_ext)
1418         if not expected_real_ext or real_ext[1:] == expected_real_ext
1419         else '{0}.{1}'.format(filename, ext))
1420
1421
1422 def replace_extension(filename, ext, expected_real_ext=None):
1423     name, real_ext = os.path.splitext(filename)
1424     return '{0}.{1}'.format(
1425         name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1426         ext)
1427
1428
1429 def check_executable(exe, args=[]):
1430     """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1431     args can be a list of arguments for a short output (like -version) """
1432     try:
1433         subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1434     except OSError:
1435         return False
1436     return exe
1437
1438
1439 def get_exe_version(exe, args=['--version'],
1440                     version_re=None, unrecognized='present'):
1441     """ Returns the version of the specified executable,
1442     or False if the executable is not present """
1443     try:
1444         out, _ = subprocess.Popen(
1445             [encodeArgument(exe)] + args,
1446             stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1447     except OSError:
1448         return False
1449     if isinstance(out, bytes):  # Python 2.x
1450         out = out.decode('ascii', 'ignore')
1451     return detect_exe_version(out, version_re, unrecognized)
1452
1453
1454 def detect_exe_version(output, version_re=None, unrecognized='present'):
1455     assert isinstance(output, compat_str)
1456     if version_re is None:
1457         version_re = r'version\s+([-0-9._a-zA-Z]+)'
1458     m = re.search(version_re, output)
1459     if m:
1460         return m.group(1)
1461     else:
1462         return unrecognized
1463
1464
1465 class PagedList(object):
1466     def __len__(self):
1467         # This is only useful for tests
1468         return len(self.getslice())
1469
1470
1471 class OnDemandPagedList(PagedList):
1472     def __init__(self, pagefunc, pagesize):
1473         self._pagefunc = pagefunc
1474         self._pagesize = pagesize
1475
1476     def getslice(self, start=0, end=None):
1477         res = []
1478         for pagenum in itertools.count(start // self._pagesize):
1479             firstid = pagenum * self._pagesize
1480             nextfirstid = pagenum * self._pagesize + self._pagesize
1481             if start >= nextfirstid:
1482                 continue
1483
1484             page_results = list(self._pagefunc(pagenum))
1485
1486             startv = (
1487                 start % self._pagesize
1488                 if firstid <= start < nextfirstid
1489                 else 0)
1490
1491             endv = (
1492                 ((end - 1) % self._pagesize) + 1
1493                 if (end is not None and firstid <= end <= nextfirstid)
1494                 else None)
1495
1496             if startv != 0 or endv is not None:
1497                 page_results = page_results[startv:endv]
1498             res.extend(page_results)
1499
1500             # A little optimization - if current page is not "full", ie. does
1501             # not contain page_size videos then we can assume that this page
1502             # is the last one - there are no more ids on further pages -
1503             # i.e. no need to query again.
1504             if len(page_results) + startv < self._pagesize:
1505                 break
1506
1507             # If we got the whole page, but the next page is not interesting,
1508             # break out early as well
1509             if end == nextfirstid:
1510                 break
1511         return res
1512
1513
1514 class InAdvancePagedList(PagedList):
1515     def __init__(self, pagefunc, pagecount, pagesize):
1516         self._pagefunc = pagefunc
1517         self._pagecount = pagecount
1518         self._pagesize = pagesize
1519
1520     def getslice(self, start=0, end=None):
1521         res = []
1522         start_page = start // self._pagesize
1523         end_page = (
1524             self._pagecount if end is None else (end // self._pagesize + 1))
1525         skip_elems = start - start_page * self._pagesize
1526         only_more = None if end is None else end - start
1527         for pagenum in range(start_page, end_page):
1528             page = list(self._pagefunc(pagenum))
1529             if skip_elems:
1530                 page = page[skip_elems:]
1531                 skip_elems = None
1532             if only_more is not None:
1533                 if len(page) < only_more:
1534                     only_more -= len(page)
1535                 else:
1536                     page = page[:only_more]
1537                     res.extend(page)
1538                     break
1539             res.extend(page)
1540         return res
1541
1542
1543 def uppercase_escape(s):
1544     unicode_escape = codecs.getdecoder('unicode_escape')
1545     return re.sub(
1546         r'\\U[0-9a-fA-F]{8}',
1547         lambda m: unicode_escape(m.group(0))[0],
1548         s)
1549
1550
1551 def lowercase_escape(s):
1552     unicode_escape = codecs.getdecoder('unicode_escape')
1553     return re.sub(
1554         r'\\u[0-9a-fA-F]{4}',
1555         lambda m: unicode_escape(m.group(0))[0],
1556         s)
1557
1558
1559 def escape_rfc3986(s):
1560     """Escape non-ASCII characters as suggested by RFC 3986"""
1561     if sys.version_info < (3, 0) and isinstance(s, compat_str):
1562         s = s.encode('utf-8')
1563     return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1564
1565
1566 def escape_url(url):
1567     """Escape URL as suggested by RFC 3986"""
1568     url_parsed = compat_urllib_parse_urlparse(url)
1569     return url_parsed._replace(
1570         path=escape_rfc3986(url_parsed.path),
1571         params=escape_rfc3986(url_parsed.params),
1572         query=escape_rfc3986(url_parsed.query),
1573         fragment=escape_rfc3986(url_parsed.fragment)
1574     ).geturl()
1575
1576 try:
1577     struct.pack('!I', 0)
1578 except TypeError:
1579     # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1580     def struct_pack(spec, *args):
1581         if isinstance(spec, compat_str):
1582             spec = spec.encode('ascii')
1583         return struct.pack(spec, *args)
1584
1585     def struct_unpack(spec, *args):
1586         if isinstance(spec, compat_str):
1587             spec = spec.encode('ascii')
1588         return struct.unpack(spec, *args)
1589 else:
1590     struct_pack = struct.pack
1591     struct_unpack = struct.unpack
1592
1593
1594 def read_batch_urls(batch_fd):
1595     def fixup(url):
1596         if not isinstance(url, compat_str):
1597             url = url.decode('utf-8', 'replace')
1598         BOM_UTF8 = '\xef\xbb\xbf'
1599         if url.startswith(BOM_UTF8):
1600             url = url[len(BOM_UTF8):]
1601         url = url.strip()
1602         if url.startswith(('#', ';', ']')):
1603             return False
1604         return url
1605
1606     with contextlib.closing(batch_fd) as fd:
1607         return [url for url in map(fixup, fd) if url]
1608
1609
1610 def urlencode_postdata(*args, **kargs):
1611     return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1612
1613
1614 try:
1615     etree_iter = xml.etree.ElementTree.Element.iter
1616 except AttributeError:  # Python <=2.6
1617     etree_iter = lambda n: n.findall('.//*')
1618
1619
1620 def parse_xml(s):
1621     class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
1622         def doctype(self, name, pubid, system):
1623             pass  # Ignore doctypes
1624
1625     parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
1626     kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
1627     tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
1628     # Fix up XML parser in Python 2.x
1629     if sys.version_info < (3, 0):
1630         for n in etree_iter(tree):
1631             if n.text is not None:
1632                 if not isinstance(n.text, compat_str):
1633                     n.text = n.text.decode('utf-8')
1634     return tree
1635
1636
1637 US_RATINGS = {
1638     'G': 0,
1639     'PG': 10,
1640     'PG-13': 13,
1641     'R': 16,
1642     'NC': 18,
1643 }
1644
1645
1646 def parse_age_limit(s):
1647     if s is None:
1648         return None
1649     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1650     return int(m.group('age')) if m else US_RATINGS.get(s, None)
1651
1652
1653 def strip_jsonp(code):
1654     return re.sub(
1655         r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1656
1657
1658 def js_to_json(code):
1659     def fix_kv(m):
1660         v = m.group(0)
1661         if v in ('true', 'false', 'null'):
1662             return v
1663         if v.startswith('"'):
1664             return v
1665         if v.startswith("'"):
1666             v = v[1:-1]
1667             v = re.sub(r"\\\\|\\'|\"", lambda m: {
1668                 '\\\\': '\\\\',
1669                 "\\'": "'",
1670                 '"': '\\"',
1671             }[m.group(0)], v)
1672         return '"%s"' % v
1673
1674     res = re.sub(r'''(?x)
1675         "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1676         '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1677         [a-zA-Z_][.a-zA-Z_0-9]*
1678         ''', fix_kv, code)
1679     res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1680     return res
1681
1682
1683 def qualities(quality_ids):
1684     """ Get a numeric quality value out of a list of possible values """
1685     def q(qid):
1686         try:
1687             return quality_ids.index(qid)
1688         except ValueError:
1689             return -1
1690     return q
1691
1692
1693 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1694
1695
1696 def limit_length(s, length):
1697     """ Add ellipses to overly long strings """
1698     if s is None:
1699         return None
1700     ELLIPSES = '...'
1701     if len(s) > length:
1702         return s[:length - len(ELLIPSES)] + ELLIPSES
1703     return s
1704
1705
1706 def version_tuple(v):
1707     return tuple(int(e) for e in re.split(r'[-.]', v))
1708
1709
1710 def is_outdated_version(version, limit, assume_new=True):
1711     if not version:
1712         return not assume_new
1713     try:
1714         return version_tuple(version) < version_tuple(limit)
1715     except ValueError:
1716         return not assume_new
1717
1718
1719 def ytdl_is_updateable():
1720     """ Returns if youtube-dl can be updated with -U """
1721     from zipimport import zipimporter
1722
1723     return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1724
1725
1726 def args_to_str(args):
1727     # Get a short string representation for a subprocess command
1728     return ' '.join(shlex_quote(a) for a in args)
1729
1730
1731 def mimetype2ext(mt):
1732     _, _, res = mt.rpartition('/')
1733
1734     return {
1735         'x-ms-wmv': 'wmv',
1736         'x-mp4-fragmented': 'mp4',
1737         'ttml+xml': 'ttml',
1738     }.get(res, res)
1739
1740
1741 def urlhandle_detect_ext(url_handle):
1742     try:
1743         url_handle.headers
1744         getheader = lambda h: url_handle.headers[h]
1745     except AttributeError:  # Python < 3
1746         getheader = url_handle.info().getheader
1747
1748     cd = getheader('Content-Disposition')
1749     if cd:
1750         m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1751         if m:
1752             e = determine_ext(m.group('filename'), default_ext=None)
1753             if e:
1754                 return e
1755
1756     return mimetype2ext(getheader('Content-Type'))
1757
1758
1759 def age_restricted(content_limit, age_limit):
1760     """ Returns True iff the content should be blocked """
1761
1762     if age_limit is None:  # No limit set
1763         return False
1764     if content_limit is None:
1765         return False  # Content available for everyone
1766     return age_limit < content_limit
1767
1768
1769 def is_html(first_bytes):
1770     """ Detect whether a file contains HTML by examining its first bytes. """
1771
1772     BOMS = [
1773         (b'\xef\xbb\xbf', 'utf-8'),
1774         (b'\x00\x00\xfe\xff', 'utf-32-be'),
1775         (b'\xff\xfe\x00\x00', 'utf-32-le'),
1776         (b'\xff\xfe', 'utf-16-le'),
1777         (b'\xfe\xff', 'utf-16-be'),
1778     ]
1779     for bom, enc in BOMS:
1780         if first_bytes.startswith(bom):
1781             s = first_bytes[len(bom):].decode(enc, 'replace')
1782             break
1783     else:
1784         s = first_bytes.decode('utf-8', 'replace')
1785
1786     return re.match(r'^\s*<', s)
1787
1788
1789 def determine_protocol(info_dict):
1790     protocol = info_dict.get('protocol')
1791     if protocol is not None:
1792         return protocol
1793
1794     url = info_dict['url']
1795     if url.startswith('rtmp'):
1796         return 'rtmp'
1797     elif url.startswith('mms'):
1798         return 'mms'
1799     elif url.startswith('rtsp'):
1800         return 'rtsp'
1801
1802     ext = determine_ext(url)
1803     if ext == 'm3u8':
1804         return 'm3u8'
1805     elif ext == 'f4m':
1806         return 'f4m'
1807
1808     return compat_urllib_parse_urlparse(url).scheme
1809
1810
1811 def render_table(header_row, data):
1812     """ Render a list of rows, each as a list of values """
1813     table = [header_row] + data
1814     max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1815     format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1816     return '\n'.join(format_str % tuple(row) for row in table)
1817
1818
1819 def _match_one(filter_part, dct):
1820     COMPARISON_OPERATORS = {
1821         '<': operator.lt,
1822         '<=': operator.le,
1823         '>': operator.gt,
1824         '>=': operator.ge,
1825         '=': operator.eq,
1826         '!=': operator.ne,
1827     }
1828     operator_rex = re.compile(r'''(?x)\s*
1829         (?P<key>[a-z_]+)
1830         \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1831         (?:
1832             (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1833             (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1834         )
1835         \s*$
1836         ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1837     m = operator_rex.search(filter_part)
1838     if m:
1839         op = COMPARISON_OPERATORS[m.group('op')]
1840         if m.group('strval') is not None:
1841             if m.group('op') not in ('=', '!='):
1842                 raise ValueError(
1843                     'Operator %s does not support string values!' % m.group('op'))
1844             comparison_value = m.group('strval')
1845         else:
1846             try:
1847                 comparison_value = int(m.group('intval'))
1848             except ValueError:
1849                 comparison_value = parse_filesize(m.group('intval'))
1850                 if comparison_value is None:
1851                     comparison_value = parse_filesize(m.group('intval') + 'B')
1852                 if comparison_value is None:
1853                     raise ValueError(
1854                         'Invalid integer value %r in filter part %r' % (
1855                             m.group('intval'), filter_part))
1856         actual_value = dct.get(m.group('key'))
1857         if actual_value is None:
1858             return m.group('none_inclusive')
1859         return op(actual_value, comparison_value)
1860
1861     UNARY_OPERATORS = {
1862         '': lambda v: v is not None,
1863         '!': lambda v: v is None,
1864     }
1865     operator_rex = re.compile(r'''(?x)\s*
1866         (?P<op>%s)\s*(?P<key>[a-z_]+)
1867         \s*$
1868         ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1869     m = operator_rex.search(filter_part)
1870     if m:
1871         op = UNARY_OPERATORS[m.group('op')]
1872         actual_value = dct.get(m.group('key'))
1873         return op(actual_value)
1874
1875     raise ValueError('Invalid filter part %r' % filter_part)
1876
1877
1878 def match_str(filter_str, dct):
1879     """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1880
1881     return all(
1882         _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1883
1884
1885 def match_filter_func(filter_str):
1886     def _match_func(info_dict):
1887         if match_str(filter_str, info_dict):
1888             return None
1889         else:
1890             video_title = info_dict.get('title', info_dict.get('id', 'video'))
1891             return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1892     return _match_func
1893
1894
1895 def parse_dfxp_time_expr(time_expr):
1896     if not time_expr:
1897         return 0.0
1898
1899     mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1900     if mobj:
1901         return float(mobj.group('time_offset'))
1902
1903     mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1904     if mobj:
1905         return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1906
1907
1908 def srt_subtitles_timecode(seconds):
1909     return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1910
1911
1912 def dfxp2srt(dfxp_data):
1913     _x = functools.partial(xpath_with_ns, ns_map={
1914         'ttml': 'http://www.w3.org/ns/ttml',
1915         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
1916     })
1917
1918     def parse_node(node):
1919         str_or_empty = functools.partial(str_or_none, default='')
1920
1921         out = str_or_empty(node.text)
1922
1923         for child in node:
1924             if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
1925                 out += '\n' + str_or_empty(child.tail)
1926             elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
1927                 out += str_or_empty(parse_node(child))
1928             else:
1929                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
1930
1931         return out
1932
1933     dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
1934     out = []
1935     paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
1936
1937     if not paras:
1938         raise ValueError('Invalid dfxp/TTML subtitle')
1939
1940     for para, index in zip(paras, itertools.count(1)):
1941         begin_time = parse_dfxp_time_expr(para.attrib['begin'])
1942         end_time = parse_dfxp_time_expr(para.attrib.get('end'))
1943         if not end_time:
1944             end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
1945         out.append('%d\n%s --> %s\n%s\n\n' % (
1946             index,
1947             srt_subtitles_timecode(begin_time),
1948             srt_subtitles_timecode(end_time),
1949             parse_node(para)))
1950
1951     return ''.join(out)
1952
1953
1954 def cli_option(params, command_option, param):
1955     param = params.get(param)
1956     return [command_option, param] if param is not None else []
1957
1958
1959 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
1960     param = params.get(param)
1961     assert isinstance(param, bool)
1962     if separator:
1963         return [command_option + separator + (true_value if param else false_value)]
1964     return [command_option, true_value if param else false_value]
1965
1966
1967 def cli_valueless_option(params, command_option, param, expected_value=True):
1968     param = params.get(param)
1969     return [command_option] if param == expected_value else []
1970
1971
1972 def cli_configuration_args(params, param, default=[]):
1973     ex_args = params.get(param)
1974     if ex_args is None:
1975         return default
1976     assert isinstance(ex_args, list)
1977     return ex_args
1978
1979
1980 class ISO639Utils(object):
1981     # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
1982     _lang_map = {
1983         'aa': 'aar',
1984         'ab': 'abk',
1985         'ae': 'ave',
1986         'af': 'afr',
1987         'ak': 'aka',
1988         'am': 'amh',
1989         'an': 'arg',
1990         'ar': 'ara',
1991         'as': 'asm',
1992         'av': 'ava',
1993         'ay': 'aym',
1994         'az': 'aze',
1995         'ba': 'bak',
1996         'be': 'bel',
1997         'bg': 'bul',
1998         'bh': 'bih',
1999         'bi': 'bis',
2000         'bm': 'bam',
2001         'bn': 'ben',
2002         'bo': 'bod',
2003         'br': 'bre',
2004         'bs': 'bos',
2005         'ca': 'cat',
2006         'ce': 'che',
2007         'ch': 'cha',
2008         'co': 'cos',
2009         'cr': 'cre',
2010         'cs': 'ces',
2011         'cu': 'chu',
2012         'cv': 'chv',
2013         'cy': 'cym',
2014         'da': 'dan',
2015         'de': 'deu',
2016         'dv': 'div',
2017         'dz': 'dzo',
2018         'ee': 'ewe',
2019         'el': 'ell',
2020         'en': 'eng',
2021         'eo': 'epo',
2022         'es': 'spa',
2023         'et': 'est',
2024         'eu': 'eus',
2025         'fa': 'fas',
2026         'ff': 'ful',
2027         'fi': 'fin',
2028         'fj': 'fij',
2029         'fo': 'fao',
2030         'fr': 'fra',
2031         'fy': 'fry',
2032         'ga': 'gle',
2033         'gd': 'gla',
2034         'gl': 'glg',
2035         'gn': 'grn',
2036         'gu': 'guj',
2037         'gv': 'glv',
2038         'ha': 'hau',
2039         'he': 'heb',
2040         'hi': 'hin',
2041         'ho': 'hmo',
2042         'hr': 'hrv',
2043         'ht': 'hat',
2044         'hu': 'hun',
2045         'hy': 'hye',
2046         'hz': 'her',
2047         'ia': 'ina',
2048         'id': 'ind',
2049         'ie': 'ile',
2050         'ig': 'ibo',
2051         'ii': 'iii',
2052         'ik': 'ipk',
2053         'io': 'ido',
2054         'is': 'isl',
2055         'it': 'ita',
2056         'iu': 'iku',
2057         'ja': 'jpn',
2058         'jv': 'jav',
2059         'ka': 'kat',
2060         'kg': 'kon',
2061         'ki': 'kik',
2062         'kj': 'kua',
2063         'kk': 'kaz',
2064         'kl': 'kal',
2065         'km': 'khm',
2066         'kn': 'kan',
2067         'ko': 'kor',
2068         'kr': 'kau',
2069         'ks': 'kas',
2070         'ku': 'kur',
2071         'kv': 'kom',
2072         'kw': 'cor',
2073         'ky': 'kir',
2074         'la': 'lat',
2075         'lb': 'ltz',
2076         'lg': 'lug',
2077         'li': 'lim',
2078         'ln': 'lin',
2079         'lo': 'lao',
2080         'lt': 'lit',
2081         'lu': 'lub',
2082         'lv': 'lav',
2083         'mg': 'mlg',
2084         'mh': 'mah',
2085         'mi': 'mri',
2086         'mk': 'mkd',
2087         'ml': 'mal',
2088         'mn': 'mon',
2089         'mr': 'mar',
2090         'ms': 'msa',
2091         'mt': 'mlt',
2092         'my': 'mya',
2093         'na': 'nau',
2094         'nb': 'nob',
2095         'nd': 'nde',
2096         'ne': 'nep',
2097         'ng': 'ndo',
2098         'nl': 'nld',
2099         'nn': 'nno',
2100         'no': 'nor',
2101         'nr': 'nbl',
2102         'nv': 'nav',
2103         'ny': 'nya',
2104         'oc': 'oci',
2105         'oj': 'oji',
2106         'om': 'orm',
2107         'or': 'ori',
2108         'os': 'oss',
2109         'pa': 'pan',
2110         'pi': 'pli',
2111         'pl': 'pol',
2112         'ps': 'pus',
2113         'pt': 'por',
2114         'qu': 'que',
2115         'rm': 'roh',
2116         'rn': 'run',
2117         'ro': 'ron',
2118         'ru': 'rus',
2119         'rw': 'kin',
2120         'sa': 'san',
2121         'sc': 'srd',
2122         'sd': 'snd',
2123         'se': 'sme',
2124         'sg': 'sag',
2125         'si': 'sin',
2126         'sk': 'slk',
2127         'sl': 'slv',
2128         'sm': 'smo',
2129         'sn': 'sna',
2130         'so': 'som',
2131         'sq': 'sqi',
2132         'sr': 'srp',
2133         'ss': 'ssw',
2134         'st': 'sot',
2135         'su': 'sun',
2136         'sv': 'swe',
2137         'sw': 'swa',
2138         'ta': 'tam',
2139         'te': 'tel',
2140         'tg': 'tgk',
2141         'th': 'tha',
2142         'ti': 'tir',
2143         'tk': 'tuk',
2144         'tl': 'tgl',
2145         'tn': 'tsn',
2146         'to': 'ton',
2147         'tr': 'tur',
2148         'ts': 'tso',
2149         'tt': 'tat',
2150         'tw': 'twi',
2151         'ty': 'tah',
2152         'ug': 'uig',
2153         'uk': 'ukr',
2154         'ur': 'urd',
2155         'uz': 'uzb',
2156         've': 'ven',
2157         'vi': 'vie',
2158         'vo': 'vol',
2159         'wa': 'wln',
2160         'wo': 'wol',
2161         'xh': 'xho',
2162         'yi': 'yid',
2163         'yo': 'yor',
2164         'za': 'zha',
2165         'zh': 'zho',
2166         'zu': 'zul',
2167     }
2168
2169     @classmethod
2170     def short2long(cls, code):
2171         """Convert language code from ISO 639-1 to ISO 639-2/T"""
2172         return cls._lang_map.get(code[:2])
2173
2174     @classmethod
2175     def long2short(cls, code):
2176         """Convert language code from ISO 639-2/T to ISO 639-1"""
2177         for short_name, long_name in cls._lang_map.items():
2178             if long_name == code:
2179                 return short_name
2180
2181
2182 class ISO3166Utils(object):
2183     # From http://data.okfn.org/data/core/country-list
2184     _country_map = {
2185         'AF': 'Afghanistan',
2186         'AX': 'Åland Islands',
2187         'AL': 'Albania',
2188         'DZ': 'Algeria',
2189         'AS': 'American Samoa',
2190         'AD': 'Andorra',
2191         'AO': 'Angola',
2192         'AI': 'Anguilla',
2193         'AQ': 'Antarctica',
2194         'AG': 'Antigua and Barbuda',
2195         'AR': 'Argentina',
2196         'AM': 'Armenia',
2197         'AW': 'Aruba',
2198         'AU': 'Australia',
2199         'AT': 'Austria',
2200         'AZ': 'Azerbaijan',
2201         'BS': 'Bahamas',
2202         'BH': 'Bahrain',
2203         'BD': 'Bangladesh',
2204         'BB': 'Barbados',
2205         'BY': 'Belarus',
2206         'BE': 'Belgium',
2207         'BZ': 'Belize',
2208         'BJ': 'Benin',
2209         'BM': 'Bermuda',
2210         'BT': 'Bhutan',
2211         'BO': 'Bolivia, Plurinational State of',
2212         'BQ': 'Bonaire, Sint Eustatius and Saba',
2213         'BA': 'Bosnia and Herzegovina',
2214         'BW': 'Botswana',
2215         'BV': 'Bouvet Island',
2216         'BR': 'Brazil',
2217         'IO': 'British Indian Ocean Territory',
2218         'BN': 'Brunei Darussalam',
2219         'BG': 'Bulgaria',
2220         'BF': 'Burkina Faso',
2221         'BI': 'Burundi',
2222         'KH': 'Cambodia',
2223         'CM': 'Cameroon',
2224         'CA': 'Canada',
2225         'CV': 'Cape Verde',
2226         'KY': 'Cayman Islands',
2227         'CF': 'Central African Republic',
2228         'TD': 'Chad',
2229         'CL': 'Chile',
2230         'CN': 'China',
2231         'CX': 'Christmas Island',
2232         'CC': 'Cocos (Keeling) Islands',
2233         'CO': 'Colombia',
2234         'KM': 'Comoros',
2235         'CG': 'Congo',
2236         'CD': 'Congo, the Democratic Republic of the',
2237         'CK': 'Cook Islands',
2238         'CR': 'Costa Rica',
2239         'CI': 'Côte d\'Ivoire',
2240         'HR': 'Croatia',
2241         'CU': 'Cuba',
2242         'CW': 'Curaçao',
2243         'CY': 'Cyprus',
2244         'CZ': 'Czech Republic',
2245         'DK': 'Denmark',
2246         'DJ': 'Djibouti',
2247         'DM': 'Dominica',
2248         'DO': 'Dominican Republic',
2249         'EC': 'Ecuador',
2250         'EG': 'Egypt',
2251         'SV': 'El Salvador',
2252         'GQ': 'Equatorial Guinea',
2253         'ER': 'Eritrea',
2254         'EE': 'Estonia',
2255         'ET': 'Ethiopia',
2256         'FK': 'Falkland Islands (Malvinas)',
2257         'FO': 'Faroe Islands',
2258         'FJ': 'Fiji',
2259         'FI': 'Finland',
2260         'FR': 'France',
2261         'GF': 'French Guiana',
2262         'PF': 'French Polynesia',
2263         'TF': 'French Southern Territories',
2264         'GA': 'Gabon',
2265         'GM': 'Gambia',
2266         'GE': 'Georgia',
2267         'DE': 'Germany',
2268         'GH': 'Ghana',
2269         'GI': 'Gibraltar',
2270         'GR': 'Greece',
2271         'GL': 'Greenland',
2272         'GD': 'Grenada',
2273         'GP': 'Guadeloupe',
2274         'GU': 'Guam',
2275         'GT': 'Guatemala',
2276         'GG': 'Guernsey',
2277         'GN': 'Guinea',
2278         'GW': 'Guinea-Bissau',
2279         'GY': 'Guyana',
2280         'HT': 'Haiti',
2281         'HM': 'Heard Island and McDonald Islands',
2282         'VA': 'Holy See (Vatican City State)',
2283         'HN': 'Honduras',
2284         'HK': 'Hong Kong',
2285         'HU': 'Hungary',
2286         'IS': 'Iceland',
2287         'IN': 'India',
2288         'ID': 'Indonesia',
2289         'IR': 'Iran, Islamic Republic of',
2290         'IQ': 'Iraq',
2291         'IE': 'Ireland',
2292         'IM': 'Isle of Man',
2293         'IL': 'Israel',
2294         'IT': 'Italy',
2295         'JM': 'Jamaica',
2296         'JP': 'Japan',
2297         'JE': 'Jersey',
2298         'JO': 'Jordan',
2299         'KZ': 'Kazakhstan',
2300         'KE': 'Kenya',
2301         'KI': 'Kiribati',
2302         'KP': 'Korea, Democratic People\'s Republic of',
2303         'KR': 'Korea, Republic of',
2304         'KW': 'Kuwait',
2305         'KG': 'Kyrgyzstan',
2306         'LA': 'Lao People\'s Democratic Republic',
2307         'LV': 'Latvia',
2308         'LB': 'Lebanon',
2309         'LS': 'Lesotho',
2310         'LR': 'Liberia',
2311         'LY': 'Libya',
2312         'LI': 'Liechtenstein',
2313         'LT': 'Lithuania',
2314         'LU': 'Luxembourg',
2315         'MO': 'Macao',
2316         'MK': 'Macedonia, the Former Yugoslav Republic of',
2317         'MG': 'Madagascar',
2318         'MW': 'Malawi',
2319         'MY': 'Malaysia',
2320         'MV': 'Maldives',
2321         'ML': 'Mali',
2322         'MT': 'Malta',
2323         'MH': 'Marshall Islands',
2324         'MQ': 'Martinique',
2325         'MR': 'Mauritania',
2326         'MU': 'Mauritius',
2327         'YT': 'Mayotte',
2328         'MX': 'Mexico',
2329         'FM': 'Micronesia, Federated States of',
2330         'MD': 'Moldova, Republic of',
2331         'MC': 'Monaco',
2332         'MN': 'Mongolia',
2333         'ME': 'Montenegro',
2334         'MS': 'Montserrat',
2335         'MA': 'Morocco',
2336         'MZ': 'Mozambique',
2337         'MM': 'Myanmar',
2338         'NA': 'Namibia',
2339         'NR': 'Nauru',
2340         'NP': 'Nepal',
2341         'NL': 'Netherlands',
2342         'NC': 'New Caledonia',
2343         'NZ': 'New Zealand',
2344         'NI': 'Nicaragua',
2345         'NE': 'Niger',
2346         'NG': 'Nigeria',
2347         'NU': 'Niue',
2348         'NF': 'Norfolk Island',
2349         'MP': 'Northern Mariana Islands',
2350         'NO': 'Norway',
2351         'OM': 'Oman',
2352         'PK': 'Pakistan',
2353         'PW': 'Palau',
2354         'PS': 'Palestine, State of',
2355         'PA': 'Panama',
2356         'PG': 'Papua New Guinea',
2357         'PY': 'Paraguay',
2358         'PE': 'Peru',
2359         'PH': 'Philippines',
2360         'PN': 'Pitcairn',
2361         'PL': 'Poland',
2362         'PT': 'Portugal',
2363         'PR': 'Puerto Rico',
2364         'QA': 'Qatar',
2365         'RE': 'Réunion',
2366         'RO': 'Romania',
2367         'RU': 'Russian Federation',
2368         'RW': 'Rwanda',
2369         'BL': 'Saint Barthélemy',
2370         'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2371         'KN': 'Saint Kitts and Nevis',
2372         'LC': 'Saint Lucia',
2373         'MF': 'Saint Martin (French part)',
2374         'PM': 'Saint Pierre and Miquelon',
2375         'VC': 'Saint Vincent and the Grenadines',
2376         'WS': 'Samoa',
2377         'SM': 'San Marino',
2378         'ST': 'Sao Tome and Principe',
2379         'SA': 'Saudi Arabia',
2380         'SN': 'Senegal',
2381         'RS': 'Serbia',
2382         'SC': 'Seychelles',
2383         'SL': 'Sierra Leone',
2384         'SG': 'Singapore',
2385         'SX': 'Sint Maarten (Dutch part)',
2386         'SK': 'Slovakia',
2387         'SI': 'Slovenia',
2388         'SB': 'Solomon Islands',
2389         'SO': 'Somalia',
2390         'ZA': 'South Africa',
2391         'GS': 'South Georgia and the South Sandwich Islands',
2392         'SS': 'South Sudan',
2393         'ES': 'Spain',
2394         'LK': 'Sri Lanka',
2395         'SD': 'Sudan',
2396         'SR': 'Suriname',
2397         'SJ': 'Svalbard and Jan Mayen',
2398         'SZ': 'Swaziland',
2399         'SE': 'Sweden',
2400         'CH': 'Switzerland',
2401         'SY': 'Syrian Arab Republic',
2402         'TW': 'Taiwan, Province of China',
2403         'TJ': 'Tajikistan',
2404         'TZ': 'Tanzania, United Republic of',
2405         'TH': 'Thailand',
2406         'TL': 'Timor-Leste',
2407         'TG': 'Togo',
2408         'TK': 'Tokelau',
2409         'TO': 'Tonga',
2410         'TT': 'Trinidad and Tobago',
2411         'TN': 'Tunisia',
2412         'TR': 'Turkey',
2413         'TM': 'Turkmenistan',
2414         'TC': 'Turks and Caicos Islands',
2415         'TV': 'Tuvalu',
2416         'UG': 'Uganda',
2417         'UA': 'Ukraine',
2418         'AE': 'United Arab Emirates',
2419         'GB': 'United Kingdom',
2420         'US': 'United States',
2421         'UM': 'United States Minor Outlying Islands',
2422         'UY': 'Uruguay',
2423         'UZ': 'Uzbekistan',
2424         'VU': 'Vanuatu',
2425         'VE': 'Venezuela, Bolivarian Republic of',
2426         'VN': 'Viet Nam',
2427         'VG': 'Virgin Islands, British',
2428         'VI': 'Virgin Islands, U.S.',
2429         'WF': 'Wallis and Futuna',
2430         'EH': 'Western Sahara',
2431         'YE': 'Yemen',
2432         'ZM': 'Zambia',
2433         'ZW': 'Zimbabwe',
2434     }
2435
2436     @classmethod
2437     def short2full(cls, code):
2438         """Convert an ISO 3166-2 country code to the corresponding full name"""
2439         return cls._country_map.get(code.upper())
2440
2441
2442 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2443     def __init__(self, proxies=None):
2444         # Set default handlers
2445         for type in ('http', 'https'):
2446             setattr(self, '%s_open' % type,
2447                     lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2448                         meth(r, proxy, type))
2449         return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2450
2451     def proxy_open(self, req, proxy, type):
2452         req_proxy = req.headers.get('Ytdl-request-proxy')
2453         if req_proxy is not None:
2454             proxy = req_proxy
2455             del req.headers['Ytdl-request-proxy']
2456
2457         if proxy == '__noproxy__':
2458             return None  # No Proxy
2459         return compat_urllib_request.ProxyHandler.proxy_open(
2460             self, req, proxy, type)