youtube_dl/utils.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import gzip
   5 import io
   6 import json
   7 import locale
   8 import os
   9 import re
  10 import sys
  11 import traceback
  12 import zlib
  13 import email.utils
  14 import json
  15
  16 try:
  17     import urllib.request as compat_urllib_request
  18 except ImportError: # Python 2
  19     import urllib2 as compat_urllib_request
  20
  21 try:
  22     import urllib.error as compat_urllib_error
  23 except ImportError: # Python 2
  24     import urllib2 as compat_urllib_error
  25
  26 try:
  27     import urllib.parse as compat_urllib_parse
  28 except ImportError: # Python 2
  29     import urllib as compat_urllib_parse
  30
  31 try:
  32     from urllib.parse import urlparse as compat_urllib_parse_urlparse
  33 except ImportError: # Python 2
  34     from urlparse import urlparse as compat_urllib_parse_urlparse
  35
  36 try:
  37     import http.cookiejar as compat_cookiejar
  38 except ImportError: # Python 2
  39     import cookielib as compat_cookiejar
  40
  41 try:
  42     import html.entities as compat_html_entities
  43 except ImportError: # Python 2
  44     import htmlentitydefs as compat_html_entities
  45
  46 try:
  47     import html.parser as compat_html_parser
  48 except ImportError: # Python 2
  49     import HTMLParser as compat_html_parser
  50
  51 try:
  52     import http.client as compat_http_client
  53 except ImportError: # Python 2
  54     import httplib as compat_http_client
  55
  56 try:
  57     from subprocess import DEVNULL
  58     compat_subprocess_get_DEVNULL = lambda: DEVNULL
  59 except ImportError:
  60     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
  61
  62 try:
  63     from urllib.parse import parse_qs as compat_parse_qs
  64 except ImportError: # Python 2
  65     # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
  66     # Python 2's version is apparently totally broken
  67     def _unquote(string, encoding='utf-8', errors='replace'):
  68         if string == '':
  69             return string
  70         res = string.split('%')
  71         if len(res) == 1:
  72             return string
  73         if encoding is None:
  74             encoding = 'utf-8'
  75         if errors is None:
  76             errors = 'replace'
  77         # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
  78         pct_sequence = b''
  79         string = res[0]
  80         for item in res[1:]:
  81             try:
  82                 if not item:
  83                     raise ValueError
  84                 pct_sequence += item[:2].decode('hex')
  85                 rest = item[2:]
  86                 if not rest:
  87                     # This segment was just a single percent-encoded character.
  88                     # May be part of a sequence of code units, so delay decoding.
  89                     # (Stored in pct_sequence).
  90                     continue
  91             except ValueError:
  92                 rest = '%' + item
  93             # Encountered non-percent-encoded characters. Flush the current
  94             # pct_sequence.
  95             string += pct_sequence.decode(encoding, errors) + rest
  96             pct_sequence = b''
  97         if pct_sequence:
  98             # Flush the final pct_sequence
  99             string += pct_sequence.decode(encoding, errors)
 100         return string
 101
 102     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
 103                 encoding='utf-8', errors='replace'):
 104         qs, _coerce_result = qs, unicode
 105         pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
 106         r = []
 107         for name_value in pairs:
 108             if not name_value and not strict_parsing:
 109                 continue
 110             nv = name_value.split('=', 1)
 111             if len(nv) != 2:
 112                 if strict_parsing:
 113                     raise ValueError("bad query field: %r" % (name_value,))
 114                 # Handle case of a control-name with no equal sign
 115                 if keep_blank_values:
 116                     nv.append('')
 117                 else:
 118                     continue
 119             if len(nv[1]) or keep_blank_values:
 120                 name = nv[0].replace('+', ' ')
 121                 name = _unquote(name, encoding=encoding, errors=errors)
 122                 name = _coerce_result(name)
 123                 value = nv[1].replace('+', ' ')
 124                 value = _unquote(value, encoding=encoding, errors=errors)
 125                 value = _coerce_result(value)
 126                 r.append((name, value))
 127         return r
 128
 129     def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
 130                 encoding='utf-8', errors='replace'):
 131         parsed_result = {}
 132         pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
 133                         encoding=encoding, errors=errors)
 134         for name, value in pairs:
 135             if name in parsed_result:
 136                 parsed_result[name].append(value)
 137             else:
 138                 parsed_result[name] = [value]
 139         return parsed_result
 140
 141 try:
 142     compat_str = unicode # Python 2
 143 except NameError:
 144     compat_str = str
 145
 146 try:
 147     compat_chr = unichr # Python 2
 148 except NameError:
 149     compat_chr = chr
 150
 151 std_headers = {
 152     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
 153     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
 154     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 155     'Accept-Encoding': 'gzip, deflate',
 156     'Accept-Language': 'en-us,en;q=0.5',
 157 }
 158
 159 def preferredencoding():
 160     """Get preferred encoding.
 161
 162     Returns the best encoding scheme for the system, based on
 163     locale.getpreferredencoding() and some further tweaks.
 164     """
 165     try:
 166         pref = locale.getpreferredencoding()
 167         u'TEST'.encode(pref)
 168     except:
 169         pref = 'UTF-8'
 170
 171     return pref
 172
 173 if sys.version_info < (3,0):
 174     def compat_print(s):
 175         print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
 176 else:
 177     def compat_print(s):
 178         assert type(s) == type(u'')
 179         print(s)
 180
 181 # In Python 2.x, json.dump expects a bytestream.
 182 # In Python 3.x, it writes to a character stream
 183 if sys.version_info < (3,0):
 184     def write_json_file(obj, fn):
 185         with open(fn, 'wb') as f:
 186             json.dump(obj, f)
 187 else:
 188     def write_json_file(obj, fn):
 189         with open(fn, 'w', encoding='utf-8') as f:
 190             json.dump(obj, f)
 191
 192 def htmlentity_transform(matchobj):
 193     """Transforms an HTML entity to a character.
 194
 195     This function receives a match object and is intended to be used with
 196     the re.sub() function.
 197     """
 198     entity = matchobj.group(1)
 199
 200     # Known non-numeric HTML entity
 201     if entity in compat_html_entities.name2codepoint:
 202         return compat_chr(compat_html_entities.name2codepoint[entity])
 203
 204     mobj = re.match(u'(?u)#(x?\\d+)', entity)
 205     if mobj is not None:
 206         numstr = mobj.group(1)
 207         if numstr.startswith(u'x'):
 208             base = 16
 209             numstr = u'0%s' % numstr
 210         else:
 211             base = 10
 212         return compat_chr(int(numstr, base))
 213
 214     # Unknown entity in name, return its literal representation
 215     return (u'&%s;' % entity)
 216
 217 compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
 218 class AttrParser(compat_html_parser.HTMLParser):
 219     """Modified HTMLParser that isolates a tag with the specified attribute"""
 220     def __init__(self, attribute, value):
 221         self.attribute = attribute
 222         self.value = value
 223         self.result = None
 224         self.started = False
 225         self.depth = {}
 226         self.html = None
 227         self.watch_startpos = False
 228         self.error_count = 0
 229         compat_html_parser.HTMLParser.__init__(self)
 230
 231     def error(self, message):
 232         if self.error_count > 10 or self.started:
 233             raise compat_html_parser.HTMLParseError(message, self.getpos())
 234         self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
 235         self.error_count += 1
 236         self.goahead(1)
 237
 238     def loads(self, html):
 239         self.html = html
 240         self.feed(html)
 241         self.close()
 242
 243     def handle_starttag(self, tag, attrs):
 244         attrs = dict(attrs)
 245         if self.started:
 246             self.find_startpos(None)
 247         if self.attribute in attrs and attrs[self.attribute] == self.value:
 248             self.result = [tag]
 249             self.started = True
 250             self.watch_startpos = True
 251         if self.started:
 252             if not tag in self.depth: self.depth[tag] = 0
 253             self.depth[tag] += 1
 254
 255     def handle_endtag(self, tag):
 256         if self.started:
 257             if tag in self.depth: self.depth[tag] -= 1
 258             if self.depth[self.result[0]] == 0:
 259                 self.started = False
 260                 self.result.append(self.getpos())
 261
 262     def find_startpos(self, x):
 263         """Needed to put the start position of the result (self.result[1])
 264         after the opening tag with the requested id"""
 265         if self.watch_startpos:
 266             self.watch_startpos = False
 267             self.result.append(self.getpos())
 268     handle_entityref = handle_charref = handle_data = handle_comment = \
 269     handle_decl = handle_pi = unknown_decl = find_startpos
 270
 271     def get_result(self):
 272         if self.result is None:
 273             return None
 274         if len(self.result) != 3:
 275             return None
 276         lines = self.html.split('\n')
 277         lines = lines[self.result[1][0]-1:self.result[2][0]]
 278         lines[0] = lines[0][self.result[1][1]:]
 279         if len(lines) == 1:
 280             lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
 281         lines[-1] = lines[-1][:self.result[2][1]]
 282         return '\n'.join(lines).strip()
 283
 284 def get_element_by_id(id, html):
 285     """Return the content of the tag with the specified ID in the passed HTML document"""
 286     return get_element_by_attribute("id", id, html)
 287
 288 def get_element_by_attribute(attribute, value, html):
 289     """Return the content of the tag with the specified attribute in the passed HTML document"""
 290     parser = AttrParser(attribute, value)
 291     try:
 292         parser.loads(html)
 293     except compat_html_parser.HTMLParseError:
 294         pass
 295     return parser.get_result()
 296
 297
 298 def clean_html(html):
 299     """Clean an HTML snippet into a readable string"""
 300     # Newline vs <br />
 301     html = html.replace('\n', ' ')
 302     html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
 303     html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
 304     # Strip html tags
 305     html = re.sub('<.*?>', '', html)
 306     # Replace html entities
 307     html = unescapeHTML(html)
 308     return html
 309
 310
 311 def sanitize_open(filename, open_mode):
 312     """Try to open the given filename, and slightly tweak it if this fails.
 313
 314     Attempts to open the given filename. If this fails, it tries to change
 315     the filename slightly, step by step, until it's either able to open it
 316     or it fails and raises a final exception, like the standard open()
 317     function.
 318
 319     It returns the tuple (stream, definitive_file_name).
 320     """
 321     try:
 322         if filename == u'-':
 323             if sys.platform == 'win32':
 324                 import msvcrt
 325                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 326             return (sys.stdout, filename)
 327         stream = open(encodeFilename(filename), open_mode)
 328         return (stream, filename)
 329     except (IOError, OSError) as err:
 330         # In case of error, try to remove win32 forbidden chars
 331         filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
 332
 333         # An exception here should be caught in the caller
 334         stream = open(encodeFilename(filename), open_mode)
 335         return (stream, filename)
 336
 337
 338 def timeconvert(timestr):
 339     """Convert RFC 2822 defined time string into system timestamp"""
 340     timestamp = None
 341     timetuple = email.utils.parsedate_tz(timestr)
 342     if timetuple is not None:
 343         timestamp = email.utils.mktime_tz(timetuple)
 344     return timestamp
 345
 346 def sanitize_filename(s, restricted=False, is_id=False):
 347     """Sanitizes a string so it could be used as part of a filename.
 348     If restricted is set, use a stricter subset of allowed characters.
 349     Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
 350     """
 351     def replace_insane(char):
 352         if char == '?' or ord(char) < 32 or ord(char) == 127:
 353             return ''
 354         elif char == '"':
 355             return '' if restricted else '\''
 356         elif char == ':':
 357             return '_-' if restricted else ' -'
 358         elif char in '\\/|*<>':
 359             return '_'
 360         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
 361             return '_'
 362         if restricted and ord(char) > 127:
 363             return '_'
 364         return char
 365
 366     result = u''.join(map(replace_insane, s))
 367     if not is_id:
 368         while '__' in result:
 369             result = result.replace('__', '_')
 370         result = result.strip('_')
 371         # Common case of "Foreign band name - English song title"
 372         if restricted and result.startswith('-_'):
 373             result = result[2:]
 374         if not result:
 375             result = '_'
 376     return result
 377
 378 def orderedSet(iterable):
 379     """ Remove all duplicates from the input iterable """
 380     res = []
 381     for el in iterable:
 382         if el not in res:
 383             res.append(el)
 384     return res
 385
 386 def unescapeHTML(s):
 387     """
 388     @param s a string
 389     """
 390     assert type(s) == type(u'')
 391
 392     result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
 393     return result
 394
 395 def encodeFilename(s):
 396     """
 397     @param s The name of the file
 398     """
 399
 400     assert type(s) == type(u'')
 401
 402     # Python 3 has a Unicode API
 403     if sys.version_info >= (3, 0):
 404         return s
 405
 406     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
 407         # Pass u'' directly to use Unicode APIs on Windows 2000 and up
 408         # (Detecting Windows NT 4 is tricky because 'major >= 4' would
 409         # match Windows 9x series as well. Besides, NT 4 is obsolete.)
 410         return s
 411     else:
 412         return s.encode(sys.getfilesystemencoding(), 'ignore')
 413
 414
 415 class ExtractorError(Exception):
 416     """Error during info extraction."""
 417     def __init__(self, msg, tb=None):
 418         """ tb, if given, is the original traceback (so that it can be printed out). """
 419         super(ExtractorError, self).__init__(msg)
 420         self.traceback = tb
 421
 422     def format_traceback(self):
 423         if self.traceback is None:
 424             return None
 425         return u''.join(traceback.format_tb(self.traceback))
 426
 427
 428 class DownloadError(Exception):
 429     """Download Error exception.
 430
 431     This exception may be thrown by FileDownloader objects if they are not
 432     configured to continue on errors. They will contain the appropriate
 433     error message.
 434     """
 435     pass
 436
 437
 438 class SameFileError(Exception):
 439     """Same File exception.
 440
 441     This exception will be thrown by FileDownloader objects if they detect
 442     multiple files would have to be downloaded to the same file on disk.
 443     """
 444     pass
 445
 446
 447 class PostProcessingError(Exception):
 448     """Post Processing exception.
 449
 450     This exception may be raised by PostProcessor's .run() method to
 451     indicate an error in the postprocessing task.
 452     """
 453     pass
 454
 455 class MaxDownloadsReached(Exception):
 456     """ --max-downloads limit has been reached. """
 457     pass
 458
 459
 460 class UnavailableVideoError(Exception):
 461     """Unavailable Format exception.
 462
 463     This exception will be thrown when a video is requested
 464     in a format that is not available for that video.
 465     """
 466     pass
 467
 468
 469 class ContentTooShortError(Exception):
 470     """Content Too Short exception.
 471
 472     This exception may be raised by FileDownloader objects when a file they
 473     download is too small for what the server announced first, indicating
 474     the connection was probably interrupted.
 475     """
 476     # Both in bytes
 477     downloaded = None
 478     expected = None
 479
 480     def __init__(self, downloaded, expected):
 481         self.downloaded = downloaded
 482         self.expected = expected
 483
 484 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 485     """Handler for HTTP requests and responses.
 486
 487     This class, when installed with an OpenerDirector, automatically adds
 488     the standard headers to every HTTP request and handles gzipped and
 489     deflated responses from web servers. If compression is to be avoided in
 490     a particular request, the original request in the program code only has
 491     to include the HTTP header "Youtubedl-No-Compression", which will be
 492     removed before making the real request.
 493
 494     Part of this code was copied from:
 495
 496     http://techknack.net/python-urllib2-handlers/
 497
 498     Andrew Rowls, the author of that code, agreed to release it to the
 499     public domain.
 500     """
 501
 502     @staticmethod
 503     def deflate(data):
 504         try:
 505             return zlib.decompress(data, -zlib.MAX_WBITS)
 506         except zlib.error:
 507             return zlib.decompress(data)
 508
 509     @staticmethod
 510     def addinfourl_wrapper(stream, headers, url, code):
 511         if hasattr(compat_urllib_request.addinfourl, 'getcode'):
 512             return compat_urllib_request.addinfourl(stream, headers, url, code)
 513         ret = compat_urllib_request.addinfourl(stream, headers, url)
 514         ret.code = code
 515         return ret
 516
 517     def http_request(self, req):
 518         for h in std_headers:
 519             if h in req.headers:
 520                 del req.headers[h]
 521             req.add_header(h, std_headers[h])
 522         if 'Youtubedl-no-compression' in req.headers:
 523             if 'Accept-encoding' in req.headers:
 524                 del req.headers['Accept-encoding']
 525             del req.headers['Youtubedl-no-compression']
 526         return req
 527
 528     def http_response(self, req, resp):
 529         old_resp = resp
 530         # gzip
 531         if resp.headers.get('Content-encoding', '') == 'gzip':
 532             gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
 533             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 534             resp.msg = old_resp.msg
 535         # deflate
 536         if resp.headers.get('Content-encoding', '') == 'deflate':
 537             gz = io.BytesIO(self.deflate(resp.read()))
 538             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
 539             resp.msg = old_resp.msg
 540         return resp
 541
 542     https_request = http_request
 543     https_response = http_response