youtube_dl/compat.py

   1 from __future__ import unicode_literals
   2
   3 import collections
   4 import getpass
   5 import optparse
   6 import os
   7 import re
   8 import shutil
   9 import socket
  10 import subprocess
  11 import sys
  12 import itertools
  13
  14
  15 try:
  16     import urllib.request as compat_urllib_request
  17 except ImportError:  # Python 2
  18     import urllib2 as compat_urllib_request
  19
  20 try:
  21     import urllib.error as compat_urllib_error
  22 except ImportError:  # Python 2
  23     import urllib2 as compat_urllib_error
  24
  25 try:
  26     import urllib.parse as compat_urllib_parse
  27 except ImportError:  # Python 2
  28     import urllib as compat_urllib_parse
  29
  30 try:
  31     from urllib.parse import urlparse as compat_urllib_parse_urlparse
  32 except ImportError:  # Python 2
  33     from urlparse import urlparse as compat_urllib_parse_urlparse
  34
  35 try:
  36     import urllib.parse as compat_urlparse
  37 except ImportError:  # Python 2
  38     import urlparse as compat_urlparse
  39
  40 try:
  41     import http.cookiejar as compat_cookiejar
  42 except ImportError:  # Python 2
  43     import cookielib as compat_cookiejar
  44
  45 try:
  46     import http.cookies as compat_cookies
  47 except ImportError:  # Python 2
  48     import Cookie as compat_cookies
  49
  50 try:
  51     import html.entities as compat_html_entities
  52 except ImportError:  # Python 2
  53     import htmlentitydefs as compat_html_entities
  54
  55 try:
  56     import http.client as compat_http_client
  57 except ImportError:  # Python 2
  58     import httplib as compat_http_client
  59
  60 try:
  61     from urllib.error import HTTPError as compat_HTTPError
  62 except ImportError:  # Python 2
  63     from urllib2 import HTTPError as compat_HTTPError
  64
  65 try:
  66     from urllib.request import urlretrieve as compat_urlretrieve
  67 except ImportError:  # Python 2
  68     from urllib import urlretrieve as compat_urlretrieve
  69
  70
  71 try:
  72     from subprocess import DEVNULL
  73     compat_subprocess_get_DEVNULL = lambda: DEVNULL
  74 except ImportError:
  75     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
  76
  77 try:
  78     import http.server as compat_http_server
  79 except ImportError:
  80     import BaseHTTPServer as compat_http_server
  81
  82 try:
  83     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
  84     from urllib.parse import unquote as compat_urllib_parse_unquote
  85     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
  86 except ImportError:  # Python 2
  87     _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
  88                 else re.compile('([\x00-\x7f]+)'))
  89
  90     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
  91     # implementations from cpython 3.4.3's stdlib. Python 2's version
  92     # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
  93
  94     def compat_urllib_parse_unquote_to_bytes(string):
  95         """unquote_to_bytes('abc%20def') -> b'abc def'."""
  96         # Note: strings are encoded as UTF-8. This is only an issue if it contains
  97         # unescaped non-ASCII characters, which URIs should not.
  98         if not string:
  99             # Is it a string-like object?
 100             string.split
 101             return b''
 102         if isinstance(string, unicode):
 103             string = string.encode('utf-8')
 104         bits = string.split(b'%')
 105         if len(bits) == 1:
 106             return string
 107         res = [bits[0]]
 108         append = res.append
 109         for item in bits[1:]:
 110             try:
 111                 append(compat_urllib_parse._hextochr[item[:2]])
 112                 append(item[2:])
 113             except KeyError:
 114                 append(b'%')
 115                 append(item)
 116         return b''.join(res)
 117
 118     def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
 119         """Replace %xx escapes by their single-character equivalent. The optional
 120         encoding and errors parameters specify how to decode percent-encoded
 121         sequences into Unicode characters, as accepted by the bytes.decode()
 122         method.
 123         By default, percent-encoded sequences are decoded with UTF-8, and invalid
 124         sequences are replaced by a placeholder character.
 125
 126         unquote('abc%20def') -> 'abc def'.
 127         """
 128         if '%' not in string:
 129             string.split
 130             return string
 131         if encoding is None:
 132             encoding = 'utf-8'
 133         if errors is None:
 134             errors = 'replace'
 135         bits = _asciire.split(string)
 136         res = [bits[0]]
 137         append = res.append
 138         for i in range(1, len(bits), 2):
 139             append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
 140             append(bits[i + 1])
 141         return ''.join(res)
 142
 143     def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
 144         """Like unquote(), but also replace plus signs by spaces, as required for
 145         unquoting HTML form values.
 146
 147         unquote_plus('%7e/abc+def') -> '~/abc def'
 148         """
 149         string = string.replace('+', ' ')
 150         return compat_urllib_parse_unquote(string, encoding, errors)
 151
 152 try:
 153     compat_str = unicode  # Python 2
 154 except NameError:
 155     compat_str = str
 156
 157 try:
 158     compat_basestring = basestring  # Python 2
 159 except NameError:
 160     compat_basestring = str
 161
 162 try:
 163     compat_chr = unichr  # Python 2
 164 except NameError:
 165     compat_chr = chr
 166
 167 try:
 168     from xml.etree.ElementTree import ParseError as compat_xml_parse_error
 169 except ImportError:  # Python 2.6
 170     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 171
 172
 173 try:
 174     from urllib.parse import parse_qs as compat_parse_qs
 175 except ImportError:  # Python 2
 176     # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
 177     # Python 2's version is apparently totally broken
 178
 179     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
 180                    encoding='utf-8', errors='replace'):
 181         qs, _coerce_result = qs, compat_str
 182         pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
 183         r = []
 184         for name_value in pairs:
 185             if not name_value and not strict_parsing:
 186                 continue
 187             nv = name_value.split('=', 1)
 188             if len(nv) != 2:
 189                 if strict_parsing:
 190                     raise ValueError("bad query field: %r" % (name_value,))
 191                 # Handle case of a control-name with no equal sign
 192                 if keep_blank_values:
 193                     nv.append('')
 194                 else:
 195                     continue
 196             if len(nv[1]) or keep_blank_values:
 197                 name = nv[0].replace('+', ' ')
 198                 name = compat_urllib_parse_unquote(
 199                     name, encoding=encoding, errors=errors)
 200                 name = _coerce_result(name)
 201                 value = nv[1].replace('+', ' ')
 202                 value = compat_urllib_parse_unquote(
 203                     value, encoding=encoding, errors=errors)
 204                 value = _coerce_result(value)
 205                 r.append((name, value))
 206         return r
 207
 208     def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
 209                         encoding='utf-8', errors='replace'):
 210         parsed_result = {}
 211         pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
 212                            encoding=encoding, errors=errors)
 213         for name, value in pairs:
 214             if name in parsed_result:
 215                 parsed_result[name].append(value)
 216             else:
 217                 parsed_result[name] = [value]
 218         return parsed_result
 219
 220 try:
 221     from shlex import quote as shlex_quote
 222 except ImportError:  # Python < 3.3
 223     def shlex_quote(s):
 224         if re.match(r'^[-_\w./]+$', s):
 225             return s
 226         else:
 227             return "'" + s.replace("'", "'\"'\"'") + "'"
 228
 229
 230 def compat_ord(c):
 231     if type(c) is int:
 232         return c
 233     else:
 234         return ord(c)
 235
 236
 237 if sys.version_info >= (3, 0):
 238     compat_getenv = os.getenv
 239     compat_expanduser = os.path.expanduser
 240 else:
 241     # Environment variables should be decoded with filesystem encoding.
 242     # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
 243
 244     def compat_getenv(key, default=None):
 245         from .utils import get_filesystem_encoding
 246         env = os.getenv(key, default)
 247         if env:
 248             env = env.decode(get_filesystem_encoding())
 249         return env
 250
 251     # HACK: The default implementations of os.path.expanduser from cpython do not decode
 252     # environment variables with filesystem encoding. We will work around this by
 253     # providing adjusted implementations.
 254     # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
 255     # for different platforms with correct environment variables decoding.
 256
 257     if os.name == 'posix':
 258         def compat_expanduser(path):
 259             """Expand ~ and ~user constructions.  If user or $HOME is unknown,
 260             do nothing."""
 261             if not path.startswith('~'):
 262                 return path
 263             i = path.find('/', 1)
 264             if i < 0:
 265                 i = len(path)
 266             if i == 1:
 267                 if 'HOME' not in os.environ:
 268                     import pwd
 269                     userhome = pwd.getpwuid(os.getuid()).pw_dir
 270                 else:
 271                     userhome = compat_getenv('HOME')
 272             else:
 273                 import pwd
 274                 try:
 275                     pwent = pwd.getpwnam(path[1:i])
 276                 except KeyError:
 277                     return path
 278                 userhome = pwent.pw_dir
 279             userhome = userhome.rstrip('/')
 280             return (userhome + path[i:]) or '/'
 281     elif os.name == 'nt' or os.name == 'ce':
 282         def compat_expanduser(path):
 283             """Expand ~ and ~user constructs.
 284
 285             If user or $HOME is unknown, do nothing."""
 286             if path[:1] != '~':
 287                 return path
 288             i, n = 1, len(path)
 289             while i < n and path[i] not in '/\\':
 290                 i = i + 1
 291
 292             if 'HOME' in os.environ:
 293                 userhome = compat_getenv('HOME')
 294             elif 'USERPROFILE' in os.environ:
 295                 userhome = compat_getenv('USERPROFILE')
 296             elif 'HOMEPATH' not in os.environ:
 297                 return path
 298             else:
 299                 try:
 300                     drive = compat_getenv('HOMEDRIVE')
 301                 except KeyError:
 302                     drive = ''
 303                 userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
 304
 305             if i != 1:  # ~user
 306                 userhome = os.path.join(os.path.dirname(userhome), path[1:i])
 307
 308             return userhome + path[i:]
 309     else:
 310         compat_expanduser = os.path.expanduser
 311
 312
 313 if sys.version_info < (3, 0):
 314     def compat_print(s):
 315         from .utils import preferredencoding
 316         print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
 317 else:
 318     def compat_print(s):
 319         assert isinstance(s, compat_str)
 320         print(s)
 321
 322
 323 try:
 324     subprocess_check_output = subprocess.check_output
 325 except AttributeError:
 326     def subprocess_check_output(*args, **kwargs):
 327         assert 'input' not in kwargs
 328         p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
 329         output, _ = p.communicate()
 330         ret = p.poll()
 331         if ret:
 332             raise subprocess.CalledProcessError(ret, p.args, output=output)
 333         return output
 334
 335 if sys.version_info < (3, 0) and sys.platform == 'win32':
 336     def compat_getpass(prompt, *args, **kwargs):
 337         if isinstance(prompt, compat_str):
 338             from .utils import preferredencoding
 339             prompt = prompt.encode(preferredencoding())
 340         return getpass.getpass(prompt, *args, **kwargs)
 341 else:
 342     compat_getpass = getpass.getpass
 343
 344 # Old 2.6 and 2.7 releases require kwargs to be bytes
 345 try:
 346     def _testfunc(x):
 347         pass
 348     _testfunc(**{'x': 0})
 349 except TypeError:
 350     def compat_kwargs(kwargs):
 351         return dict((bytes(k), v) for k, v in kwargs.items())
 352 else:
 353     compat_kwargs = lambda kwargs: kwargs
 354
 355
 356 if sys.version_info < (2, 7):
 357     def compat_socket_create_connection(address, timeout, source_address=None):
 358         host, port = address
 359         err = None
 360         for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
 361             af, socktype, proto, canonname, sa = res
 362             sock = None
 363             try:
 364                 sock = socket.socket(af, socktype, proto)
 365                 sock.settimeout(timeout)
 366                 if source_address:
 367                     sock.bind(source_address)
 368                 sock.connect(sa)
 369                 return sock
 370             except socket.error as _:
 371                 err = _
 372                 if sock is not None:
 373                     sock.close()
 374         if err is not None:
 375             raise err
 376         else:
 377             raise socket.error("getaddrinfo returns an empty list")
 378 else:
 379     compat_socket_create_connection = socket.create_connection
 380
 381
 382 # Fix https://github.com/rg3/youtube-dl/issues/4223
 383 # See http://bugs.python.org/issue9161 for what is broken
 384 def workaround_optparse_bug9161():
 385     op = optparse.OptionParser()
 386     og = optparse.OptionGroup(op, 'foo')
 387     try:
 388         og.add_option('-t')
 389     except TypeError:
 390         real_add_option = optparse.OptionGroup.add_option
 391
 392         def _compat_add_option(self, *args, **kwargs):
 393             enc = lambda v: (
 394                 v.encode('ascii', 'replace') if isinstance(v, compat_str)
 395                 else v)
 396             bargs = [enc(a) for a in args]
 397             bkwargs = dict(
 398                 (k, enc(v)) for k, v in kwargs.items())
 399             return real_add_option(self, *bargs, **bkwargs)
 400         optparse.OptionGroup.add_option = _compat_add_option
 401
 402 if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
 403     compat_get_terminal_size = shutil.get_terminal_size
 404 else:
 405     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 406
 407     def compat_get_terminal_size():
 408         columns = compat_getenv('COLUMNS', None)
 409         if columns:
 410             columns = int(columns)
 411         else:
 412             columns = None
 413         lines = compat_getenv('LINES', None)
 414         if lines:
 415             lines = int(lines)
 416         else:
 417             lines = None
 418
 419         try:
 420             sp = subprocess.Popen(
 421                 ['stty', 'size'],
 422                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 423             out, err = sp.communicate()
 424             lines, columns = map(int, out.split())
 425         except Exception:
 426             pass
 427         return _terminal_size(columns, lines)
 428
 429 try:
 430     itertools.count(start=0, step=1)
 431     compat_itertools_count = itertools.count
 432 except TypeError:  # Python 2.6
 433     def compat_itertools_count(start=0, step=1):
 434         n = start
 435         while True:
 436             yield n
 437             n += step
 438
 439 __all__ = [
 440     'compat_HTTPError',
 441     'compat_basestring',
 442     'compat_chr',
 443     'compat_cookiejar',
 444     'compat_cookies',
 445     'compat_expanduser',
 446     'compat_get_terminal_size',
 447     'compat_getenv',
 448     'compat_getpass',
 449     'compat_html_entities',
 450     'compat_http_client',
 451     'compat_http_server',
 452     'compat_itertools_count',
 453     'compat_kwargs',
 454     'compat_ord',
 455     'compat_parse_qs',
 456     'compat_print',
 457     'compat_socket_create_connection',
 458     'compat_str',
 459     'compat_subprocess_get_DEVNULL',
 460     'compat_urllib_error',
 461     'compat_urllib_parse',
 462     'compat_urllib_parse_unquote',
 463     'compat_urllib_parse_unquote_plus',
 464     'compat_urllib_parse_unquote_to_bytes',
 465     'compat_urllib_parse_urlparse',
 466     'compat_urllib_request',
 467     'compat_urlparse',
 468     'compat_urlretrieve',
 469     'compat_xml_parse_error',
 470     'shlex_quote',
 471     'subprocess_check_output',
 472     'workaround_optparse_bug9161',
 473 ]