[yt-dlp.git] / youtube_dl / compat.py

import getpass
import os
import subprocess
import sys


try:
    import urllib.request as compat_urllib_request
except ImportError: # Python 2
    import urllib2 as compat_urllib_request

try:
    import urllib.error as compat_urllib_error
except ImportError: # Python 2
    import urllib2 as compat_urllib_error

try:
    import urllib.parse as compat_urllib_parse
except ImportError: # Python 2
    import urllib as compat_urllib_parse

try:
    from urllib.parse import urlparse as compat_urllib_parse_urlparse
except ImportError: # Python 2
    from urlparse import urlparse as compat_urllib_parse_urlparse

try:
    import urllib.parse as compat_urlparse
except ImportError: # Python 2
    import urlparse as compat_urlparse

try:
    import http.cookiejar as compat_cookiejar
except ImportError: # Python 2
    import cookielib as compat_cookiejar

try:
    import html.entities as compat_html_entities
except ImportError: # Python 2
    import htmlentitydefs as compat_html_entities

try:
    import html.parser as compat_html_parser
except ImportError: # Python 2
    import HTMLParser as compat_html_parser

try:
    import http.client as compat_http_client
except ImportError: # Python 2
    import httplib as compat_http_client

try:
    from urllib.error import HTTPError as compat_HTTPError
except ImportError:  # Python 2
    from urllib2 import HTTPError as compat_HTTPError

try:
    from urllib.request import urlretrieve as compat_urlretrieve
except ImportError:  # Python 2
    from urllib import urlretrieve as compat_urlretrieve


try:
    from subprocess import DEVNULL
    compat_subprocess_get_DEVNULL = lambda: DEVNULL
except ImportError:
    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')

try:
    from urllib.parse import unquote as compat_urllib_parse_unquote
except ImportError:
    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
        if string == '':
            return string
        res = string.split('%')
        if len(res) == 1:
            return string
        if encoding is None:
            encoding = 'utf-8'
        if errors is None:
            errors = 'replace'
        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
        pct_sequence = b''
        string = res[0]
        for item in res[1:]:
            try:
                if not item:
                    raise ValueError
                pct_sequence += item[:2].decode('hex')
                rest = item[2:]
                if not rest:
                    # This segment was just a single percent-encoded character.
                    # May be part of a sequence of code units, so delay decoding.
                    # (Stored in pct_sequence).
                    continue
            except ValueError:
                rest = '%' + item
            # Encountered non-percent-encoded characters. Flush the current
            # pct_sequence.
            string += pct_sequence.decode(encoding, errors) + rest
            pct_sequence = b''
        if pct_sequence:
            # Flush the final pct_sequence
            string += pct_sequence.decode(encoding, errors)
        return string


try:
    from urllib.parse import parse_qs as compat_parse_qs
except ImportError: # Python 2
    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
    # Python 2's version is apparently totally broken

    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                encoding='utf-8', errors='replace'):
        qs, _coerce_result = qs, unicode
        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
        r = []
        for name_value in pairs:
            if not name_value and not strict_parsing:
                continue
            nv = name_value.split('=', 1)
            if len(nv) != 2:
                if strict_parsing:
                    raise ValueError("bad query field: %r" % (name_value,))
                # Handle case of a control-name with no equal sign
                if keep_blank_values:
                    nv.append('')
                else:
                    continue
            if len(nv[1]) or keep_blank_values:
                name = nv[0].replace('+', ' ')
                name = compat_urllib_parse_unquote(
                    name, encoding=encoding, errors=errors)
                name = _coerce_result(name)
                value = nv[1].replace('+', ' ')
                value = compat_urllib_parse_unquote(
                    value, encoding=encoding, errors=errors)
                value = _coerce_result(value)
                r.append((name, value))
        return r

    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
                encoding='utf-8', errors='replace'):
        parsed_result = {}
        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
                        encoding=encoding, errors=errors)
        for name, value in pairs:
            if name in parsed_result:
                parsed_result[name].append(value)
            else:
                parsed_result[name] = [value]
        return parsed_result

try:
    compat_str = unicode # Python 2
except NameError:
    compat_str = str

try:
    compat_chr = unichr # Python 2
except NameError:
    compat_chr = chr

try:
    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
except ImportError:  # Python 2.6
    from xml.parsers.expat import ExpatError as compat_xml_parse_error

try:
    from shlex import quote as shlex_quote
except ImportError:  # Python < 3.3
    def shlex_quote(s):
        return "'" + s.replace("'", "'\"'\"'") + "'"


def compat_ord(c):
    if type(c) is int: return c
    else: return ord(c)


if sys.version_info >= (3, 0):
    compat_getenv = os.getenv
    compat_expanduser = os.path.expanduser
else:
    # Environment variables should be decoded with filesystem encoding.
    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)

    def compat_getenv(key, default=None):
        from .utils import get_filesystem_encoding
        env = os.getenv(key, default)
        if env:
            env = env.decode(get_filesystem_encoding())
        return env

    # HACK: The default implementations of os.path.expanduser from cpython do not decode
    # environment variables with filesystem encoding. We will work around this by
    # providing adjusted implementations.
    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
    # for different platforms with correct environment variables decoding.

    if os.name == 'posix':
        def compat_expanduser(path):
            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
            do nothing."""
            if not path.startswith('~'):
                return path
            i = path.find('/', 1)
            if i < 0:
                i = len(path)
            if i == 1:
                if 'HOME' not in os.environ:
                    import pwd
                    userhome = pwd.getpwuid(os.getuid()).pw_dir
                else:
                    userhome = compat_getenv('HOME')
            else:
                import pwd
                try:
                    pwent = pwd.getpwnam(path[1:i])
                except KeyError:
                    return path
                userhome = pwent.pw_dir
            userhome = userhome.rstrip('/')
            return (userhome + path[i:]) or '/'
    elif os.name == 'nt' or os.name == 'ce':
        def compat_expanduser(path):
            """Expand ~ and ~user constructs.

            If user or $HOME is unknown, do nothing."""
            if path[:1] != '~':
                return path
            i, n = 1, len(path)
            while i < n and path[i] not in '/\\':
                i = i + 1

            if 'HOME' in os.environ:
                userhome = compat_getenv('HOME')
            elif 'USERPROFILE' in os.environ:
                userhome = compat_getenv('USERPROFILE')
            elif not 'HOMEPATH' in os.environ:
                return path
            else:
                try:
                    drive = compat_getenv('HOMEDRIVE')
                except KeyError:
                    drive = ''
                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))

            if i != 1: #~user
                userhome = os.path.join(os.path.dirname(userhome), path[1:i])

            return userhome + path[i:]
    else:
        compat_expanduser = os.path.expanduser


if sys.version_info < (3, 0):
    def compat_print(s):
        from .utils import preferredencoding
        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
else:
    def compat_print(s):
        assert type(s) == type(u'')
        print(s)


try:
    subprocess_check_output = subprocess.check_output
except AttributeError:
    def subprocess_check_output(*args, **kwargs):
        assert 'input' not in kwargs
        p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
        output, _ = p.communicate()
        ret = p.poll()
        if ret:
            raise subprocess.CalledProcessError(ret, p.args, output=output)
        return output

if sys.version_info < (3, 0) and sys.platform == 'win32':
    def compat_getpass(prompt, *args, **kwargs):
        if isinstance(prompt, compat_str):
            prompt = prompt.encode(preferredencoding())
        return getpass.getpass(prompt, *args, **kwargs)
else:
    compat_getpass = getpass.getpass


__all__ = [
    'compat_HTTPError',
    'compat_chr',
    'compat_cookiejar',
    'compat_expanduser',
    'compat_getenv',
    'compat_getpass',
    'compat_html_entities',
    'compat_html_parser',
    'compat_http_client',
    'compat_ord',
    'compat_parse_qs',
    'compat_print',
    'compat_str',
    'compat_subprocess_get_DEVNULL',
    'compat_urllib_error',
    'compat_urllib_parse',
    'compat_urllib_parse_unquote',
    'compat_urllib_parse_urlparse',
    'compat_urllib_request',
    'compat_urlparse',
    'compat_urlretrieve',
    'compat_xml_parse_error',
    'shlex_quote',
    'subprocess_check_output',
]
Commit	Line	Data
8c25f81b PH	1	import getpass
	2	import os
	3	import subprocess
	4	import sys
	5
	6
	7	try:
	8	import urllib.request as compat_urllib_request
	9	except ImportError: # Python 2
	10	import urllib2 as compat_urllib_request
	11
	12	try:
	13	import urllib.error as compat_urllib_error
	14	except ImportError: # Python 2
	15	import urllib2 as compat_urllib_error
	16
	17	try:
	18	import urllib.parse as compat_urllib_parse
	19	except ImportError: # Python 2
	20	import urllib as compat_urllib_parse
	21
	22	try:
	23	from urllib.parse import urlparse as compat_urllib_parse_urlparse
	24	except ImportError: # Python 2
	25	from urlparse import urlparse as compat_urllib_parse_urlparse
	26
	27	try:
	28	import urllib.parse as compat_urlparse
	29	except ImportError: # Python 2
	30	import urlparse as compat_urlparse
	31
	32	try:
	33	import http.cookiejar as compat_cookiejar
	34	except ImportError: # Python 2
	35	import cookielib as compat_cookiejar
	36
	37	try:
	38	import html.entities as compat_html_entities
	39	except ImportError: # Python 2
	40	import htmlentitydefs as compat_html_entities
	41
	42	try:
	43	import html.parser as compat_html_parser
	44	except ImportError: # Python 2
	45	import HTMLParser as compat_html_parser
	46
	47	try:
	48	import http.client as compat_http_client
	49	except ImportError: # Python 2
	50	import httplib as compat_http_client
	51
	52	try:
	53	from urllib.error import HTTPError as compat_HTTPError
	54	except ImportError: # Python 2
	55	from urllib2 import HTTPError as compat_HTTPError
	56
	57	try:
	58	from urllib.request import urlretrieve as compat_urlretrieve
	59	except ImportError: # Python 2
	60	from urllib import urlretrieve as compat_urlretrieve
	61
	62
	63	try:
	64	from subprocess import DEVNULL
65	compat_subprocess_get_DEVNULL = lambda: DEVNULL
66	except ImportError:
67	compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
68
69	try:
70	from urllib.parse import unquote as compat_urllib_parse_unquote
71	except ImportError:
72	def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
73	if string == '':
74	return string
75	res = string.split('%')
76	if len(res) == 1:
77	return string
78	if encoding is None:
79	encoding = 'utf-8'
80	if errors is None:
81	errors = 'replace'
82	# pct_sequence: contiguous sequence of percent-encoded bytes, decoded
83	pct_sequence = b''
84	string = res[0]
85	for item in res[1:]:
86	try:
87	if not item:
88	raise ValueError
89	pct_sequence += item[:2].decode('hex')
90	rest = item[2:]
91	if not rest:
92	# This segment was just a single percent-encoded character.
93	# May be part of a sequence of code units, so delay decoding.
94	# (Stored in pct_sequence).
95	continue
96	except ValueError:
97	rest = '%' + item
98	# Encountered non-percent-encoded characters. Flush the current
99	# pct_sequence.
100	string += pct_sequence.decode(encoding, errors) + rest
101	pct_sequence = b''
102	if pct_sequence:
103	# Flush the final pct_sequence
104	string += pct_sequence.decode(encoding, errors)
105	return string
106
107
108	try:
109	from urllib.parse import parse_qs as compat_parse_qs
110	except ImportError: # Python 2
111	# HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
112	# Python 2's version is apparently totally broken
113
114	def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
115	encoding='utf-8', errors='replace'):
116	qs, _coerce_result = qs, unicode
117	pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
118	r = []
119	for name_value in pairs:
120	if not name_value and not strict_parsing:
121	continue
122	nv = name_value.split('=', 1)
123	if len(nv) != 2:
124	if strict_parsing:
125	raise ValueError("bad query field: %r" % (name_value,))
126	# Handle case of a control-name with no equal sign
127	if keep_blank_values:
128	nv.append('')
129	else:
130	continue
131	if len(nv[1]) or keep_blank_values:
132	name = nv[0].replace('+', ' ')
133	name = compat_urllib_parse_unquote(
134	name, encoding=encoding, errors=errors)
135	name = _coerce_result(name)
136	value = nv[1].replace('+', ' ')
137	value = compat_urllib_parse_unquote(
138	value, encoding=encoding, errors=errors)
139	value = _coerce_result(value)
140	r.append((name, value))
141	return r
142
143	def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
144	encoding='utf-8', errors='replace'):
145	parsed_result = {}
146	pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
147	encoding=encoding, errors=errors)
148	for name, value in pairs:
149	if name in parsed_result:
150	parsed_result[name].append(value)
151	else:
152	parsed_result[name] = [value]
153	return parsed_result
154
155	try:
156	compat_str = unicode # Python 2
157	except NameError:
158	compat_str = str
159
160	try:
161	compat_chr = unichr # Python 2
162	except NameError:
163	compat_chr = chr
164
165	try:
166	from xml.etree.ElementTree import ParseError as compat_xml_parse_error
167	except ImportError: # Python 2.6
168	from xml.parsers.expat import ExpatError as compat_xml_parse_error
169
170	try:
171	from shlex import quote as shlex_quote
172	except ImportError: # Python < 3.3
173	def shlex_quote(s):
174	return "'" + s.replace("'", "'\"'\"'") + "'"
175
176
177	def compat_ord(c):
178	if type(c) is int: return c
179	else: return ord(c)
180
181
182	if sys.version_info >= (3, 0):
183	compat_getenv = os.getenv
184	compat_expanduser = os.path.expanduser
185	else:
186	# Environment variables should be decoded with filesystem encoding.
187	# Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
188
189	def compat_getenv(key, default=None):
190	from .utils import get_filesystem_encoding
191	env = os.getenv(key, default)
192	if env:
193	env = env.decode(get_filesystem_encoding())
194	return env
195
196	# HACK: The default implementations of os.path.expanduser from cpython do not decode
197	# environment variables with filesystem encoding. We will work around this by
198	# providing adjusted implementations.
199	# The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
200	# for different platforms with correct environment variables decoding.
201
202	if os.name == 'posix':
203	def compat_expanduser(path):
204	"""Expand ~ and ~user constructions. If user or $HOME is unknown,
205	do nothing."""
206	if not path.startswith('~'):
207	return path
208	i = path.find('/', 1)
209	if i < 0:
210	i = len(path)
211	if i == 1:
212	if 'HOME' not in os.environ:
213	import pwd
214	userhome = pwd.getpwuid(os.getuid()).pw_dir
215	else:
216	userhome = compat_getenv('HOME')
217	else:
218	import pwd
219	try:
220	pwent = pwd.getpwnam(path[1:i])
221	except KeyError:
222	return path
223	userhome = pwent.pw_dir
224	userhome = userhome.rstrip('/')
225	return (userhome + path[i:]) or '/'
226	elif os.name == 'nt' or os.name == 'ce':
227	def compat_expanduser(path):
228	"""Expand ~ and ~user constructs.
229
230	If user or $HOME is unknown, do nothing."""
231	if path[:1] != '~':
232	return path
233	i, n = 1, len(path)
234	while i < n and path[i] not in '/\\':
235	i = i + 1
236
237	if 'HOME' in os.environ:
238	userhome = compat_getenv('HOME')
239	elif 'USERPROFILE' in os.environ:
240	userhome = compat_getenv('USERPROFILE')
241	elif not 'HOMEPATH' in os.environ:
242	return path
243	else:
244	try:
245	drive = compat_getenv('HOMEDRIVE')
246	except KeyError:
247	drive = ''
248	userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
249
250	if i != 1: #~user
251	userhome = os.path.join(os.path.dirname(userhome), path[1:i])
252
253	return userhome + path[i:]
254	else:
255	compat_expanduser = os.path.expanduser
256
257
258	if sys.version_info < (3, 0):
259	def compat_print(s):
260	from .utils import preferredencoding
261	print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
262	else:
263	def compat_print(s):
264	assert type(s) == type(u'')
265	print(s)
266
267
268	try:
269	subprocess_check_output = subprocess.check_output
270	except AttributeError:
271	def subprocess_check_output(args, *kwargs):
272	assert 'input' not in kwargs
273	p = subprocess.Popen(args, stdout=subprocess.PIPE, *kwargs)
274	output, _ = p.communicate()
275	ret = p.poll()
276	if ret:
277	raise subprocess.CalledProcessError(ret, p.args, output=output)
278	return output
279
280	if sys.version_info < (3, 0) and sys.platform == 'win32':
281	def compat_getpass(prompt, args, *kwargs):
282	if isinstance(prompt, compat_str):
283	prompt = prompt.encode(preferredencoding())
284	return getpass.getpass(prompt, args, *kwargs)
285	else:
286	compat_getpass = getpass.getpass
287
288
289	__all__ = [
290	'compat_HTTPError',
291	'compat_chr',
292	'compat_cookiejar',
293	'compat_expanduser',
294	'compat_getenv',
295	'compat_getpass',
296	'compat_html_entities',
297	'compat_html_parser',
298	'compat_http_client',
299	'compat_ord',
300	'compat_parse_qs',
301	'compat_print',
302	'compat_str',
303	'compat_subprocess_get_DEVNULL',
304	'compat_urllib_error',
305	'compat_urllib_parse',
306	'compat_urllib_parse_unquote',
307	'compat_urllib_parse_urlparse',
308	'compat_urllib_request',
309	'compat_urlparse',
310	'compat_urlretrieve',
311	'compat_xml_parse_error',
312	'shlex_quote',
313	'subprocess_check_output',
314	]