1 from __future__
import unicode_literals
15 import urllib
.request
as compat_urllib_request
16 except ImportError: # Python 2
17 import urllib2
as compat_urllib_request
20 import urllib
.error
as compat_urllib_error
21 except ImportError: # Python 2
22 import urllib2
as compat_urllib_error
25 import urllib
.parse
as compat_urllib_parse
26 except ImportError: # Python 2
27 import urllib
as compat_urllib_parse
30 from urllib
.parse
import urlparse
as compat_urllib_parse_urlparse
31 except ImportError: # Python 2
32 from urlparse
import urlparse
as compat_urllib_parse_urlparse
35 import urllib
.parse
as compat_urlparse
36 except ImportError: # Python 2
37 import urlparse
as compat_urlparse
40 import http
.cookiejar
as compat_cookiejar
41 except ImportError: # Python 2
42 import cookielib
as compat_cookiejar
45 import html
.entities
as compat_html_entities
46 except ImportError: # Python 2
47 import htmlentitydefs
as compat_html_entities
50 import html
.parser
as compat_html_parser
51 except ImportError: # Python 2
52 import HTMLParser
as compat_html_parser
55 import http
.client
as compat_http_client
56 except ImportError: # Python 2
57 import httplib
as compat_http_client
60 from urllib
.error
import HTTPError
as compat_HTTPError
61 except ImportError: # Python 2
62 from urllib2
import HTTPError
as compat_HTTPError
65 from urllib
.request
import urlretrieve
as compat_urlretrieve
66 except ImportError: # Python 2
67 from urllib
import urlretrieve
as compat_urlretrieve
71 from subprocess
import DEVNULL
72 compat_subprocess_get_DEVNULL
= lambda: DEVNULL
74 compat_subprocess_get_DEVNULL
= lambda: open(os
.path
.devnull
, 'w')
77 import http
.server
as compat_http_server
79 import BaseHTTPServer
as compat_http_server
82 from urllib
.parse
import unquote
as compat_urllib_parse_unquote
84 def compat_urllib_parse_unquote(string
, encoding
='utf-8', errors
='replace'):
87 res
= string
.split('%')
94 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
101 pct_sequence
+= item
[:2].decode('hex')
104 # This segment was just a single percent-encoded character.
105 # May be part of a sequence of code units, so delay decoding.
106 # (Stored in pct_sequence).
110 # Encountered non-percent-encoded characters. Flush the current
112 string
+= pct_sequence
.decode(encoding
, errors
) + rest
115 # Flush the final pct_sequence
116 string
+= pct_sequence
.decode(encoding
, errors
)
120 compat_str
= unicode # Python 2
125 compat_basestring
= basestring
# Python 2
127 compat_basestring
= str
130 compat_chr
= unichr # Python 2
135 from xml
.etree
.ElementTree
import ParseError
as compat_xml_parse_error
136 except ImportError: # Python 2.6
137 from xml
.parsers
.expat
import ExpatError
as compat_xml_parse_error
141 from urllib
.parse
import parse_qs
as compat_parse_qs
142 except ImportError: # Python 2
143 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
144 # Python 2's version is apparently totally broken
146 def _parse_qsl(qs
, keep_blank_values
=False, strict_parsing
=False,
147 encoding
='utf-8', errors
='replace'):
148 qs
, _coerce_result
= qs
, compat_str
149 pairs
= [s2
for s1
in qs
.split('&') for s2
in s1
.split(';')]
151 for name_value
in pairs
:
152 if not name_value
and not strict_parsing
:
154 nv
= name_value
.split('=', 1)
157 raise ValueError("bad query field: %r" % (name_value
,))
158 # Handle case of a control-name with no equal sign
159 if keep_blank_values
:
163 if len(nv
[1]) or keep_blank_values
:
164 name
= nv
[0].replace('+', ' ')
165 name
= compat_urllib_parse_unquote(
166 name
, encoding
=encoding
, errors
=errors
)
167 name
= _coerce_result(name
)
168 value
= nv
[1].replace('+', ' ')
169 value
= compat_urllib_parse_unquote(
170 value
, encoding
=encoding
, errors
=errors
)
171 value
= _coerce_result(value
)
172 r
.append((name
, value
))
175 def compat_parse_qs(qs
, keep_blank_values
=False, strict_parsing
=False,
176 encoding
='utf-8', errors
='replace'):
178 pairs
= _parse_qsl(qs
, keep_blank_values
, strict_parsing
,
179 encoding
=encoding
, errors
=errors
)
180 for name
, value
in pairs
:
181 if name
in parsed_result
:
182 parsed_result
[name
].append(value
)
184 parsed_result
[name
] = [value
]
188 from shlex
import quote
as shlex_quote
189 except ImportError: # Python < 3.3
191 if re
.match(r
'^[-_\w./]+$', s
):
194 return "'" + s
.replace("'", "'\"'\"'") + "'"
204 if sys
.version_info
>= (3, 0):
205 compat_getenv
= os
.getenv
206 compat_expanduser
= os
.path
.expanduser
208 # Environment variables should be decoded with filesystem encoding.
209 # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
211 def compat_getenv(key
, default
=None):
212 from .utils
import get_filesystem_encoding
213 env
= os
.getenv(key
, default
)
215 env
= env
.decode(get_filesystem_encoding())
218 # HACK: The default implementations of os.path.expanduser from cpython do not decode
219 # environment variables with filesystem encoding. We will work around this by
220 # providing adjusted implementations.
221 # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
222 # for different platforms with correct environment variables decoding.
224 if os
.name
== 'posix':
225 def compat_expanduser(path
):
226 """Expand ~ and ~user constructions. If user or $HOME is unknown,
228 if not path
.startswith('~'):
230 i
= path
.find('/', 1)
234 if 'HOME' not in os
.environ
:
236 userhome
= pwd
.getpwuid(os
.getuid()).pw_dir
238 userhome
= compat_getenv('HOME')
242 pwent
= pwd
.getpwnam(path
[1:i
])
245 userhome
= pwent
.pw_dir
246 userhome
= userhome
.rstrip('/')
247 return (userhome
+ path
[i
:]) or '/'
248 elif os
.name
== 'nt' or os
.name
== 'ce':
249 def compat_expanduser(path
):
250 """Expand ~ and ~user constructs.
252 If user or $HOME is unknown, do nothing."""
256 while i
< n
and path
[i
] not in '/\\':
259 if 'HOME' in os
.environ
:
260 userhome
= compat_getenv('HOME')
261 elif 'USERPROFILE' in os
.environ
:
262 userhome
= compat_getenv('USERPROFILE')
263 elif 'HOMEPATH' not in os
.environ
:
267 drive
= compat_getenv('HOMEDRIVE')
270 userhome
= os
.path
.join(drive
, compat_getenv('HOMEPATH'))
273 userhome
= os
.path
.join(os
.path
.dirname(userhome
), path
[1:i
])
275 return userhome
+ path
[i
:]
277 compat_expanduser
= os
.path
.expanduser
280 if sys
.version_info
< (3, 0):
282 from .utils
import preferredencoding
283 print(s
.encode(preferredencoding(), 'xmlcharrefreplace'))
286 assert isinstance(s
, compat_str
)
291 subprocess_check_output
= subprocess
.check_output
292 except AttributeError:
293 def subprocess_check_output(*args
, **kwargs
):
294 assert 'input' not in kwargs
295 p
= subprocess
.Popen(*args
, stdout
=subprocess
.PIPE
, **kwargs
)
296 output
, _
= p
.communicate()
299 raise subprocess
.CalledProcessError(ret
, p
.args
, output
=output
)
302 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
303 def compat_getpass(prompt
, *args
, **kwargs
):
304 if isinstance(prompt
, compat_str
):
305 from .utils
import preferredencoding
306 prompt
= prompt
.encode(preferredencoding())
307 return getpass
.getpass(prompt
, *args
, **kwargs
)
309 compat_getpass
= getpass
.getpass
311 # Old 2.6 and 2.7 releases require kwargs to be bytes
315 _testfunc(**{'x': 0}
)
317 def compat_kwargs(kwargs
):
318 return dict((bytes(k
), v
) for k
, v
in kwargs
.items())
320 compat_kwargs
= lambda kwargs
: kwargs
323 if sys
.version_info
< (2, 7):
324 def compat_socket_create_connection(address
, timeout
, source_address
=None):
327 for res
in socket
.getaddrinfo(host
, port
, 0, socket
.SOCK_STREAM
):
328 af
, socktype
, proto
, canonname
, sa
= res
331 sock
= socket
.socket(af
, socktype
, proto
)
332 sock
.settimeout(timeout
)
334 sock
.bind(source_address
)
337 except socket
.error
as _
:
344 raise socket
.error("getaddrinfo returns an empty list")
346 compat_socket_create_connection
= socket
.create_connection
349 # Fix https://github.com/rg3/youtube-dl/issues/4223
350 # See http://bugs.python.org/issue9161 for what is broken
351 def workaround_optparse_bug9161():
352 op
= optparse
.OptionParser()
353 og
= optparse
.OptionGroup(op
, 'foo')
357 real_add_option
= optparse
.OptionGroup
.add_option
359 def _compat_add_option(self
, *args
, **kwargs
):
361 v
.encode('ascii', 'replace') if isinstance(v
, compat_str
)
363 bargs
= [enc(a
) for a
in args
]
365 (k
, enc(v
)) for k
, v
in kwargs
.items())
366 return real_add_option(self
, *bargs
, **bkwargs
)
367 optparse
.OptionGroup
.add_option
= _compat_add_option
369 if hasattr(shutil
, 'get_terminal_size'): # Python >= 3.3
370 compat_get_terminal_size
= shutil
.get_terminal_size
372 _terminal_size
= collections
.namedtuple('terminal_size', ['columns', 'lines'])
374 def compat_get_terminal_size():
375 columns
= compat_getenv('COLUMNS', None)
377 columns
= int(columns
)
380 lines
= compat_getenv('LINES', None)
387 sp
= subprocess
.Popen(
389 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
390 out
, err
= sp
.communicate()
391 lines
, columns
= map(int, out
.split())
394 return _terminal_size(columns
, lines
)
403 'compat_get_terminal_size',
406 'compat_html_entities',
407 'compat_html_parser',
408 'compat_http_client',
409 'compat_http_server',
414 'compat_socket_create_connection',
416 'compat_subprocess_get_DEVNULL',
417 'compat_urllib_error',
418 'compat_urllib_parse',
419 'compat_urllib_parse_unquote',
420 'compat_urllib_parse_urlparse',
421 'compat_urllib_request',
423 'compat_urlretrieve',
424 'compat_xml_parse_error',
426 'subprocess_check_output',
427 'workaround_optparse_bug9161',