2 # -*- coding: utf-8 -*-
29 import xml
.etree
.ElementTree
33 import urllib
.request
as compat_urllib_request
34 except ImportError: # Python 2
35 import urllib2
as compat_urllib_request
38 import urllib
.error
as compat_urllib_error
39 except ImportError: # Python 2
40 import urllib2
as compat_urllib_error
43 import urllib
.parse
as compat_urllib_parse
44 except ImportError: # Python 2
45 import urllib
as compat_urllib_parse
48 from urllib
.parse
import urlparse
as compat_urllib_parse_urlparse
49 except ImportError: # Python 2
50 from urlparse
import urlparse
as compat_urllib_parse_urlparse
53 import urllib
.parse
as compat_urlparse
54 except ImportError: # Python 2
55 import urlparse
as compat_urlparse
58 import http
.cookiejar
as compat_cookiejar
59 except ImportError: # Python 2
60 import cookielib
as compat_cookiejar
63 import html
.entities
as compat_html_entities
64 except ImportError: # Python 2
65 import htmlentitydefs
as compat_html_entities
68 import html
.parser
as compat_html_parser
69 except ImportError: # Python 2
70 import HTMLParser
as compat_html_parser
73 import http
.client
as compat_http_client
74 except ImportError: # Python 2
75 import httplib
as compat_http_client
78 from urllib
.error
import HTTPError
as compat_HTTPError
79 except ImportError: # Python 2
80 from urllib2
import HTTPError
as compat_HTTPError
83 from urllib
.request
import urlretrieve
as compat_urlretrieve
84 except ImportError: # Python 2
85 from urllib
import urlretrieve
as compat_urlretrieve
89 from subprocess
import DEVNULL
90 compat_subprocess_get_DEVNULL
= lambda: DEVNULL
92 compat_subprocess_get_DEVNULL
= lambda: open(os
.path
.devnull
, 'w')
95 from urllib
.parse
import unquote
as compat_urllib_parse_unquote
97 def compat_urllib_parse_unquote(string
, encoding
='utf-8', errors
='replace'):
100 res
= string
.split('%')
107 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
114 pct_sequence
+= item
[:2].decode('hex')
117 # This segment was just a single percent-encoded character.
118 # May be part of a sequence of code units, so delay decoding.
119 # (Stored in pct_sequence).
123 # Encountered non-percent-encoded characters. Flush the current
125 string
+= pct_sequence
.decode(encoding
, errors
) + rest
128 # Flush the final pct_sequence
129 string
+= pct_sequence
.decode(encoding
, errors
)
134 from urllib
.parse
import parse_qs
as compat_parse_qs
135 except ImportError: # Python 2
136 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
137 # Python 2's version is apparently totally broken
139 def _parse_qsl(qs
, keep_blank_values
=False, strict_parsing
=False,
140 encoding
='utf-8', errors
='replace'):
141 qs
, _coerce_result
= qs
, unicode
142 pairs
= [s2
for s1
in qs
.split('&') for s2
in s1
.split(';')]
144 for name_value
in pairs
:
145 if not name_value
and not strict_parsing
:
147 nv
= name_value
.split('=', 1)
150 raise ValueError("bad query field: %r" % (name_value
,))
151 # Handle case of a control-name with no equal sign
152 if keep_blank_values
:
156 if len(nv
[1]) or keep_blank_values
:
157 name
= nv
[0].replace('+', ' ')
158 name
= compat_urllib_parse_unquote(
159 name
, encoding
=encoding
, errors
=errors
)
160 name
= _coerce_result(name
)
161 value
= nv
[1].replace('+', ' ')
162 value
= compat_urllib_parse_unquote(
163 value
, encoding
=encoding
, errors
=errors
)
164 value
= _coerce_result(value
)
165 r
.append((name
, value
))
168 def compat_parse_qs(qs
, keep_blank_values
=False, strict_parsing
=False,
169 encoding
='utf-8', errors
='replace'):
171 pairs
= _parse_qsl(qs
, keep_blank_values
, strict_parsing
,
172 encoding
=encoding
, errors
=errors
)
173 for name
, value
in pairs
:
174 if name
in parsed_result
:
175 parsed_result
[name
].append(value
)
177 parsed_result
[name
] = [value
]
181 compat_str
= unicode # Python 2
186 compat_chr
= unichr # Python 2
191 from xml
.etree
.ElementTree
import ParseError
as compat_xml_parse_error
192 except ImportError: # Python 2.6
193 from xml
.parsers
.expat
import ExpatError
as compat_xml_parse_error
196 from shlex
import quote
as shlex_quote
197 except ImportError: # Python < 3.3
199 return "'" + s
.replace("'", "'\"'\"'") + "'"
203 if type(c
) is int: return c
206 # This is not clearly defined otherwise
207 compiled_regex_type
= type(re
.compile(''))
210 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)',
211 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
212 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
213 'Accept-Encoding': 'gzip, deflate',
214 'Accept-Language': 'en-us,en;q=0.5',
217 def preferredencoding():
218 """Get preferred encoding.
220 Returns the best encoding scheme for the system, based on
221 locale.getpreferredencoding() and some further tweaks.
224 pref
= locale
.getpreferredencoding()
231 if sys
.version_info
< (3,0):
233 print(s
.encode(preferredencoding(), 'xmlcharrefreplace'))
236 assert type(s
) == type(u
'')
240 def write_json_file(obj
, fn
):
241 """ Encode obj as JSON and write it to fn, atomically """
245 'prefix': os
.path
.basename(fn
) + '.',
246 'dir': os
.path
.dirname(fn
),
250 # In Python 2.x, json.dump expects a bytestream.
251 # In Python 3.x, it writes to a character stream
252 if sys
.version_info
< (3, 0):
260 tf
= tempfile
.NamedTemporaryFile(**args
)
265 os
.rename(tf
.name
, fn
)
274 if sys
.version_info
>= (2, 7):
275 def find_xpath_attr(node
, xpath
, key
, val
):
276 """ Find the xpath xpath[@key=val] """
277 assert re
.match(r
'^[a-zA-Z-]+$', key
)
278 assert re
.match(r
'^[a-zA-Z0-9@\s:._-]*$', val
)
279 expr
= xpath
+ u
"[@%s='%s']" % (key
, val
)
280 return node
.find(expr
)
282 def find_xpath_attr(node
, xpath
, key
, val
):
283 for f
in node
.findall(xpath
):
284 if f
.attrib
.get(key
) == val
:
288 # On python2.6 the xml.etree.ElementTree.Element methods don't support
289 # the namespace parameter
290 def xpath_with_ns(path
, ns_map
):
291 components
= [c
.split(':') for c
in path
.split('/')]
295 replaced
.append(c
[0])
298 replaced
.append('{%s}%s' % (ns_map
[ns
], tag
))
299 return '/'.join(replaced
)
301 def htmlentity_transform(matchobj
):
302 """Transforms an HTML entity to a character.
304 This function receives a match object and is intended to be used with
305 the re.sub() function.
307 entity
= matchobj
.group(1)
309 # Known non-numeric HTML entity
310 if entity
in compat_html_entities
.name2codepoint
:
311 return compat_chr(compat_html_entities
.name2codepoint
[entity
])
313 mobj
= re
.match(u
'(?u)#(x?\\d+)', entity
)
315 numstr
= mobj
.group(1)
316 if numstr
.startswith(u
'x'):
318 numstr
= u
'0%s' % numstr
321 return compat_chr(int(numstr
, base
))
323 # Unknown entity in name, return its literal representation
324 return (u
'&%s;' % entity
)
326 compat_html_parser
.locatestarttagend
= re
.compile(r
"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re
.VERBOSE
) # backport bugfix
327 class BaseHTMLParser(compat_html_parser
.HTMLParser
):
329 compat_html_parser
.HTMLParser
.__init
__(self
)
332 def loads(self
, html
):
337 class AttrParser(BaseHTMLParser
):
338 """Modified HTMLParser that isolates a tag with the specified attribute"""
339 def __init__(self
, attribute
, value
):
340 self
.attribute
= attribute
345 self
.watch_startpos
= False
347 BaseHTMLParser
.__init
__(self
)
349 def error(self
, message
):
350 if self
.error_count
> 10 or self
.started
:
351 raise compat_html_parser
.HTMLParseError(message
, self
.getpos())
352 self
.rawdata
= '\n'.join(self
.html
.split('\n')[self
.getpos()[0]:]) # skip one line
353 self
.error_count
+= 1
356 def handle_starttag(self
, tag
, attrs
):
359 self
.find_startpos(None)
360 if self
.attribute
in attrs
and attrs
[self
.attribute
] == self
.value
:
363 self
.watch_startpos
= True
365 if not tag
in self
.depth
: self
.depth
[tag
] = 0
368 def handle_endtag(self
, tag
):
370 if tag
in self
.depth
: self
.depth
[tag
] -= 1
371 if self
.depth
[self
.result
[0]] == 0:
373 self
.result
.append(self
.getpos())
375 def find_startpos(self
, x
):
376 """Needed to put the start position of the result (self.result[1])
377 after the opening tag with the requested id"""
378 if self
.watch_startpos
:
379 self
.watch_startpos
= False
380 self
.result
.append(self
.getpos())
381 handle_entityref
= handle_charref
= handle_data
= handle_comment
= \
382 handle_decl
= handle_pi
= unknown_decl
= find_startpos
384 def get_result(self
):
385 if self
.result
is None:
387 if len(self
.result
) != 3:
389 lines
= self
.html
.split('\n')
390 lines
= lines
[self
.result
[1][0]-1:self
.result
[2][0]]
391 lines
[0] = lines
[0][self
.result
[1][1]:]
393 lines
[-1] = lines
[-1][:self
.result
[2][1]-self
.result
[1][1]]
394 lines
[-1] = lines
[-1][:self
.result
[2][1]]
395 return '\n'.join(lines
).strip()
396 # Hack for https://github.com/rg3/youtube-dl/issues/662
397 if sys
.version_info
< (2, 7, 3):
398 AttrParser
.parse_endtag
= (lambda self
, i
:
399 i
+ len("</scr'+'ipt>")
400 if self
.rawdata
[i
:].startswith("</scr'+'ipt>")
401 else compat_html_parser
.HTMLParser
.parse_endtag(self
, i
))
403 def get_element_by_id(id, html
):
404 """Return the content of the tag with the specified ID in the passed HTML document"""
405 return get_element_by_attribute("id", id, html
)
407 def get_element_by_attribute(attribute
, value
, html
):
408 """Return the content of the tag with the specified attribute in the passed HTML document"""
409 parser
= AttrParser(attribute
, value
)
412 except compat_html_parser
.HTMLParseError
:
414 return parser
.get_result()
416 class MetaParser(BaseHTMLParser
):
418 Modified HTMLParser that isolates a meta tag with the specified name
421 def __init__(self
, name
):
422 BaseHTMLParser
.__init
__(self
)
427 def handle_starttag(self
, tag
, attrs
):
431 if attrs
.get('name') == self
.name
:
432 self
.result
= attrs
.get('content')
434 def get_result(self
):
437 def get_meta_content(name
, html
):
439 Return the content attribute from the meta tag with the given name attribute.
441 parser
= MetaParser(name
)
444 except compat_html_parser
.HTMLParseError
:
446 return parser
.get_result()
449 def clean_html(html
):
450 """Clean an HTML snippet into a readable string"""
452 html
= html
.replace('\n', ' ')
453 html
= re
.sub(r
'\s*<\s*br\s*/?\s*>\s*', '\n', html
)
454 html
= re
.sub(r
'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html
)
456 html
= re
.sub('<.*?>', '', html
)
457 # Replace html entities
458 html
= unescapeHTML(html
)
462 def sanitize_open(filename
, open_mode
):
463 """Try to open the given filename, and slightly tweak it if this fails.
465 Attempts to open the given filename. If this fails, it tries to change
466 the filename slightly, step by step, until it's either able to open it
467 or it fails and raises a final exception, like the standard open()
470 It returns the tuple (stream, definitive_file_name).
474 if sys
.platform
== 'win32':
476 msvcrt
.setmode(sys
.stdout
.fileno(), os
.O_BINARY
)
477 return (sys
.stdout
.buffer if hasattr(sys
.stdout
, 'buffer') else sys
.stdout
, filename
)
478 stream
= open(encodeFilename(filename
), open_mode
)
479 return (stream
, filename
)
480 except (IOError, OSError) as err
:
481 if err
.errno
in (errno
.EACCES
,):
484 # In case of error, try to remove win32 forbidden chars
485 alt_filename
= os
.path
.join(
486 re
.sub(u
'[/<>:"\\|\\\\?\\*]', u
'#', path_part
)
487 for path_part
in os
.path
.split(filename
)
489 if alt_filename
== filename
:
492 # An exception here should be caught in the caller
493 stream
= open(encodeFilename(filename
), open_mode
)
494 return (stream
, alt_filename
)
497 def timeconvert(timestr
):
498 """Convert RFC 2822 defined time string into system timestamp"""
500 timetuple
= email
.utils
.parsedate_tz(timestr
)
501 if timetuple
is not None:
502 timestamp
= email
.utils
.mktime_tz(timetuple
)
505 def sanitize_filename(s
, restricted
=False, is_id
=False):
506 """Sanitizes a string so it could be used as part of a filename.
507 If restricted is set, use a stricter subset of allowed characters.
508 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
510 def replace_insane(char
):
511 if char
== '?' or ord(char
) < 32 or ord(char
) == 127:
514 return '' if restricted
else '\''
516 return '_-' if restricted
else ' -'
517 elif char
in '\\/|*<>':
519 if restricted
and (char
in '!&\'()[]{}$;`^,#' or char
.isspace()):
521 if restricted
and ord(char
) > 127:
525 result
= u
''.join(map(replace_insane
, s
))
527 while '__' in result
:
528 result
= result
.replace('__', '_')
529 result
= result
.strip('_')
530 # Common case of "Foreign band name - English song title"
531 if restricted
and result
.startswith('-_'):
537 def orderedSet(iterable
):
538 """ Remove all duplicates from the input iterable """
549 assert type(s
) == compat_str
551 result
= re
.sub(r
'(?u)&(.+?);', htmlentity_transform
, s
)
555 def encodeFilename(s
, for_subprocess
=False):
557 @param s The name of the file
560 assert type(s
) == compat_str
562 # Python 3 has a Unicode API
563 if sys
.version_info
>= (3, 0):
566 if sys
.platform
== 'win32' and sys
.getwindowsversion()[0] >= 5:
567 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
568 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
569 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
570 if not for_subprocess
:
573 # For subprocess calls, encode with locale encoding
574 # Refer to http://stackoverflow.com/a/9951851/35070
575 encoding
= preferredencoding()
577 encoding
= sys
.getfilesystemencoding()
580 return s
.encode(encoding
, 'ignore')
583 def encodeArgument(s
):
584 if not isinstance(s
, compat_str
):
585 # Legacy code that uses byte strings
586 # Uncomment the following line after fixing all post processors
587 #assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
588 s
= s
.decode('ascii')
589 return encodeFilename(s
, True)
592 def decodeOption(optval
):
595 if isinstance(optval
, bytes):
596 optval
= optval
.decode(preferredencoding())
598 assert isinstance(optval
, compat_str
)
601 def formatSeconds(secs
):
603 return '%d:%02d:%02d' % (secs
// 3600, (secs
% 3600) // 60, secs
% 60)
605 return '%d:%02d' % (secs
// 60, secs
% 60)
610 def make_HTTPS_handler(opts_no_check_certificate
, **kwargs
):
611 if sys
.version_info
< (3, 2):
614 class HTTPSConnectionV3(httplib
.HTTPSConnection
):
615 def __init__(self
, *args
, **kwargs
):
616 httplib
.HTTPSConnection
.__init
__(self
, *args
, **kwargs
)
619 sock
= socket
.create_connection((self
.host
, self
.port
), self
.timeout
)
620 if getattr(self
, '_tunnel_host', False):
624 self
.sock
= ssl
.wrap_socket(sock
, self
.key_file
, self
.cert_file
, ssl_version
=ssl
.PROTOCOL_SSLv3
)
626 self
.sock
= ssl
.wrap_socket(sock
, self
.key_file
, self
.cert_file
, ssl_version
=ssl
.PROTOCOL_SSLv23
)
628 class HTTPSHandlerV3(compat_urllib_request
.HTTPSHandler
):
629 def https_open(self
, req
):
630 return self
.do_open(HTTPSConnectionV3
, req
)
631 return HTTPSHandlerV3(**kwargs
)
633 context
= ssl
.SSLContext(ssl
.PROTOCOL_SSLv3
)
634 context
.verify_mode
= (ssl
.CERT_NONE
635 if opts_no_check_certificate
636 else ssl
.CERT_REQUIRED
)
637 context
.set_default_verify_paths()
639 context
.load_default_certs()
640 except AttributeError:
642 return compat_urllib_request
.HTTPSHandler(context
=context
, **kwargs
)
644 class ExtractorError(Exception):
645 """Error during info extraction."""
646 def __init__(self
, msg
, tb
=None, expected
=False, cause
=None, video_id
=None):
647 """ tb, if given, is the original traceback (so that it can be printed out).
648 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
651 if sys
.exc_info()[0] in (compat_urllib_error
.URLError
, socket
.timeout
, UnavailableVideoError
):
653 if video_id
is not None:
654 msg
= video_id
+ ': ' + msg
656 msg
= msg
+ u
'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type youtube-dl -U to update.'
657 super(ExtractorError
, self
).__init
__(msg
)
660 self
.exc_info
= sys
.exc_info() # preserve original exception
662 self
.video_id
= video_id
664 def format_traceback(self
):
665 if self
.traceback
is None:
667 return u
''.join(traceback
.format_tb(self
.traceback
))
670 class RegexNotFoundError(ExtractorError
):
671 """Error when a regex didn't match"""
675 class DownloadError(Exception):
676 """Download Error exception.
678 This exception may be thrown by FileDownloader objects if they are not
679 configured to continue on errors. They will contain the appropriate
682 def __init__(self
, msg
, exc_info
=None):
683 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
684 super(DownloadError
, self
).__init
__(msg
)
685 self
.exc_info
= exc_info
688 class SameFileError(Exception):
689 """Same File exception.
691 This exception will be thrown by FileDownloader objects if they detect
692 multiple files would have to be downloaded to the same file on disk.
697 class PostProcessingError(Exception):
698 """Post Processing exception.
700 This exception may be raised by PostProcessor's .run() method to
701 indicate an error in the postprocessing task.
703 def __init__(self
, msg
):
706 class MaxDownloadsReached(Exception):
707 """ --max-downloads limit has been reached. """
711 class UnavailableVideoError(Exception):
712 """Unavailable Format exception.
714 This exception will be thrown when a video is requested
715 in a format that is not available for that video.
720 class ContentTooShortError(Exception):
721 """Content Too Short exception.
723 This exception may be raised by FileDownloader objects when a file they
724 download is too small for what the server announced first, indicating
725 the connection was probably interrupted.
731 def __init__(self
, downloaded
, expected
):
732 self
.downloaded
= downloaded
733 self
.expected
= expected
735 class YoutubeDLHandler(compat_urllib_request
.HTTPHandler
):
736 """Handler for HTTP requests and responses.
738 This class, when installed with an OpenerDirector, automatically adds
739 the standard headers to every HTTP request and handles gzipped and
740 deflated responses from web servers. If compression is to be avoided in
741 a particular request, the original request in the program code only has
742 to include the HTTP header "Youtubedl-No-Compression", which will be
743 removed before making the real request.
745 Part of this code was copied from:
747 http://techknack.net/python-urllib2-handlers/
749 Andrew Rowls, the author of that code, agreed to release it to the
756 return zlib
.decompress(data
, -zlib
.MAX_WBITS
)
758 return zlib
.decompress(data
)
761 def addinfourl_wrapper(stream
, headers
, url
, code
):
762 if hasattr(compat_urllib_request
.addinfourl
, 'getcode'):
763 return compat_urllib_request
.addinfourl(stream
, headers
, url
, code
)
764 ret
= compat_urllib_request
.addinfourl(stream
, headers
, url
)
768 def http_request(self
, req
):
769 for h
,v
in std_headers
.items():
773 if 'Youtubedl-no-compression' in req
.headers
:
774 if 'Accept-encoding' in req
.headers
:
775 del req
.headers
['Accept-encoding']
776 del req
.headers
['Youtubedl-no-compression']
777 if 'Youtubedl-user-agent' in req
.headers
:
778 if 'User-agent' in req
.headers
:
779 del req
.headers
['User-agent']
780 req
.headers
['User-agent'] = req
.headers
['Youtubedl-user-agent']
781 del req
.headers
['Youtubedl-user-agent']
784 def http_response(self
, req
, resp
):
787 if resp
.headers
.get('Content-encoding', '') == 'gzip':
788 content
= resp
.read()
789 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
), mode
='rb')
791 uncompressed
= io
.BytesIO(gz
.read())
792 except IOError as original_ioerror
:
793 # There may be junk add the end of the file
794 # See http://stackoverflow.com/q/4928560/35070 for details
795 for i
in range(1, 1024):
797 gz
= gzip
.GzipFile(fileobj
=io
.BytesIO(content
[:-i
]), mode
='rb')
798 uncompressed
= io
.BytesIO(gz
.read())
803 raise original_ioerror
804 resp
= self
.addinfourl_wrapper(uncompressed
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
805 resp
.msg
= old_resp
.msg
807 if resp
.headers
.get('Content-encoding', '') == 'deflate':
808 gz
= io
.BytesIO(self
.deflate(resp
.read()))
809 resp
= self
.addinfourl_wrapper(gz
, old_resp
.headers
, old_resp
.url
, old_resp
.code
)
810 resp
.msg
= old_resp
.msg
813 https_request
= http_request
814 https_response
= http_response
817 def parse_iso8601(date_str
, delimiter
='T'):
818 """ Return a UNIX timestamp from the given date """
824 r
'Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$',
827 timezone
= datetime
.timedelta()
829 date_str
= date_str
[:-len(m
.group(0))]
830 if not m
.group('sign'):
831 timezone
= datetime
.timedelta()
833 sign
= 1 if m
.group('sign') == '+' else -1
834 timezone
= datetime
.timedelta(
835 hours
=sign
* int(m
.group('hours')),
836 minutes
=sign
* int(m
.group('minutes')))
837 date_format
= '%Y-%m-%d{0}%H:%M:%S'.format(delimiter
)
838 dt
= datetime
.datetime
.strptime(date_str
, date_format
) - timezone
839 return calendar
.timegm(dt
.timetuple())
842 def unified_strdate(date_str
):
843 """Return a string with the date in the format YYYYMMDD"""
850 date_str
= date_str
.replace(',', ' ')
851 # %z (UTC offset) is only supported in python>=3.2
852 date_str
= re
.sub(r
' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str
)
853 format_expressions
= [
858 '%b %dst %Y %I:%M%p',
859 '%b %dnd %Y %I:%M%p',
860 '%b %dth %Y %I:%M%p',
870 '%Y-%m-%dT%H:%M:%SZ',
871 '%Y-%m-%dT%H:%M:%S.%fZ',
872 '%Y-%m-%dT%H:%M:%S.%f0Z',
874 '%Y-%m-%dT%H:%M:%S.%f',
877 for expression
in format_expressions
:
879 upload_date
= datetime
.datetime
.strptime(date_str
, expression
).strftime('%Y%m%d')
882 if upload_date
is None:
883 timetuple
= email
.utils
.parsedate_tz(date_str
)
885 upload_date
= datetime
.datetime(*timetuple
[:6]).strftime('%Y%m%d')
888 def determine_ext(url
, default_ext
=u
'unknown_video'):
891 guess
= url
.partition(u
'?')[0].rpartition(u
'.')[2]
892 if re
.match(r
'^[A-Za-z0-9]+$', guess
):
897 def subtitles_filename(filename
, sub_lang
, sub_format
):
898 return filename
.rsplit('.', 1)[0] + u
'.' + sub_lang
+ u
'.' + sub_format
900 def date_from_str(date_str
):
902 Return a datetime object from a string in the format YYYYMMDD or
903 (now|today)[+-][0-9](day|week|month|year)(s)?"""
904 today
= datetime
.date
.today()
905 if date_str
== 'now'or date_str
== 'today':
907 match
= re
.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str
)
908 if match
is not None:
909 sign
= match
.group('sign')
910 time
= int(match
.group('time'))
913 unit
= match
.group('unit')
922 delta
= datetime
.timedelta(**{unit: time}
)
924 return datetime
.datetime
.strptime(date_str
, "%Y%m%d").date()
926 def hyphenate_date(date_str
):
928 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
929 match
= re
.match(r
'^(\d\d\d\d)(\d\d)(\d\d)$', date_str
)
930 if match
is not None:
931 return '-'.join(match
.groups())
935 class DateRange(object):
936 """Represents a time interval between two dates"""
937 def __init__(self
, start
=None, end
=None):
938 """start and end must be strings in the format accepted by date"""
939 if start
is not None:
940 self
.start
= date_from_str(start
)
942 self
.start
= datetime
.datetime
.min.date()
944 self
.end
= date_from_str(end
)
946 self
.end
= datetime
.datetime
.max.date()
947 if self
.start
> self
.end
:
948 raise ValueError('Date range: "%s" , the start date must be before the end date' % self
)
951 """Returns a range that only contains the given day"""
953 def __contains__(self
, date
):
954 """Check if the date is in the range"""
955 if not isinstance(date
, datetime
.date
):
956 date
= date_from_str(date
)
957 return self
.start
<= date
<= self
.end
959 return '%s - %s' % ( self
.start
.isoformat(), self
.end
.isoformat())
963 """ Returns the platform name as a compat_str """
964 res
= platform
.platform()
965 if isinstance(res
, bytes):
966 res
= res
.decode(preferredencoding())
968 assert isinstance(res
, compat_str
)
972 def _windows_write_string(s
, out
):
973 """ Returns True if the string was written using special methods,
974 False if it has yet to be written out."""
975 # Adapted from http://stackoverflow.com/a/3259271/35070
978 import ctypes
.wintypes
986 fileno
= out
.fileno()
987 except AttributeError:
988 # If the output stream doesn't have a fileno, it's virtual
990 if fileno
not in WIN_OUTPUT_IDS
:
993 GetStdHandle
= ctypes
.WINFUNCTYPE(
994 ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.DWORD
)(
995 ("GetStdHandle", ctypes
.windll
.kernel32
))
996 h
= GetStdHandle(WIN_OUTPUT_IDS
[fileno
])
998 WriteConsoleW
= ctypes
.WINFUNCTYPE(
999 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
, ctypes
.wintypes
.LPWSTR
,
1000 ctypes
.wintypes
.DWORD
, ctypes
.POINTER(ctypes
.wintypes
.DWORD
),
1001 ctypes
.wintypes
.LPVOID
)(("WriteConsoleW", ctypes
.windll
.kernel32
))
1002 written
= ctypes
.wintypes
.DWORD(0)
1004 GetFileType
= ctypes
.WINFUNCTYPE(ctypes
.wintypes
.DWORD
, ctypes
.wintypes
.DWORD
)(("GetFileType", ctypes
.windll
.kernel32
))
1005 FILE_TYPE_CHAR
= 0x0002
1006 FILE_TYPE_REMOTE
= 0x8000
1007 GetConsoleMode
= ctypes
.WINFUNCTYPE(
1008 ctypes
.wintypes
.BOOL
, ctypes
.wintypes
.HANDLE
,
1009 ctypes
.POINTER(ctypes
.wintypes
.DWORD
))(
1010 ("GetConsoleMode", ctypes
.windll
.kernel32
))
1011 INVALID_HANDLE_VALUE
= ctypes
.wintypes
.DWORD(-1).value
1013 def not_a_console(handle
):
1014 if handle
== INVALID_HANDLE_VALUE
or handle
is None:
1016 return ((GetFileType(handle
) & ~FILE_TYPE_REMOTE
) != FILE_TYPE_CHAR
1017 or GetConsoleMode(handle
, ctypes
.byref(ctypes
.wintypes
.DWORD())) == 0)
1019 if not_a_console(h
):
1022 def next_nonbmp_pos(s
):
1024 return next(i
for i
, c
in enumerate(s
) if ord(c
) > 0xffff)
1025 except StopIteration:
1029 count
= min(next_nonbmp_pos(s
), 1024)
1031 ret
= WriteConsoleW(
1032 h
, s
, count
if count
else 2, ctypes
.byref(written
), None)
1034 raise OSError('Failed to write string')
1035 if not count
: # We just wrote a non-BMP character
1036 assert written
.value
== 2
1039 assert written
.value
> 0
1040 s
= s
[written
.value
:]
1044 def write_string(s
, out
=None, encoding
=None):
1047 assert type(s
) == compat_str
1049 if sys
.platform
== 'win32' and encoding
is None and hasattr(out
, 'fileno'):
1050 if _windows_write_string(s
, out
):
1053 if ('b' in getattr(out
, 'mode', '') or
1054 sys
.version_info
[0] < 3): # Python 2 lies about mode of sys.stderr
1055 byt
= s
.encode(encoding
or preferredencoding(), 'ignore')
1057 elif hasattr(out
, 'buffer'):
1058 enc
= encoding
or getattr(out
, 'encoding', None) or preferredencoding()
1059 byt
= s
.encode(enc
, 'ignore')
1060 out
.buffer.write(byt
)
1066 def bytes_to_intlist(bs
):
1069 if isinstance(bs
[0], int): # Python 3
1072 return [ord(c
) for c
in bs
]
1075 def intlist_to_bytes(xs
):
1078 if isinstance(chr(0), bytes): # Python 2
1079 return ''.join([chr(x
) for x
in xs
])
1084 def get_cachedir(params
={}):
1085 cache_root
= os
.environ
.get('XDG_CACHE_HOME',
1086 os
.path
.expanduser('~/.cache'))
1087 return params
.get('cachedir', os
.path
.join(cache_root
, 'youtube-dl'))
1090 # Cross-platform file locking
1091 if sys
.platform
== 'win32':
1092 import ctypes
.wintypes
1095 class OVERLAPPED(ctypes
.Structure
):
1097 ('Internal', ctypes
.wintypes
.LPVOID
),
1098 ('InternalHigh', ctypes
.wintypes
.LPVOID
),
1099 ('Offset', ctypes
.wintypes
.DWORD
),
1100 ('OffsetHigh', ctypes
.wintypes
.DWORD
),
1101 ('hEvent', ctypes
.wintypes
.HANDLE
),
1104 kernel32
= ctypes
.windll
.kernel32
1105 LockFileEx
= kernel32
.LockFileEx
1106 LockFileEx
.argtypes
= [
1107 ctypes
.wintypes
.HANDLE
, # hFile
1108 ctypes
.wintypes
.DWORD
, # dwFlags
1109 ctypes
.wintypes
.DWORD
, # dwReserved
1110 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
1111 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
1112 ctypes
.POINTER(OVERLAPPED
) # Overlapped
1114 LockFileEx
.restype
= ctypes
.wintypes
.BOOL
1115 UnlockFileEx
= kernel32
.UnlockFileEx
1116 UnlockFileEx
.argtypes
= [
1117 ctypes
.wintypes
.HANDLE
, # hFile
1118 ctypes
.wintypes
.DWORD
, # dwReserved
1119 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockLow
1120 ctypes
.wintypes
.DWORD
, # nNumberOfBytesToLockHigh
1121 ctypes
.POINTER(OVERLAPPED
) # Overlapped
1123 UnlockFileEx
.restype
= ctypes
.wintypes
.BOOL
1124 whole_low
= 0xffffffff
1125 whole_high
= 0x7fffffff
1127 def _lock_file(f
, exclusive
):
1128 overlapped
= OVERLAPPED()
1129 overlapped
.Offset
= 0
1130 overlapped
.OffsetHigh
= 0
1131 overlapped
.hEvent
= 0
1132 f
._lock
_file
_overlapped
_p
= ctypes
.pointer(overlapped
)
1133 handle
= msvcrt
.get_osfhandle(f
.fileno())
1134 if not LockFileEx(handle
, 0x2 if exclusive
else 0x0, 0,
1135 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
1136 raise OSError('Locking file failed: %r' % ctypes
.FormatError())
1138 def _unlock_file(f
):
1139 assert f
._lock
_file
_overlapped
_p
1140 handle
= msvcrt
.get_osfhandle(f
.fileno())
1141 if not UnlockFileEx(handle
, 0,
1142 whole_low
, whole_high
, f
._lock
_file
_overlapped
_p
):
1143 raise OSError('Unlocking file failed: %r' % ctypes
.FormatError())
1148 def _lock_file(f
, exclusive
):
1149 fcntl
.lockf(f
, fcntl
.LOCK_EX
if exclusive
else fcntl
.LOCK_SH
)
1151 def _unlock_file(f
):
1152 fcntl
.lockf(f
, fcntl
.LOCK_UN
)
1155 class locked_file(object):
1156 def __init__(self
, filename
, mode
, encoding
=None):
1157 assert mode
in ['r', 'a', 'w']
1158 self
.f
= io
.open(filename
, mode
, encoding
=encoding
)
1161 def __enter__(self
):
1162 exclusive
= self
.mode
!= 'r'
1164 _lock_file(self
.f
, exclusive
)
1170 def __exit__(self
, etype
, value
, traceback
):
1172 _unlock_file(self
.f
)
1179 def write(self
, *args
):
1180 return self
.f
.write(*args
)
1182 def read(self
, *args
):
1183 return self
.f
.read(*args
)
1186 def shell_quote(args
):
1188 encoding
= sys
.getfilesystemencoding()
1189 if encoding
is None:
1192 if isinstance(a
, bytes):
1193 # We may get a filename encoded with 'encodeFilename'
1194 a
= a
.decode(encoding
)
1195 quoted_args
.append(pipes
.quote(a
))
1196 return u
' '.join(quoted_args
)
1199 def takewhile_inclusive(pred
, seq
):
1200 """ Like itertools.takewhile, but include the latest evaluated element
1201 (the first element so that Not pred(e)) """
1208 def smuggle_url(url
, data
):
1209 """ Pass additional data in a URL for internal use. """
1211 sdata
= compat_urllib_parse
.urlencode(
1212 {u'__youtubedl_smuggle': json.dumps(data)}
)
1213 return url
+ u
'#' + sdata
1216 def unsmuggle_url(smug_url
, default
=None):
1217 if not '#__youtubedl_smuggle' in smug_url
:
1218 return smug_url
, default
1219 url
, _
, sdata
= smug_url
.rpartition(u
'#')
1220 jsond
= compat_parse_qs(sdata
)[u
'__youtubedl_smuggle'][0]
1221 data
= json
.loads(jsond
)
1225 def format_bytes(bytes):
1228 if type(bytes) is str:
1229 bytes = float(bytes)
1233 exponent
= int(math
.log(bytes, 1024.0))
1234 suffix
= [u
'B', u
'KiB', u
'MiB', u
'GiB', u
'TiB', u
'PiB', u
'EiB', u
'ZiB', u
'YiB'][exponent
]
1235 converted
= float(bytes) / float(1024 ** exponent
)
1236 return u
'%.2f%s' % (converted
, suffix
)
1239 def get_term_width():
1240 columns
= os
.environ
.get('COLUMNS', None)
1245 sp
= subprocess
.Popen(
1247 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
1248 out
, err
= sp
.communicate()
1249 return int(out
.split()[1])
1255 def month_by_name(name
):
1256 """ Return the number of a month by (locale-independently) English name """
1259 u
'January', u
'February', u
'March', u
'April', u
'May', u
'June',
1260 u
'July', u
'August', u
'September', u
'October', u
'November', u
'December']
1262 return ENGLISH_NAMES
.index(name
) + 1
1267 def fix_xml_ampersands(xml_str
):
1268 """Replace all the '&' by '&' in XML"""
1270 r
'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1275 def setproctitle(title
):
1276 assert isinstance(title
, compat_str
)
1278 libc
= ctypes
.cdll
.LoadLibrary("libc.so.6")
1281 title_bytes
= title
.encode('utf-8')
1282 buf
= ctypes
.create_string_buffer(len(title_bytes
))
1283 buf
.value
= title_bytes
1285 libc
.prctl(15, buf
, 0, 0, 0)
1286 except AttributeError:
1287 return # Strange libc, just skip this
1290 def remove_start(s
, start
):
1291 if s
.startswith(start
):
1292 return s
[len(start
):]
1296 def remove_end(s
, end
):
1298 return s
[:-len(end
)]
1302 def url_basename(url
):
1303 path
= compat_urlparse
.urlparse(url
).path
1304 return path
.strip(u
'/').split(u
'/')[-1]
1307 class HEADRequest(compat_urllib_request
.Request
):
1308 def get_method(self
):
1312 def int_or_none(v
, scale
=1, default
=None, get_attr
=None, invscale
=1):
1315 v
= getattr(v
, get_attr
, None)
1318 return default
if v
is None else (int(v
) * invscale
// scale
)
1321 def str_or_none(v
, default
=None):
1322 return default
if v
is None else compat_str(v
)
1325 def str_to_int(int_str
):
1328 int_str
= re
.sub(r
'[,\.]', u
'', int_str
)
1332 def float_or_none(v
, scale
=1, invscale
=1, default
=None):
1333 return default
if v
is None else (float(v
) * invscale
/ scale
)
1336 def parse_duration(s
):
1341 r
'(?:(?:(?P<hours>[0-9]+)[:h])?(?P<mins>[0-9]+)[:m])?(?P<secs>[0-9]+)s?(?::[0-9]+)?(?P<ms>\.[0-9]+)?$', s
)
1344 res
= int(m
.group('secs'))
1346 res
+= int(m
.group('mins')) * 60
1347 if m
.group('hours'):
1348 res
+= int(m
.group('hours')) * 60 * 60
1350 res
+= float(m
.group('ms'))
1354 def prepend_extension(filename
, ext
):
1355 name
, real_ext
= os
.path
.splitext(filename
)
1356 return u
'{0}.{1}{2}'.format(name
, ext
, real_ext
)
1359 def check_executable(exe
, args
=[]):
1360 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1361 args can be a list of arguments for a short output (like -version) """
1363 subprocess
.Popen([exe
] + args
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
).communicate()
1369 class PagedList(object):
1370 def __init__(self
, pagefunc
, pagesize
):
1371 self
._pagefunc
= pagefunc
1372 self
._pagesize
= pagesize
1375 # This is only useful for tests
1376 return len(self
.getslice())
1378 def getslice(self
, start
=0, end
=None):
1380 for pagenum
in itertools
.count(start
// self
._pagesize
):
1381 firstid
= pagenum
* self
._pagesize
1382 nextfirstid
= pagenum
* self
._pagesize
+ self
._pagesize
1383 if start
>= nextfirstid
:
1386 page_results
= list(self
._pagefunc
(pagenum
))
1389 start
% self
._pagesize
1390 if firstid
<= start
< nextfirstid
1394 ((end
- 1) % self
._pagesize
) + 1
1395 if (end
is not None and firstid
<= end
<= nextfirstid
)
1398 if startv
!= 0 or endv
is not None:
1399 page_results
= page_results
[startv
:endv
]
1400 res
.extend(page_results
)
1402 # A little optimization - if current page is not "full", ie. does
1403 # not contain page_size videos then we can assume that this page
1404 # is the last one - there are no more ids on further pages -
1405 # i.e. no need to query again.
1406 if len(page_results
) + startv
< self
._pagesize
:
1409 # If we got the whole page, but the next page is not interesting,
1410 # break out early as well
1411 if end
== nextfirstid
:
1416 def uppercase_escape(s
):
1417 unicode_escape
= codecs
.getdecoder('unicode_escape')
1419 r
'\\U[0-9a-fA-F]{8}',
1420 lambda m
: unicode_escape(m
.group(0))[0],
1424 struct
.pack(u
'!I', 0)
1426 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1427 def struct_pack(spec
, *args
):
1428 if isinstance(spec
, compat_str
):
1429 spec
= spec
.encode('ascii')
1430 return struct
.pack(spec
, *args
)
1432 def struct_unpack(spec
, *args
):
1433 if isinstance(spec
, compat_str
):
1434 spec
= spec
.encode('ascii')
1435 return struct
.unpack(spec
, *args
)
1437 struct_pack
= struct
.pack
1438 struct_unpack
= struct
.unpack
1441 def read_batch_urls(batch_fd
):
1443 if not isinstance(url
, compat_str
):
1444 url
= url
.decode('utf-8', 'replace')
1445 BOM_UTF8
= u
'\xef\xbb\xbf'
1446 if url
.startswith(BOM_UTF8
):
1447 url
= url
[len(BOM_UTF8
):]
1449 if url
.startswith(('#', ';', ']')):
1453 with contextlib
.closing(batch_fd
) as fd
:
1454 return [url
for url
in map(fixup
, fd
) if url
]
1457 def urlencode_postdata(*args
, **kargs
):
1458 return compat_urllib_parse
.urlencode(*args
, **kargs
).encode('ascii')
1462 etree_iter
= xml
.etree
.ElementTree
.Element
.iter
1463 except AttributeError: # Python <=2.6
1464 etree_iter
= lambda n
: n
.findall('.//*')
1468 class TreeBuilder(xml
.etree
.ElementTree
.TreeBuilder
):
1469 def doctype(self
, name
, pubid
, system
):
1470 pass # Ignore doctypes
1472 parser
= xml
.etree
.ElementTree
.XMLParser(target
=TreeBuilder())
1473 kwargs
= {'parser': parser}
if sys
.version_info
>= (2, 7) else {}
1474 tree
= xml
.etree
.ElementTree
.XML(s
.encode('utf-8'), **kwargs
)
1475 # Fix up XML parser in Python 2.x
1476 if sys
.version_info
< (3, 0):
1477 for n
in etree_iter(tree
):
1478 if n
.text
is not None:
1479 if not isinstance(n
.text
, compat_str
):
1480 n
.text
= n
.text
.decode('utf-8')
1484 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
1485 def compat_getpass(prompt
, *args
, **kwargs
):
1486 if isinstance(prompt
, compat_str
):
1487 prompt
= prompt
.encode(preferredencoding())
1488 return getpass
.getpass(prompt
, *args
, **kwargs
)
1490 compat_getpass
= getpass
.getpass
1502 def strip_jsonp(code
):
1503 return re
.sub(r
'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?\s*$', r
'\1', code
)
1506 def js_to_json(code
):
1509 if key
.startswith("'"):
1510 assert key
.endswith("'")
1511 assert '"' not in key
1512 key
= '"%s"' % key
[1:-1]
1513 elif not key
.startswith('"'):
1517 if value
.startswith("'"):
1518 assert value
.endswith("'")
1519 assert '"' not in value
1520 value
= '"%s"' % value
[1:-1]
1522 return m
.group(1) + key
+ m
.group(3) + value
1524 res
= re
.sub(r
'''(?x)
1526 ("[^"]*"|\'[^
\']*\'|
[a
-z0
-9A
-Z
]+)
1528 ([0-9.]+|true|false|
"[^"]*"|\'[^\']*\'|\[|\{)
1530 res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
1534 def qualities(quality_ids):
1535 """ Get a numeric quality value out of a list of possible values """
1538 return quality_ids.index(qid)
1544 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1547 subprocess_check_output = subprocess.check_output
1548 except AttributeError:
1549 def subprocess_check_output(*args, **kwargs):
1550 assert 'input' not in kwargs
1551 p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
1552 output, _ = p.communicate()
1555 raise subprocess.CalledProcessError(ret, p.args, output=output)