2 # -*- coding: utf-8 -*-
4 from __future__
import absolute_import
, unicode_literals
33 compat_get_terminal_size
,
38 compat_urllib_request
,
57 PerRequestProxyHandler
,
68 UnavailableVideoError
,
78 from .cache
import Cache
79 from .extractor
import get_info_extractor
, gen_extractors
80 from .downloader
import get_suitable_downloader
81 from .downloader
.rtmp
import rtmpdump_version
82 from .postprocessor
import (
84 FFmpegFixupStretchedPP
,
89 from .version
import __version__
92 class YoutubeDL(object):
95 YoutubeDL objects are the ones responsible of downloading the
96 actual video file and writing it to disk if the user has requested
97 it, among some other tasks. In most cases there should be one per
98 program. As, given a video URL, the downloader doesn't know how to
99 extract all the needed information, task that InfoExtractors do, it
100 has to pass the URL to one of them.
102 For this, YoutubeDL objects have a method that allows
103 InfoExtractors to be registered in a given order. When it is passed
104 a URL, the YoutubeDL object handles it to the first InfoExtractor it
105 finds that reports being able to handle it. The InfoExtractor extracts
106 all the information about the video or videos the URL refers to, and
107 YoutubeDL process the extracted information, possibly using a File
108 Downloader to download the video.
110 YoutubeDL objects accept a lot of parameters. In order not to saturate
111 the object constructor with arguments, it receives a dictionary of
112 options instead. These options are available through the params
113 attribute for the InfoExtractors to use. The YoutubeDL also
114 registers itself as the downloader in charge for the InfoExtractors
115 that are added to it, so this is a "mutual registration".
119 username: Username for authentication purposes.
120 password: Password for authentication purposes.
121 videopassword: Password for acces a video.
122 usenetrc: Use netrc for authentication instead.
123 verbose: Print additional info to stdout.
124 quiet: Do not print messages to stdout.
125 no_warnings: Do not print out anything for warnings.
126 forceurl: Force printing final URL.
127 forcetitle: Force printing title.
128 forceid: Force printing ID.
129 forcethumbnail: Force printing thumbnail URL.
130 forcedescription: Force printing description.
131 forcefilename: Force printing final filename.
132 forceduration: Force printing duration.
133 forcejson: Force printing info_dict as JSON.
134 dump_single_json: Force printing the info_dict of the whole playlist
135 (or video) as a single JSON line.
136 simulate: Do not download the video files.
137 format: Video format code. See options.py for more information.
138 format_limit: Highest quality format to try.
139 outtmpl: Template for output names.
140 restrictfilenames: Do not allow "&" and spaces in file names
141 ignoreerrors: Do not stop on download errors.
142 nooverwrites: Prevent overwriting files.
143 playliststart: Playlist item to start at.
144 playlistend: Playlist item to end at.
145 playlist_items: Specific indices of playlist to download.
146 playlistreverse: Download playlist items in reverse order.
147 matchtitle: Download only matching titles.
148 rejecttitle: Reject downloads for matching titles.
149 logger: Log messages to a logging.Logger instance.
150 logtostderr: Log messages to stderr instead of stdout.
151 writedescription: Write the video description to a .description file
152 writeinfojson: Write the video description to a .info.json file
153 writeannotations: Write the video annotations to a .annotations.xml file
154 writethumbnail: Write the thumbnail image to a file
155 write_all_thumbnails: Write all thumbnail formats to files
156 writesubtitles: Write the video subtitles to a file
157 writeautomaticsub: Write the automatic subtitles to a file
158 allsubtitles: Downloads all the subtitles of the video
159 (requires writesubtitles or writeautomaticsub)
160 listsubtitles: Lists all available subtitles for the video
161 subtitlesformat: The format code for subtitles
162 subtitleslangs: List of languages of the subtitles to download
163 keepvideo: Keep the video file after post-processing
164 daterange: A DateRange object, download only if the upload_date is in the range.
165 skip_download: Skip the actual download of the video file
166 cachedir: Location of the cache files in the filesystem.
167 False to disable filesystem cache.
168 noplaylist: Download single video instead of a playlist if in doubt.
169 age_limit: An integer representing the user's age in years.
170 Unsuitable videos for the given age are skipped.
171 min_views: An integer representing the minimum view count the video
172 must have in order to not be skipped.
173 Videos without view count information are always
174 downloaded. None for no limit.
175 max_views: An integer representing the maximum view count.
176 Videos that are more popular than that are not
178 Videos without view count information are always
179 downloaded. None for no limit.
180 download_archive: File name of a file where all downloads are recorded.
181 Videos already present in the file are not downloaded
183 cookiefile: File name where cookies should be read from and dumped to.
184 nocheckcertificate:Do not verify SSL certificates
185 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
186 At the moment, this is only supported by YouTube.
187 proxy: URL of the proxy server to use
188 cn_verification_proxy: URL of the proxy to use for IP address verification
189 on Chinese sites. (Experimental)
190 socket_timeout: Time to wait for unresponsive hosts, in seconds
191 bidi_workaround: Work around buggy terminals without bidirectional text
192 support, using fridibi
193 debug_printtraffic:Print out sent and received HTTP traffic
194 include_ads: Download ads as well
195 default_search: Prepend this string if an input url is not valid.
196 'auto' for elaborate guessing
197 encoding: Use this encoding instead of the system-specified.
198 extract_flat: Do not resolve URLs, return the immediate result.
199 Pass in 'in_playlist' to only show this behavior for
201 postprocessors: A list of dictionaries, each with an entry
202 * key: The name of the postprocessor. See
203 youtube_dl/postprocessor/__init__.py for a list.
204 as well as any further keyword arguments for the
206 progress_hooks: A list of functions that get called on download
207 progress, with a dictionary with the entries
208 * status: One of "downloading", "error", or "finished".
209 Check this first and ignore unknown values.
211 If status is one of "downloading", or "finished", the
212 following properties may also be present:
213 * filename: The final filename (always present)
214 * tmpfilename: The filename we're currently writing to
215 * downloaded_bytes: Bytes on disk
216 * total_bytes: Size of the whole file, None if unknown
217 * total_bytes_estimate: Guess of the eventual file size,
219 * elapsed: The number of seconds since download started.
220 * eta: The estimated time in seconds, None if unknown
221 * speed: The download speed in bytes/second, None if
223 * fragment_index: The counter of the currently
224 downloaded video fragment.
225 * fragment_count: The number of fragments (= individual
226 files that will be merged)
228 Progress hooks are guaranteed to be called at least once
229 (with status "finished") if the download is successful.
230 merge_output_format: Extension to use when merging formats.
231 fixup: Automatically correct known faults of the file.
233 - "never": do nothing
234 - "warn": only emit a warning
235 - "detect_or_warn": check whether we can do anything
236 about it, warn otherwise (default)
237 source_address: (Experimental) Client-side IP address to bind to.
238 call_home: Boolean, true iff we are allowed to contact the
239 youtube-dl servers for debugging.
240 sleep_interval: Number of seconds to sleep before each download.
241 listformats: Print an overview of available video formats and exit.
242 list_thumbnails: Print a table of all thumbnails and exit.
243 match_filter: A function that gets called with the info_dict of
245 If it returns a message, the video is ignored.
246 If it returns None, the video is downloaded.
247 match_filter_func in utils.py is one example for this.
248 no_color: Do not emit color codes in output.
250 The following options determine which downloader is picked:
251 external_downloader: Executable of the external downloader to call.
252 None or unset for standard (built-in) downloader.
253 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
255 The following parameters are not used by YoutubeDL itself, they are used by
256 the downloader (see youtube_dl/downloader/common.py):
257 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
258 noresizebuffer, retries, continuedl, noprogress, consoletitle,
259 xattr_set_filesize, external_downloader_args.
261 The following options are used by the post processors:
262 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
263 otherwise prefer avconv.
264 exec_cmd: Arbitrary command to run after downloading
270 _download_retcode
= None
271 _num_downloads
= None
274 def __init__(self
, params
=None, auto_init
=True):
275 """Create a FileDownloader object with the given options."""
279 self
._ies
_instances
= {}
281 self
._progress
_hooks
= []
282 self
._download
_retcode
= 0
283 self
._num
_downloads
= 0
284 self
._screen
_file
= [sys
.stdout
, sys
.stderr
][params
.get('logtostderr', False)]
285 self
._err
_file
= sys
.stderr
287 self
.cache
= Cache(self
)
289 if params
.get('bidi_workaround', False):
292 master
, slave
= pty
.openpty()
293 width
= compat_get_terminal_size().columns
297 width_args
= ['-w', str(width
)]
299 stdin
=subprocess
.PIPE
,
301 stderr
=self
._err
_file
)
303 self
._output
_process
= subprocess
.Popen(
304 ['bidiv'] + width_args
, **sp_kwargs
307 self
._output
_process
= subprocess
.Popen(
308 ['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
309 self
._output
_channel
= os
.fdopen(master
, 'rb')
310 except OSError as ose
:
312 self
.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
316 if (sys
.version_info
>= (3,) and sys
.platform
!= 'win32' and
317 sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
318 not params
.get('restrictfilenames', False)):
319 # On Python 3, the Unicode filesystem API will throw errors (#1474)
321 'Assuming --restrict-filenames since file system encoding '
322 'cannot encode all characters. '
323 'Set the LC_ALL environment variable to fix this.')
324 self
.params
['restrictfilenames'] = True
326 if isinstance(params
.get('outtmpl'), bytes):
328 'Parameter outtmpl is bytes, but should be a unicode string. '
329 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
334 self
.print_debug_header()
335 self
.add_default_info_extractors()
337 for pp_def_raw
in self
.params
.get('postprocessors', []):
338 pp_class
= get_postprocessor(pp_def_raw
['key'])
339 pp_def
= dict(pp_def_raw
)
341 pp
= pp_class(self
, **compat_kwargs(pp_def
))
342 self
.add_post_processor(pp
)
344 for ph
in self
.params
.get('progress_hooks', []):
345 self
.add_progress_hook(ph
)
347 def warn_if_short_id(self
, argv
):
348 # short YouTube ID starting with dash?
350 i
for i
, a
in enumerate(argv
)
351 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
355 [a
for i
, a
in enumerate(argv
) if i
not in idxs
] +
356 ['--'] + [argv
[i
] for i
in idxs
]
359 'Long argument string detected. '
360 'Use -- to separate parameters and URLs, like this:\n%s\n' %
361 args_to_str(correct_argv
))
363 def add_info_extractor(self
, ie
):
364 """Add an InfoExtractor object to the end of the list."""
366 self
._ies
_instances
[ie
.ie_key()] = ie
367 ie
.set_downloader(self
)
369 def get_info_extractor(self
, ie_key
):
371 Get an instance of an IE with name ie_key, it will try to get one from
372 the _ies list, if there's no instance it will create a new one and add
373 it to the extractor list.
375 ie
= self
._ies
_instances
.get(ie_key
)
377 ie
= get_info_extractor(ie_key
)()
378 self
.add_info_extractor(ie
)
381 def add_default_info_extractors(self
):
383 Add the InfoExtractors returned by gen_extractors to the end of the list
385 for ie
in gen_extractors():
386 self
.add_info_extractor(ie
)
388 def add_post_processor(self
, pp
):
389 """Add a PostProcessor object to the end of the chain."""
391 pp
.set_downloader(self
)
393 def add_progress_hook(self
, ph
):
394 """Add the progress hook (currently only for the file downloader)"""
395 self
._progress
_hooks
.append(ph
)
397 def _bidi_workaround(self
, message
):
398 if not hasattr(self
, '_output_channel'):
401 assert hasattr(self
, '_output_process')
402 assert isinstance(message
, compat_str
)
403 line_count
= message
.count('\n') + 1
404 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
405 self
._output
_process
.stdin
.flush()
406 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
407 for _
in range(line_count
))
408 return res
[:-len('\n')]
410 def to_screen(self
, message
, skip_eol
=False):
411 """Print message to stdout if not in quiet mode."""
412 return self
.to_stdout(message
, skip_eol
, check_quiet
=True)
414 def _write_string(self
, s
, out
=None):
415 write_string(s
, out
=out
, encoding
=self
.params
.get('encoding'))
417 def to_stdout(self
, message
, skip_eol
=False, check_quiet
=False):
418 """Print message to stdout if not in quiet mode."""
419 if self
.params
.get('logger'):
420 self
.params
['logger'].debug(message
)
421 elif not check_quiet
or not self
.params
.get('quiet', False):
422 message
= self
._bidi
_workaround
(message
)
423 terminator
= ['\n', ''][skip_eol
]
424 output
= message
+ terminator
426 self
._write
_string
(output
, self
._screen
_file
)
428 def to_stderr(self
, message
):
429 """Print message to stderr."""
430 assert isinstance(message
, compat_str
)
431 if self
.params
.get('logger'):
432 self
.params
['logger'].error(message
)
434 message
= self
._bidi
_workaround
(message
)
435 output
= message
+ '\n'
436 self
._write
_string
(output
, self
._err
_file
)
438 def to_console_title(self
, message
):
439 if not self
.params
.get('consoletitle', False):
441 if os
.name
== 'nt' and ctypes
.windll
.kernel32
.GetConsoleWindow():
442 # c_wchar_p() might not be necessary if `message` is
443 # already of type unicode()
444 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
445 elif 'TERM' in os
.environ
:
446 self
._write
_string
('\033]0;%s\007' % message
, self
._screen
_file
)
448 def save_console_title(self
):
449 if not self
.params
.get('consoletitle', False):
451 if 'TERM' in os
.environ
:
452 # Save the title on stack
453 self
._write
_string
('\033[22;0t', self
._screen
_file
)
455 def restore_console_title(self
):
456 if not self
.params
.get('consoletitle', False):
458 if 'TERM' in os
.environ
:
459 # Restore the title from stack
460 self
._write
_string
('\033[23;0t', self
._screen
_file
)
463 self
.save_console_title()
466 def __exit__(self
, *args
):
467 self
.restore_console_title()
469 if self
.params
.get('cookiefile') is not None:
470 self
.cookiejar
.save()
472 def trouble(self
, message
=None, tb
=None):
473 """Determine action to take when a download problem appears.
475 Depending on if the downloader has been configured to ignore
476 download errors or not, this method may throw an exception or
477 not when errors are found, after printing the message.
479 tb, if given, is additional traceback information.
481 if message
is not None:
482 self
.to_stderr(message
)
483 if self
.params
.get('verbose'):
485 if sys
.exc_info()[0]: # if .trouble has been called from an except block
487 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
488 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
489 tb
+= compat_str(traceback
.format_exc())
491 tb_data
= traceback
.format_list(traceback
.extract_stack())
492 tb
= ''.join(tb_data
)
494 if not self
.params
.get('ignoreerrors', False):
495 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
496 exc_info
= sys
.exc_info()[1].exc_info
498 exc_info
= sys
.exc_info()
499 raise DownloadError(message
, exc_info
)
500 self
._download
_retcode
= 1
502 def report_warning(self
, message
):
504 Print the message to stderr, it will be prefixed with 'WARNING:'
505 If stderr is a tty file the 'WARNING:' will be colored
507 if self
.params
.get('logger') is not None:
508 self
.params
['logger'].warning(message
)
510 if self
.params
.get('no_warnings'):
512 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and os
.name
!= 'nt':
513 _msg_header
= '\033[0;33mWARNING:\033[0m'
515 _msg_header
= 'WARNING:'
516 warning_message
= '%s %s' % (_msg_header
, message
)
517 self
.to_stderr(warning_message
)
519 def report_error(self
, message
, tb
=None):
521 Do the same as trouble, but prefixes the message with 'ERROR:', colored
522 in red if stderr is a tty file.
524 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and os
.name
!= 'nt':
525 _msg_header
= '\033[0;31mERROR:\033[0m'
527 _msg_header
= 'ERROR:'
528 error_message
= '%s %s' % (_msg_header
, message
)
529 self
.trouble(error_message
, tb
)
531 def report_file_already_downloaded(self
, file_name
):
532 """Report file has already been fully downloaded."""
534 self
.to_screen('[download] %s has already been downloaded' % file_name
)
535 except UnicodeEncodeError:
536 self
.to_screen('[download] The file has already been downloaded')
538 def prepare_filename(self
, info_dict
):
539 """Generate the output filename."""
541 template_dict
= dict(info_dict
)
543 template_dict
['epoch'] = int(time
.time())
544 autonumber_size
= self
.params
.get('autonumber_size')
545 if autonumber_size
is None:
547 autonumber_templ
= '%0' + str(autonumber_size
) + 'd'
548 template_dict
['autonumber'] = autonumber_templ
% self
._num
_downloads
549 if template_dict
.get('playlist_index') is not None:
550 template_dict
['playlist_index'] = '%0*d' % (len(str(template_dict
['n_entries'])), template_dict
['playlist_index'])
551 if template_dict
.get('resolution') is None:
552 if template_dict
.get('width') and template_dict
.get('height'):
553 template_dict
['resolution'] = '%dx%d' % (template_dict
['width'], template_dict
['height'])
554 elif template_dict
.get('height'):
555 template_dict
['resolution'] = '%sp' % template_dict
['height']
556 elif template_dict
.get('width'):
557 template_dict
['resolution'] = '?x%d' % template_dict
['width']
559 sanitize
= lambda k
, v
: sanitize_filename(
561 restricted
=self
.params
.get('restrictfilenames'),
563 template_dict
= dict((k
, sanitize(k
, v
))
564 for k
, v
in template_dict
.items()
566 template_dict
= collections
.defaultdict(lambda: 'NA', template_dict
)
568 outtmpl
= sanitize_path(self
.params
.get('outtmpl', DEFAULT_OUTTMPL
))
569 tmpl
= compat_expanduser(outtmpl
)
570 filename
= tmpl
% template_dict
571 # Temporary fix for #4787
572 # 'Treat' all problem characters by passing filename through preferredencoding
573 # to workaround encoding issues with subprocess on python2 @ Windows
574 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
575 filename
= encodeFilename(filename
, True).decode(preferredencoding())
577 except ValueError as err
:
578 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
581 def _match_entry(self
, info_dict
, incomplete
):
582 """ Returns None iff the file should be downloaded """
584 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
585 if 'title' in info_dict
:
586 # This can happen when we're just evaluating the playlist
587 title
= info_dict
['title']
588 matchtitle
= self
.params
.get('matchtitle', False)
590 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
591 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
592 rejecttitle
= self
.params
.get('rejecttitle', False)
594 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
595 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
596 date
= info_dict
.get('upload_date', None)
598 dateRange
= self
.params
.get('daterange', DateRange())
599 if date
not in dateRange
:
600 return '%s upload date is not in range %s' % (date_from_str(date
).isoformat(), dateRange
)
601 view_count
= info_dict
.get('view_count', None)
602 if view_count
is not None:
603 min_views
= self
.params
.get('min_views')
604 if min_views
is not None and view_count
< min_views
:
605 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
606 max_views
= self
.params
.get('max_views')
607 if max_views
is not None and view_count
> max_views
:
608 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
609 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
610 return 'Skipping "%s" because it is age restricted' % video_title
611 if self
.in_download_archive(info_dict
):
612 return '%s has already been recorded in archive' % video_title
615 match_filter
= self
.params
.get('match_filter')
616 if match_filter
is not None:
617 ret
= match_filter(info_dict
)
624 def add_extra_info(info_dict
, extra_info
):
625 '''Set the keys from extra_info in info dict if they are missing'''
626 for key
, value
in extra_info
.items():
627 info_dict
.setdefault(key
, value
)
629 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
={},
632 Returns a list with a dictionary for each video we find.
633 If 'download', also downloads the videos.
634 extra_info is a dict containing the extra values to add to each result
638 ies
= [self
.get_info_extractor(ie_key
)]
643 if not ie
.suitable(url
):
647 self
.report_warning('The program functionality for this site has been marked as broken, '
648 'and will probably not work.')
651 ie_result
= ie
.extract(url
)
652 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
654 if isinstance(ie_result
, list):
655 # Backwards compatibility: old IE result format
657 '_type': 'compat_list',
658 'entries': ie_result
,
660 self
.add_default_extra_info(ie_result
, ie
, url
)
662 return self
.process_ie_result(ie_result
, download
, extra_info
)
665 except ExtractorError
as de
: # An error we somewhat expected
666 self
.report_error(compat_str(de
), de
.format_traceback())
668 except MaxDownloadsReached
:
670 except Exception as e
:
671 if self
.params
.get('ignoreerrors', False):
672 self
.report_error(compat_str(e
), tb
=compat_str(traceback
.format_exc()))
677 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
679 def add_default_extra_info(self
, ie_result
, ie
, url
):
680 self
.add_extra_info(ie_result
, {
681 'extractor': ie
.IE_NAME
,
683 'webpage_url_basename': url_basename(url
),
684 'extractor_key': ie
.ie_key(),
687 def process_ie_result(self
, ie_result
, download
=True, extra_info
={}):
689 Take the result of the ie(may be modified) and resolve all unresolved
690 references (URLs, playlist items).
692 It will also download the videos if 'download'.
693 Returns the resolved ie_result.
696 result_type
= ie_result
.get('_type', 'video')
698 if result_type
in ('url', 'url_transparent'):
699 extract_flat
= self
.params
.get('extract_flat', False)
700 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
) or
701 extract_flat
is True):
702 if self
.params
.get('forcejson', False):
703 self
.to_stdout(json
.dumps(ie_result
))
706 if result_type
== 'video':
707 self
.add_extra_info(ie_result
, extra_info
)
708 return self
.process_video_result(ie_result
, download
=download
)
709 elif result_type
== 'url':
710 # We have to add extra_info to the results because it may be
711 # contained in a playlist
712 return self
.extract_info(ie_result
['url'],
714 ie_key
=ie_result
.get('ie_key'),
715 extra_info
=extra_info
)
716 elif result_type
== 'url_transparent':
717 # Use the information from the embedding page
718 info
= self
.extract_info(
719 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
720 extra_info
=extra_info
, download
=False, process
=False)
722 force_properties
= dict(
723 (k
, v
) for k
, v
in ie_result
.items() if v
is not None)
724 for f
in ('_type', 'url'):
725 if f
in force_properties
:
726 del force_properties
[f
]
727 new_result
= info
.copy()
728 new_result
.update(force_properties
)
730 assert new_result
.get('_type') != 'url_transparent'
732 return self
.process_ie_result(
733 new_result
, download
=download
, extra_info
=extra_info
)
734 elif result_type
== 'playlist' or result_type
== 'multi_video':
735 # We process each entry in the playlist
736 playlist
= ie_result
.get('title', None) or ie_result
.get('id', None)
737 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
739 playlist_results
= []
741 playliststart
= self
.params
.get('playliststart', 1) - 1
742 playlistend
= self
.params
.get('playlistend', None)
743 # For backwards compatibility, interpret -1 as whole list
744 if playlistend
== -1:
747 playlistitems_str
= self
.params
.get('playlist_items', None)
749 if playlistitems_str
is not None:
750 def iter_playlistitems(format
):
751 for string_segment
in format
.split(','):
752 if '-' in string_segment
:
753 start
, end
= string_segment
.split('-')
754 for item
in range(int(start
), int(end
) + 1):
757 yield int(string_segment
)
758 playlistitems
= iter_playlistitems(playlistitems_str
)
760 ie_entries
= ie_result
['entries']
761 if isinstance(ie_entries
, list):
762 n_all_entries
= len(ie_entries
)
764 entries
= [ie_entries
[i
- 1] for i
in playlistitems
]
766 entries
= ie_entries
[playliststart
:playlistend
]
767 n_entries
= len(entries
)
769 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
770 (ie_result
['extractor'], playlist
, n_all_entries
, n_entries
))
771 elif isinstance(ie_entries
, PagedList
):
774 for item
in playlistitems
:
775 entries
.extend(ie_entries
.getslice(
779 entries
= ie_entries
.getslice(
780 playliststart
, playlistend
)
781 n_entries
= len(entries
)
783 "[%s] playlist %s: Downloading %d videos" %
784 (ie_result
['extractor'], playlist
, n_entries
))
787 entry_list
= list(ie_entries
)
788 entries
= [entry_list
[i
- 1] for i
in playlistitems
]
790 entries
= list(itertools
.islice(
791 ie_entries
, playliststart
, playlistend
))
792 n_entries
= len(entries
)
794 "[%s] playlist %s: Downloading %d videos" %
795 (ie_result
['extractor'], playlist
, n_entries
))
797 if self
.params
.get('playlistreverse', False):
798 entries
= entries
[::-1]
800 for i
, entry
in enumerate(entries
, 1):
801 self
.to_screen('[download] Downloading video %s of %s' % (i
, n_entries
))
803 'n_entries': n_entries
,
804 'playlist': playlist
,
805 'playlist_id': ie_result
.get('id'),
806 'playlist_title': ie_result
.get('title'),
807 'playlist_index': i
+ playliststart
,
808 'extractor': ie_result
['extractor'],
809 'webpage_url': ie_result
['webpage_url'],
810 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
811 'extractor_key': ie_result
['extractor_key'],
814 reason
= self
._match
_entry
(entry
, incomplete
=True)
815 if reason
is not None:
816 self
.to_screen('[download] ' + reason
)
819 entry_result
= self
.process_ie_result(entry
,
822 playlist_results
.append(entry_result
)
823 ie_result
['entries'] = playlist_results
825 elif result_type
== 'compat_list':
827 'Extractor %s returned a compat_list result. '
828 'It needs to be updated.' % ie_result
.get('extractor'))
834 'extractor': ie_result
['extractor'],
835 'webpage_url': ie_result
['webpage_url'],
836 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
837 'extractor_key': ie_result
['extractor_key'],
841 ie_result
['entries'] = [
842 self
.process_ie_result(_fixup(r
), download
, extra_info
)
843 for r
in ie_result
['entries']
847 raise Exception('Invalid result type: %s' % result_type
)
849 def _apply_format_filter(self
, format_spec
, available_formats
):
850 " Returns a tuple of the remaining format_spec and filtered formats "
860 operator_rex
= re
.compile(r
'''(?x)\s*\[
861 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
862 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
863 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
865 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
866 m
= operator_rex
.search(format_spec
)
869 comparison_value
= int(m
.group('value'))
871 comparison_value
= parse_filesize(m
.group('value'))
872 if comparison_value
is None:
873 comparison_value
= parse_filesize(m
.group('value') + 'B')
874 if comparison_value
is None:
876 'Invalid value %r in format specification %r' % (
877 m
.group('value'), format_spec
))
878 op
= OPERATORS
[m
.group('op')]
885 str_operator_rex
= re
.compile(r
'''(?x)\s*\[
886 \s*(?P<key>ext|acodec|vcodec|container|protocol)
887 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
888 \s*(?P<value>[a-zA-Z0-9_-]+)
890 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
891 m
= str_operator_rex
.search(format_spec
)
893 comparison_value
= m
.group('value')
894 op
= STR_OPERATORS
[m
.group('op')]
897 raise ValueError('Invalid format specification %r' % format_spec
)
900 actual_value
= f
.get(m
.group('key'))
901 if actual_value
is None:
902 return m
.group('none_inclusive')
903 return op(actual_value
, comparison_value
)
904 new_formats
= [f
for f
in available_formats
if _filter(f
)]
906 new_format_spec
= format_spec
[:-len(m
.group(0))]
907 if not new_format_spec
:
908 new_format_spec
= 'best'
910 return (new_format_spec
, new_formats
)
912 def select_format(self
, format_spec
, available_formats
):
913 while format_spec
.endswith(']'):
914 format_spec
, available_formats
= self
._apply
_format
_filter
(
915 format_spec
, available_formats
)
916 if not available_formats
:
919 if format_spec
== 'best' or format_spec
is None:
920 return available_formats
[-1]
921 elif format_spec
== 'worst':
922 audiovideo_formats
= [
923 f
for f
in available_formats
924 if f
.get('vcodec') != 'none' and f
.get('acodec') != 'none']
925 if audiovideo_formats
:
926 return audiovideo_formats
[0]
927 return available_formats
[0]
928 elif format_spec
== 'bestaudio':
930 f
for f
in available_formats
931 if f
.get('vcodec') == 'none']
933 return audio_formats
[-1]
934 elif format_spec
== 'worstaudio':
936 f
for f
in available_formats
937 if f
.get('vcodec') == 'none']
939 return audio_formats
[0]
940 elif format_spec
== 'bestvideo':
942 f
for f
in available_formats
943 if f
.get('acodec') == 'none']
945 return video_formats
[-1]
946 elif format_spec
== 'worstvideo':
948 f
for f
in available_formats
949 if f
.get('acodec') == 'none']
951 return video_formats
[0]
953 extensions
= ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
954 if format_spec
in extensions
:
955 filter_f
= lambda f
: f
['ext'] == format_spec
957 filter_f
= lambda f
: f
['format_id'] == format_spec
958 matches
= list(filter(filter_f
, available_formats
))
963 def _calc_headers(self
, info_dict
):
964 res
= std_headers
.copy()
966 add_headers
= info_dict
.get('http_headers')
968 res
.update(add_headers
)
970 cookies
= self
._calc
_cookies
(info_dict
)
972 res
['Cookie'] = cookies
976 def _calc_cookies(self
, info_dict
):
977 pr
= compat_urllib_request
.Request(info_dict
['url'])
978 self
.cookiejar
.add_cookie_header(pr
)
979 return pr
.get_header('Cookie')
981 def process_video_result(self
, info_dict
, download
=True):
982 assert info_dict
.get('_type', 'video') == 'video'
984 if 'id' not in info_dict
:
985 raise ExtractorError('Missing "id" field in extractor result')
986 if 'title' not in info_dict
:
987 raise ExtractorError('Missing "title" field in extractor result')
989 if 'playlist' not in info_dict
:
990 # It isn't part of a playlist
991 info_dict
['playlist'] = None
992 info_dict
['playlist_index'] = None
994 thumbnails
= info_dict
.get('thumbnails')
995 if thumbnails
is None:
996 thumbnail
= info_dict
.get('thumbnail')
998 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
1000 thumbnails
.sort(key
=lambda t
: (
1001 t
.get('preference'), t
.get('width'), t
.get('height'),
1002 t
.get('id'), t
.get('url')))
1003 for i
, t
in enumerate(thumbnails
):
1004 if 'width' in t
and 'height' in t
:
1005 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
1006 if t
.get('id') is None:
1009 if thumbnails
and 'thumbnail' not in info_dict
:
1010 info_dict
['thumbnail'] = thumbnails
[-1]['url']
1012 if 'display_id' not in info_dict
and 'id' in info_dict
:
1013 info_dict
['display_id'] = info_dict
['id']
1015 if info_dict
.get('upload_date') is None and info_dict
.get('timestamp') is not None:
1016 # Working around negative timestamps in Windows
1017 # (see http://bugs.python.org/issue1646728)
1018 if info_dict
['timestamp'] < 0 and os
.name
== 'nt':
1019 info_dict
['timestamp'] = 0
1020 upload_date
= datetime
.datetime
.utcfromtimestamp(
1021 info_dict
['timestamp'])
1022 info_dict
['upload_date'] = upload_date
.strftime('%Y%m%d')
1024 if self
.params
.get('listsubtitles', False):
1025 if 'automatic_captions' in info_dict
:
1026 self
.list_subtitles(info_dict
['id'], info_dict
.get('automatic_captions'), 'automatic captions')
1027 self
.list_subtitles(info_dict
['id'], info_dict
.get('subtitles'), 'subtitles')
1029 info_dict
['requested_subtitles'] = self
.process_subtitles(
1030 info_dict
['id'], info_dict
.get('subtitles'),
1031 info_dict
.get('automatic_captions'))
1033 # This extractors handle format selection themselves
1034 if info_dict
['extractor'] in ['Youku']:
1036 self
.process_info(info_dict
)
1039 # We now pick which formats have to be downloaded
1040 if info_dict
.get('formats') is None:
1041 # There's only one format available
1042 formats
= [info_dict
]
1044 formats
= info_dict
['formats']
1047 raise ExtractorError('No video formats found!')
1049 # We check that all the formats have the format and format_id fields
1050 for i
, format
in enumerate(formats
):
1051 if 'url' not in format
:
1052 raise ExtractorError('Missing "url" key in result (index %d)' % i
)
1054 if format
.get('format_id') is None:
1055 format
['format_id'] = compat_str(i
)
1056 if format
.get('format') is None:
1057 format
['format'] = '{id} - {res}{note}'.format(
1058 id=format
['format_id'],
1059 res
=self
.format_resolution(format
),
1060 note
=' ({0})'.format(format
['format_note']) if format
.get('format_note') is not None else '',
1062 # Automatically determine file extension if missing
1063 if 'ext' not in format
:
1064 format
['ext'] = determine_ext(format
['url']).lower()
1065 # Add HTTP headers, so that external programs can use them from the
1067 full_format_info
= info_dict
.copy()
1068 full_format_info
.update(format
)
1069 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
1071 format_limit
= self
.params
.get('format_limit', None)
1073 formats
= list(takewhile_inclusive(
1074 lambda f
: f
['format_id'] != format_limit
, formats
1077 # TODO Central sorting goes here
1079 if formats
[0] is not info_dict
:
1080 # only set the 'formats' fields if the original info_dict list them
1081 # otherwise we end up with a circular reference, the first (and unique)
1082 # element in the 'formats' field in info_dict is info_dict itself,
1083 # wich can't be exported to json
1084 info_dict
['formats'] = formats
1085 if self
.params
.get('listformats'):
1086 self
.list_formats(info_dict
)
1088 if self
.params
.get('list_thumbnails'):
1089 self
.list_thumbnails(info_dict
)
1092 req_format
= self
.params
.get('format')
1093 if req_format
is None:
1094 req_format_list
= []
1095 if info_dict
['extractor'] in ['youtube', 'ted'] and FFmpegMergerPP(self
).available
:
1096 req_format_list
.append('bestvideo+bestaudio')
1097 req_format_list
.append('best')
1098 req_format
= '/'.join(req_format_list
)
1099 formats_to_download
= []
1100 if req_format
== 'all':
1101 formats_to_download
= formats
1103 for rfstr
in req_format
.split(','):
1104 # We can accept formats requested in the format: 34/5/best, we pick
1105 # the first that is available, starting from left
1106 req_formats
= rfstr
.split('/')
1107 for rf
in req_formats
:
1108 if re
.match(r
'.+?\+.+?', rf
) is not None:
1109 # Two formats have been requested like '137+139'
1110 format_1
, format_2
= rf
.split('+')
1111 formats_info
= (self
.select_format(format_1
, formats
),
1112 self
.select_format(format_2
, formats
))
1113 if all(formats_info
):
1114 # The first format must contain the video and the
1116 if formats_info
[0].get('vcodec') == 'none':
1117 self
.report_error('The first format must '
1118 'contain the video, try using '
1119 '"-f %s+%s"' % (format_2
, format_1
))
1122 formats_info
[0]['ext']
1123 if self
.params
.get('merge_output_format') is None
1124 else self
.params
['merge_output_format'])
1126 'requested_formats': formats_info
,
1127 'format': '%s+%s' % (formats_info
[0].get('format'),
1128 formats_info
[1].get('format')),
1129 'format_id': '%s+%s' % (formats_info
[0].get('format_id'),
1130 formats_info
[1].get('format_id')),
1131 'width': formats_info
[0].get('width'),
1132 'height': formats_info
[0].get('height'),
1133 'resolution': formats_info
[0].get('resolution'),
1134 'fps': formats_info
[0].get('fps'),
1135 'vcodec': formats_info
[0].get('vcodec'),
1136 'vbr': formats_info
[0].get('vbr'),
1137 'stretched_ratio': formats_info
[0].get('stretched_ratio'),
1138 'acodec': formats_info
[1].get('acodec'),
1139 'abr': formats_info
[1].get('abr'),
1143 selected_format
= None
1145 selected_format
= self
.select_format(rf
, formats
)
1146 if selected_format
is not None:
1147 formats_to_download
.append(selected_format
)
1149 if not formats_to_download
:
1150 raise ExtractorError('requested format not available',
1154 if len(formats_to_download
) > 1:
1155 self
.to_screen('[info] %s: downloading video in %s formats' % (info_dict
['id'], len(formats_to_download
)))
1156 for format
in formats_to_download
:
1157 new_info
= dict(info_dict
)
1158 new_info
.update(format
)
1159 self
.process_info(new_info
)
1160 # We update the info dict with the best quality format (backwards compatibility)
1161 info_dict
.update(formats_to_download
[-1])
1164 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
1165 """Select the requested subtitles and their format"""
1167 if normal_subtitles
and self
.params
.get('writesubtitles'):
1168 available_subs
.update(normal_subtitles
)
1169 if automatic_captions
and self
.params
.get('writeautomaticsub'):
1170 for lang
, cap_info
in automatic_captions
.items():
1171 if lang
not in available_subs
:
1172 available_subs
[lang
] = cap_info
1174 if (not self
.params
.get('writesubtitles') and not
1175 self
.params
.get('writeautomaticsub') or not
1179 if self
.params
.get('allsubtitles', False):
1180 requested_langs
= available_subs
.keys()
1182 if self
.params
.get('subtitleslangs', False):
1183 requested_langs
= self
.params
.get('subtitleslangs')
1184 elif 'en' in available_subs
:
1185 requested_langs
= ['en']
1187 requested_langs
= [list(available_subs
.keys())[0]]
1189 formats_query
= self
.params
.get('subtitlesformat', 'best')
1190 formats_preference
= formats_query
.split('/') if formats_query
else []
1192 for lang
in requested_langs
:
1193 formats
= available_subs
.get(lang
)
1195 self
.report_warning('%s subtitles not available for %s' % (lang
, video_id
))
1197 for ext
in formats_preference
:
1201 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
1207 self
.report_warning(
1208 'No subtitle format found matching "%s" for language %s, '
1209 'using %s' % (formats_query
, lang
, f
['ext']))
1213 def process_info(self
, info_dict
):
1214 """Process a single resolved IE result."""
1216 assert info_dict
.get('_type', 'video') == 'video'
1218 max_downloads
= self
.params
.get('max_downloads')
1219 if max_downloads
is not None:
1220 if self
._num
_downloads
>= int(max_downloads
):
1221 raise MaxDownloadsReached()
1223 info_dict
['fulltitle'] = info_dict
['title']
1224 if len(info_dict
['title']) > 200:
1225 info_dict
['title'] = info_dict
['title'][:197] + '...'
1227 if 'format' not in info_dict
:
1228 info_dict
['format'] = info_dict
['ext']
1230 reason
= self
._match
_entry
(info_dict
, incomplete
=False)
1231 if reason
is not None:
1232 self
.to_screen('[download] ' + reason
)
1235 self
._num
_downloads
+= 1
1237 info_dict
['_filename'] = filename
= self
.prepare_filename(info_dict
)
1240 if self
.params
.get('forcetitle', False):
1241 self
.to_stdout(info_dict
['fulltitle'])
1242 if self
.params
.get('forceid', False):
1243 self
.to_stdout(info_dict
['id'])
1244 if self
.params
.get('forceurl', False):
1245 if info_dict
.get('requested_formats') is not None:
1246 for f
in info_dict
['requested_formats']:
1247 self
.to_stdout(f
['url'] + f
.get('play_path', ''))
1249 # For RTMP URLs, also include the playpath
1250 self
.to_stdout(info_dict
['url'] + info_dict
.get('play_path', ''))
1251 if self
.params
.get('forcethumbnail', False) and info_dict
.get('thumbnail') is not None:
1252 self
.to_stdout(info_dict
['thumbnail'])
1253 if self
.params
.get('forcedescription', False) and info_dict
.get('description') is not None:
1254 self
.to_stdout(info_dict
['description'])
1255 if self
.params
.get('forcefilename', False) and filename
is not None:
1256 self
.to_stdout(filename
)
1257 if self
.params
.get('forceduration', False) and info_dict
.get('duration') is not None:
1258 self
.to_stdout(formatSeconds(info_dict
['duration']))
1259 if self
.params
.get('forceformat', False):
1260 self
.to_stdout(info_dict
['format'])
1261 if self
.params
.get('forcejson', False):
1262 self
.to_stdout(json
.dumps(info_dict
))
1264 # Do nothing else if in simulate mode
1265 if self
.params
.get('simulate', False):
1268 if filename
is None:
1272 dn
= os
.path
.dirname(sanitize_path(encodeFilename(filename
)))
1273 if dn
and not os
.path
.exists(dn
):
1275 except (OSError, IOError) as err
:
1276 self
.report_error('unable to create directory ' + compat_str(err
))
1279 if self
.params
.get('writedescription', False):
1280 descfn
= filename
+ '.description'
1281 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(descfn
)):
1282 self
.to_screen('[info] Video description is already present')
1283 elif info_dict
.get('description') is None:
1284 self
.report_warning('There\'s no description to write.')
1287 self
.to_screen('[info] Writing video description to: ' + descfn
)
1288 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
1289 descfile
.write(info_dict
['description'])
1290 except (OSError, IOError):
1291 self
.report_error('Cannot write description file ' + descfn
)
1294 if self
.params
.get('writeannotations', False):
1295 annofn
= filename
+ '.annotations.xml'
1296 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(annofn
)):
1297 self
.to_screen('[info] Video annotations are already present')
1300 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
1301 with io
.open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
1302 annofile
.write(info_dict
['annotations'])
1303 except (KeyError, TypeError):
1304 self
.report_warning('There are no annotations to write.')
1305 except (OSError, IOError):
1306 self
.report_error('Cannot write annotations file: ' + annofn
)
1309 subtitles_are_requested
= any([self
.params
.get('writesubtitles', False),
1310 self
.params
.get('writeautomaticsub')])
1312 if subtitles_are_requested
and info_dict
.get('requested_subtitles'):
1313 # subtitles download errors are already managed as troubles in relevant IE
1314 # that way it will silently go on when used with unsupporting IE
1315 subtitles
= info_dict
['requested_subtitles']
1316 ie
= self
.get_info_extractor(info_dict
['extractor_key'])
1317 for sub_lang
, sub_info
in subtitles
.items():
1318 sub_format
= sub_info
['ext']
1319 if sub_info
.get('data') is not None:
1320 sub_data
= sub_info
['data']
1323 sub_data
= ie
._download
_webpage
(
1324 sub_info
['url'], info_dict
['id'], note
=False)
1325 except ExtractorError
as err
:
1326 self
.report_warning('Unable to download subtitle for "%s": %s' %
1327 (sub_lang
, compat_str(err
.cause
)))
1330 sub_filename
= subtitles_filename(filename
, sub_lang
, sub_format
)
1331 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(sub_filename
)):
1332 self
.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang
, sub_format
))
1334 self
.to_screen('[info] Writing video subtitles to: ' + sub_filename
)
1335 with io
.open(encodeFilename(sub_filename
), 'w', encoding
='utf-8') as subfile
:
1336 subfile
.write(sub_data
)
1337 except (OSError, IOError):
1338 self
.report_error('Cannot write subtitles file ' + sub_filename
)
1341 if self
.params
.get('writeinfojson', False):
1342 infofn
= os
.path
.splitext(filename
)[0] + '.info.json'
1343 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(infofn
)):
1344 self
.to_screen('[info] Video description metadata is already present')
1346 self
.to_screen('[info] Writing video description metadata as JSON to: ' + infofn
)
1348 write_json_file(info_dict
, infofn
)
1349 except (OSError, IOError):
1350 self
.report_error('Cannot write metadata to JSON file ' + infofn
)
1353 self
._write
_thumbnails
(info_dict
, filename
)
1355 if not self
.params
.get('skip_download', False):
1358 fd
= get_suitable_downloader(info
, self
.params
)(self
, self
.params
)
1359 for ph
in self
._progress
_hooks
:
1360 fd
.add_progress_hook(ph
)
1361 if self
.params
.get('verbose'):
1362 self
.to_stdout('[debug] Invoking downloader on %r' % info
.get('url'))
1363 return fd
.download(name
, info
)
1365 if info_dict
.get('requested_formats') is not None:
1368 merger
= FFmpegMergerPP(self
)
1369 if not merger
.available
:
1371 self
.report_warning('You have requested multiple '
1372 'formats but ffmpeg or avconv are not installed.'
1373 ' The formats won\'t be merged')
1375 postprocessors
= [merger
]
1377 def compatible_formats(formats
):
1378 video
, audio
= formats
1380 video_ext
, audio_ext
= audio
.get('ext'), video
.get('ext')
1381 if video_ext
and audio_ext
:
1383 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1386 for exts
in COMPATIBLE_EXTS
:
1387 if video_ext
in exts
and audio_ext
in exts
:
1389 # TODO: Check acodec/vcodec
1392 requested_formats
= info_dict
['requested_formats']
1393 if self
.params
.get('merge_output_format') is None and not compatible_formats(requested_formats
):
1394 filename
= os
.path
.splitext(filename
)[0] + '.mkv'
1395 self
.report_warning('You have requested formats uncompatible for merge. '
1396 'The formats will be merged into mkv')
1397 if os
.path
.exists(encodeFilename(filename
)):
1399 '[download] %s has already been downloaded and '
1400 'merged' % filename
)
1402 for f
in requested_formats
:
1403 new_info
= dict(info_dict
)
1405 fname
= self
.prepare_filename(new_info
)
1406 fname
= prepend_extension(fname
, 'f%s' % f
['format_id'])
1407 downloaded
.append(fname
)
1408 partial_success
= dl(fname
, new_info
)
1409 success
= success
and partial_success
1410 info_dict
['__postprocessors'] = postprocessors
1411 info_dict
['__files_to_merge'] = downloaded
1413 # Just a single file
1414 success
= dl(filename
, info_dict
)
1415 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
1416 self
.report_error('unable to download video data: %s' % str(err
))
1418 except (OSError, IOError) as err
:
1419 raise UnavailableVideoError(err
)
1420 except (ContentTooShortError
, ) as err
:
1421 self
.report_error('content too short (expected %s bytes and served %s)' % (err
.expected
, err
.downloaded
))
1426 fixup_policy
= self
.params
.get('fixup')
1427 if fixup_policy
is None:
1428 fixup_policy
= 'detect_or_warn'
1430 stretched_ratio
= info_dict
.get('stretched_ratio')
1431 if stretched_ratio
is not None and stretched_ratio
!= 1:
1432 if fixup_policy
== 'warn':
1433 self
.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1434 info_dict
['id'], stretched_ratio
))
1435 elif fixup_policy
== 'detect_or_warn':
1436 stretched_pp
= FFmpegFixupStretchedPP(self
)
1437 if stretched_pp
.available
:
1438 info_dict
.setdefault('__postprocessors', [])
1439 info_dict
['__postprocessors'].append(stretched_pp
)
1441 self
.report_warning(
1442 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1443 info_dict
['id'], stretched_ratio
))
1445 assert fixup_policy
in ('ignore', 'never')
1447 if info_dict
.get('requested_formats') is None and info_dict
.get('container') == 'm4a_dash':
1448 if fixup_policy
== 'warn':
1449 self
.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1451 elif fixup_policy
== 'detect_or_warn':
1452 fixup_pp
= FFmpegFixupM4aPP(self
)
1453 if fixup_pp
.available
:
1454 info_dict
.setdefault('__postprocessors', [])
1455 info_dict
['__postprocessors'].append(fixup_pp
)
1457 self
.report_warning(
1458 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1461 assert fixup_policy
in ('ignore', 'never')
1464 self
.post_process(filename
, info_dict
)
1465 except (PostProcessingError
) as err
:
1466 self
.report_error('postprocessing: %s' % str(err
))
1468 self
.record_download_archive(info_dict
)
1470 def download(self
, url_list
):
1471 """Download a given list of URLs."""
1472 outtmpl
= self
.params
.get('outtmpl', DEFAULT_OUTTMPL
)
1473 if (len(url_list
) > 1 and
1474 '%' not in outtmpl
and
1475 self
.params
.get('max_downloads') != 1):
1476 raise SameFileError(outtmpl
)
1478 for url
in url_list
:
1480 # It also downloads the videos
1481 res
= self
.extract_info(url
)
1482 except UnavailableVideoError
:
1483 self
.report_error('unable to download video')
1484 except MaxDownloadsReached
:
1485 self
.to_screen('[info] Maximum number of downloaded files reached.')
1488 if self
.params
.get('dump_single_json', False):
1489 self
.to_stdout(json
.dumps(res
))
1491 return self
._download
_retcode
1493 def download_with_info_file(self
, info_filename
):
1494 with contextlib
.closing(fileinput
.FileInput(
1495 [info_filename
], mode
='r',
1496 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
1497 # FileInput doesn't have a read method, we can't call json.load
1498 info
= json
.loads('\n'.join(f
))
1500 self
.process_ie_result(info
, download
=True)
1501 except DownloadError
:
1502 webpage_url
= info
.get('webpage_url')
1503 if webpage_url
is not None:
1504 self
.report_warning('The info failed to download, trying with "%s"' % webpage_url
)
1505 return self
.download([webpage_url
])
1508 return self
._download
_retcode
1510 def post_process(self
, filename
, ie_info
):
1511 """Run all the postprocessors on the given file."""
1512 info
= dict(ie_info
)
1513 info
['filepath'] = filename
1515 if ie_info
.get('__postprocessors') is not None:
1516 pps_chain
.extend(ie_info
['__postprocessors'])
1517 pps_chain
.extend(self
._pps
)
1518 for pp
in pps_chain
:
1520 files_to_delete
, info
= pp
.run(info
)
1521 except PostProcessingError
as e
:
1522 self
.report_error(e
.msg
)
1523 if files_to_delete
and not self
.params
.get('keepvideo', False):
1524 for old_filename
in files_to_delete
:
1525 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
1527 os
.remove(encodeFilename(old_filename
))
1528 except (IOError, OSError):
1529 self
.report_warning('Unable to remove downloaded original file')
1531 def _make_archive_id(self
, info_dict
):
1532 # Future-proof against any change in case
1533 # and backwards compatibility with prior versions
1534 extractor
= info_dict
.get('extractor_key')
1535 if extractor
is None:
1536 if 'id' in info_dict
:
1537 extractor
= info_dict
.get('ie_key') # key in a playlist
1538 if extractor
is None:
1539 return None # Incomplete video information
1540 return extractor
.lower() + ' ' + info_dict
['id']
1542 def in_download_archive(self
, info_dict
):
1543 fn
= self
.params
.get('download_archive')
1547 vid_id
= self
._make
_archive
_id
(info_dict
)
1549 return False # Incomplete video information
1552 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
1553 for line
in archive_file
:
1554 if line
.strip() == vid_id
:
1556 except IOError as ioe
:
1557 if ioe
.errno
!= errno
.ENOENT
:
1561 def record_download_archive(self
, info_dict
):
1562 fn
= self
.params
.get('download_archive')
1565 vid_id
= self
._make
_archive
_id
(info_dict
)
1567 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
1568 archive_file
.write(vid_id
+ '\n')
1571 def format_resolution(format
, default
='unknown'):
1572 if format
.get('vcodec') == 'none':
1574 if format
.get('resolution') is not None:
1575 return format
['resolution']
1576 if format
.get('height') is not None:
1577 if format
.get('width') is not None:
1578 res
= '%sx%s' % (format
['width'], format
['height'])
1580 res
= '%sp' % format
['height']
1581 elif format
.get('width') is not None:
1582 res
= '?x%d' % format
['width']
1587 def _format_note(self
, fdict
):
1589 if fdict
.get('ext') in ['f4f', 'f4m']:
1590 res
+= '(unsupported) '
1591 if fdict
.get('format_note') is not None:
1592 res
+= fdict
['format_note'] + ' '
1593 if fdict
.get('tbr') is not None:
1594 res
+= '%4dk ' % fdict
['tbr']
1595 if fdict
.get('container') is not None:
1598 res
+= '%s container' % fdict
['container']
1599 if (fdict
.get('vcodec') is not None and
1600 fdict
.get('vcodec') != 'none'):
1603 res
+= fdict
['vcodec']
1604 if fdict
.get('vbr') is not None:
1606 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
1608 if fdict
.get('vbr') is not None:
1609 res
+= '%4dk' % fdict
['vbr']
1610 if fdict
.get('fps') is not None:
1611 res
+= ', %sfps' % fdict
['fps']
1612 if fdict
.get('acodec') is not None:
1615 if fdict
['acodec'] == 'none':
1618 res
+= '%-5s' % fdict
['acodec']
1619 elif fdict
.get('abr') is not None:
1623 if fdict
.get('abr') is not None:
1624 res
+= '@%3dk' % fdict
['abr']
1625 if fdict
.get('asr') is not None:
1626 res
+= ' (%5dHz)' % fdict
['asr']
1627 if fdict
.get('filesize') is not None:
1630 res
+= format_bytes(fdict
['filesize'])
1631 elif fdict
.get('filesize_approx') is not None:
1634 res
+= '~' + format_bytes(fdict
['filesize_approx'])
1637 def list_formats(self
, info_dict
):
1638 formats
= info_dict
.get('formats', [info_dict
])
1640 [f
['format_id'], f
['ext'], self
.format_resolution(f
), self
._format
_note
(f
)]
1642 if f
.get('preference') is None or f
['preference'] >= -1000]
1643 if len(formats
) > 1:
1644 table
[-1][-1] += (' ' if table
[-1][-1] else '') + '(best)'
1646 header_line
= ['format code', 'extension', 'resolution', 'note']
1648 '[info] Available formats for %s:\n%s' %
1649 (info_dict
['id'], render_table(header_line
, table
)))
1651 def list_thumbnails(self
, info_dict
):
1652 thumbnails
= info_dict
.get('thumbnails')
1654 tn_url
= info_dict
.get('thumbnail')
1656 thumbnails
= [{'id': '0', 'url': tn_url}
]
1659 '[info] No thumbnails present for %s' % info_dict
['id'])
1663 '[info] Thumbnails for %s:' % info_dict
['id'])
1664 self
.to_screen(render_table(
1665 ['ID', 'width', 'height', 'URL'],
1666 [[t
['id'], t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
]))
1668 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
1670 self
.to_screen('%s has no %s' % (video_id
, name
))
1673 'Available %s for %s:' % (name
, video_id
))
1674 self
.to_screen(render_table(
1675 ['Language', 'formats'],
1676 [[lang
, ', '.join(f
['ext'] for f
in reversed(formats
))]
1677 for lang
, formats
in subtitles
.items()]))
1679 def urlopen(self
, req
):
1680 """ Start an HTTP download """
1682 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1683 # always respected by websites, some tend to give out URLs with non percent-encoded
1684 # non-ASCII characters (see telemb.py, ard.py [#3412])
1685 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1686 # To work around aforementioned issue we will replace request's original URL with
1687 # percent-encoded one
1688 req_is_string
= isinstance(req
, compat_basestring
)
1689 url
= req
if req_is_string
else req
.get_full_url()
1690 url_escaped
= escape_url(url
)
1692 # Substitute URL if any change after escaping
1693 if url
!= url_escaped
:
1697 req
= compat_urllib_request
.Request(
1698 url_escaped
, data
=req
.data
, headers
=req
.headers
,
1699 origin_req_host
=req
.origin_req_host
, unverifiable
=req
.unverifiable
)
1701 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
1703 def print_debug_header(self
):
1704 if not self
.params
.get('verbose'):
1707 if type('') is not compat_str
:
1708 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1709 self
.report_warning(
1710 'Your Python is broken! Update to a newer and supported version')
1712 stdout_encoding
= getattr(
1713 sys
.stdout
, 'encoding', 'missing (%s)' % type(sys
.stdout
).__name
__)
1715 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1716 locale
.getpreferredencoding(),
1717 sys
.getfilesystemencoding(),
1719 self
.get_encoding()))
1720 write_string(encoding_str
, encoding
=None)
1722 self
._write
_string
('[debug] youtube-dl version ' + __version__
+ '\n')
1724 sp
= subprocess
.Popen(
1725 ['git', 'rev-parse', '--short', 'HEAD'],
1726 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
1727 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
1728 out
, err
= sp
.communicate()
1729 out
= out
.decode().strip()
1730 if re
.match('[0-9a-f]+', out
):
1731 self
._write
_string
('[debug] Git HEAD: ' + out
+ '\n')
1737 self
._write
_string
('[debug] Python version %s - %s\n' % (
1738 platform
.python_version(), platform_name()))
1740 exe_versions
= FFmpegPostProcessor
.get_versions(self
)
1741 exe_versions
['rtmpdump'] = rtmpdump_version()
1742 exe_str
= ', '.join(
1744 for exe
, v
in sorted(exe_versions
.items())
1749 self
._write
_string
('[debug] exe versions: %s\n' % exe_str
)
1752 for handler
in self
._opener
.handlers
:
1753 if hasattr(handler
, 'proxies'):
1754 proxy_map
.update(handler
.proxies
)
1755 self
._write
_string
('[debug] Proxy map: ' + compat_str(proxy_map
) + '\n')
1757 if self
.params
.get('call_home', False):
1758 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1759 self
._write
_string
('[debug] Public IP address: %s\n' % ipaddr
)
1760 latest_version
= self
.urlopen(
1761 'https://yt-dl.org/latest/version').read().decode('utf-8')
1762 if version_tuple(latest_version
) > version_tuple(__version__
):
1763 self
.report_warning(
1764 'You are using an outdated version (newest version: %s)! '
1765 'See https://yt-dl.org/update if you need help updating.' %
1768 def _setup_opener(self
):
1769 timeout_val
= self
.params
.get('socket_timeout')
1770 self
._socket
_timeout
= 600 if timeout_val
is None else float(timeout_val
)
1772 opts_cookiefile
= self
.params
.get('cookiefile')
1773 opts_proxy
= self
.params
.get('proxy')
1775 if opts_cookiefile
is None:
1776 self
.cookiejar
= compat_cookiejar
.CookieJar()
1778 self
.cookiejar
= compat_cookiejar
.MozillaCookieJar(
1780 if os
.access(opts_cookiefile
, os
.R_OK
):
1781 self
.cookiejar
.load()
1783 cookie_processor
= compat_urllib_request
.HTTPCookieProcessor(
1785 if opts_proxy
is not None:
1786 if opts_proxy
== '':
1789 proxies
= {'http': opts_proxy, 'https': opts_proxy}
1791 proxies
= compat_urllib_request
.getproxies()
1792 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1793 if 'http' in proxies
and 'https' not in proxies
:
1794 proxies
['https'] = proxies
['http']
1795 proxy_handler
= PerRequestProxyHandler(proxies
)
1797 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
1798 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
1799 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
1800 opener
= compat_urllib_request
.build_opener(
1801 proxy_handler
, https_handler
, cookie_processor
, ydlh
)
1803 # Delete the default user-agent header, which would otherwise apply in
1804 # cases where our custom HTTP handler doesn't come into play
1805 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1806 opener
.addheaders
= []
1807 self
._opener
= opener
1809 def encode(self
, s
):
1810 if isinstance(s
, bytes):
1811 return s
# Already encoded
1814 return s
.encode(self
.get_encoding())
1815 except UnicodeEncodeError as err
:
1816 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
1819 def get_encoding(self
):
1820 encoding
= self
.params
.get('encoding')
1821 if encoding
is None:
1822 encoding
= preferredencoding()
1825 def _write_thumbnails(self
, info_dict
, filename
):
1826 if self
.params
.get('writethumbnail', False):
1827 thumbnails
= info_dict
.get('thumbnails')
1829 thumbnails
= [thumbnails
[-1]]
1830 elif self
.params
.get('write_all_thumbnails', False):
1831 thumbnails
= info_dict
.get('thumbnails')
1836 # No thumbnails present, so return immediately
1839 for t
in thumbnails
:
1840 thumb_ext
= determine_ext(t
['url'], 'jpg')
1841 suffix
= '_%s' % t
['id'] if len(thumbnails
) > 1 else ''
1842 thumb_display_id
= '%s ' % t
['id'] if len(thumbnails
) > 1 else ''
1843 thumb_filename
= os
.path
.splitext(filename
)[0] + suffix
+ '.' + thumb_ext
1845 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(thumb_filename
)):
1846 self
.to_screen('[%s] %s: Thumbnail %sis already present' %
1847 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
1849 self
.to_screen('[%s] %s: Downloading thumbnail %s...' %
1850 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
1852 uf
= self
.urlopen(t
['url'])
1853 with open(thumb_filename
, 'wb') as thumbf
:
1854 shutil
.copyfileobj(uf
, thumbf
)
1855 self
.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1856 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
, thumb_filename
))
1857 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
1858 self
.report_warning('Unable to download thumbnail "%s": %s' %
1859 (t
['url'], compat_str(err
)))