2 # -*- coding: utf-8 -*-
4 from __future__
import absolute_import
, unicode_literals
33 compat_get_terminal_size
,
38 compat_urllib_request
,
58 PerRequestProxyHandler
,
68 UnavailableVideoError
,
79 from .cache
import Cache
80 from .extractor
import get_info_extractor
, gen_extractors
81 from .downloader
import get_suitable_downloader
82 from .downloader
.rtmp
import rtmpdump_version
83 from .postprocessor
import (
85 FFmpegFixupStretchedPP
,
90 from .version
import __version__
93 class YoutubeDL(object):
96 YoutubeDL objects are the ones responsible of downloading the
97 actual video file and writing it to disk if the user has requested
98 it, among some other tasks. In most cases there should be one per
99 program. As, given a video URL, the downloader doesn't know how to
100 extract all the needed information, task that InfoExtractors do, it
101 has to pass the URL to one of them.
103 For this, YoutubeDL objects have a method that allows
104 InfoExtractors to be registered in a given order. When it is passed
105 a URL, the YoutubeDL object handles it to the first InfoExtractor it
106 finds that reports being able to handle it. The InfoExtractor extracts
107 all the information about the video or videos the URL refers to, and
108 YoutubeDL process the extracted information, possibly using a File
109 Downloader to download the video.
111 YoutubeDL objects accept a lot of parameters. In order not to saturate
112 the object constructor with arguments, it receives a dictionary of
113 options instead. These options are available through the params
114 attribute for the InfoExtractors to use. The YoutubeDL also
115 registers itself as the downloader in charge for the InfoExtractors
116 that are added to it, so this is a "mutual registration".
120 username: Username for authentication purposes.
121 password: Password for authentication purposes.
122 videopassword: Password for acces a video.
123 usenetrc: Use netrc for authentication instead.
124 verbose: Print additional info to stdout.
125 quiet: Do not print messages to stdout.
126 no_warnings: Do not print out anything for warnings.
127 forceurl: Force printing final URL.
128 forcetitle: Force printing title.
129 forceid: Force printing ID.
130 forcethumbnail: Force printing thumbnail URL.
131 forcedescription: Force printing description.
132 forcefilename: Force printing final filename.
133 forceduration: Force printing duration.
134 forcejson: Force printing info_dict as JSON.
135 dump_single_json: Force printing the info_dict of the whole playlist
136 (or video) as a single JSON line.
137 simulate: Do not download the video files.
138 format: Video format code. See options.py for more information.
139 outtmpl: Template for output names.
140 restrictfilenames: Do not allow "&" and spaces in file names
141 ignoreerrors: Do not stop on download errors.
142 nooverwrites: Prevent overwriting files.
143 playliststart: Playlist item to start at.
144 playlistend: Playlist item to end at.
145 playlist_items: Specific indices of playlist to download.
146 playlistreverse: Download playlist items in reverse order.
147 matchtitle: Download only matching titles.
148 rejecttitle: Reject downloads for matching titles.
149 logger: Log messages to a logging.Logger instance.
150 logtostderr: Log messages to stderr instead of stdout.
151 writedescription: Write the video description to a .description file
152 writeinfojson: Write the video description to a .info.json file
153 writeannotations: Write the video annotations to a .annotations.xml file
154 writethumbnail: Write the thumbnail image to a file
155 write_all_thumbnails: Write all thumbnail formats to files
156 writesubtitles: Write the video subtitles to a file
157 writeautomaticsub: Write the automatic subtitles to a file
158 allsubtitles: Downloads all the subtitles of the video
159 (requires writesubtitles or writeautomaticsub)
160 listsubtitles: Lists all available subtitles for the video
161 subtitlesformat: The format code for subtitles
162 subtitleslangs: List of languages of the subtitles to download
163 keepvideo: Keep the video file after post-processing
164 daterange: A DateRange object, download only if the upload_date is in the range.
165 skip_download: Skip the actual download of the video file
166 cachedir: Location of the cache files in the filesystem.
167 False to disable filesystem cache.
168 noplaylist: Download single video instead of a playlist if in doubt.
169 age_limit: An integer representing the user's age in years.
170 Unsuitable videos for the given age are skipped.
171 min_views: An integer representing the minimum view count the video
172 must have in order to not be skipped.
173 Videos without view count information are always
174 downloaded. None for no limit.
175 max_views: An integer representing the maximum view count.
176 Videos that are more popular than that are not
178 Videos without view count information are always
179 downloaded. None for no limit.
180 download_archive: File name of a file where all downloads are recorded.
181 Videos already present in the file are not downloaded
183 cookiefile: File name where cookies should be read from and dumped to.
184 nocheckcertificate:Do not verify SSL certificates
185 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
186 At the moment, this is only supported by YouTube.
187 proxy: URL of the proxy server to use
188 cn_verification_proxy: URL of the proxy to use for IP address verification
189 on Chinese sites. (Experimental)
190 socket_timeout: Time to wait for unresponsive hosts, in seconds
191 bidi_workaround: Work around buggy terminals without bidirectional text
192 support, using fridibi
193 debug_printtraffic:Print out sent and received HTTP traffic
194 include_ads: Download ads as well
195 default_search: Prepend this string if an input url is not valid.
196 'auto' for elaborate guessing
197 encoding: Use this encoding instead of the system-specified.
198 extract_flat: Do not resolve URLs, return the immediate result.
199 Pass in 'in_playlist' to only show this behavior for
201 postprocessors: A list of dictionaries, each with an entry
202 * key: The name of the postprocessor. See
203 youtube_dl/postprocessor/__init__.py for a list.
204 as well as any further keyword arguments for the
206 progress_hooks: A list of functions that get called on download
207 progress, with a dictionary with the entries
208 * status: One of "downloading", "error", or "finished".
209 Check this first and ignore unknown values.
211 If status is one of "downloading", or "finished", the
212 following properties may also be present:
213 * filename: The final filename (always present)
214 * tmpfilename: The filename we're currently writing to
215 * downloaded_bytes: Bytes on disk
216 * total_bytes: Size of the whole file, None if unknown
217 * total_bytes_estimate: Guess of the eventual file size,
219 * elapsed: The number of seconds since download started.
220 * eta: The estimated time in seconds, None if unknown
221 * speed: The download speed in bytes/second, None if
223 * fragment_index: The counter of the currently
224 downloaded video fragment.
225 * fragment_count: The number of fragments (= individual
226 files that will be merged)
228 Progress hooks are guaranteed to be called at least once
229 (with status "finished") if the download is successful.
230 merge_output_format: Extension to use when merging formats.
231 fixup: Automatically correct known faults of the file.
233 - "never": do nothing
234 - "warn": only emit a warning
235 - "detect_or_warn": check whether we can do anything
236 about it, warn otherwise (default)
237 source_address: (Experimental) Client-side IP address to bind to.
238 call_home: Boolean, true iff we are allowed to contact the
239 youtube-dl servers for debugging.
240 sleep_interval: Number of seconds to sleep before each download.
241 listformats: Print an overview of available video formats and exit.
242 list_thumbnails: Print a table of all thumbnails and exit.
243 match_filter: A function that gets called with the info_dict of
245 If it returns a message, the video is ignored.
246 If it returns None, the video is downloaded.
247 match_filter_func in utils.py is one example for this.
248 no_color: Do not emit color codes in output.
250 The following options determine which downloader is picked:
251 external_downloader: Executable of the external downloader to call.
252 None or unset for standard (built-in) downloader.
253 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
255 The following parameters are not used by YoutubeDL itself, they are used by
256 the downloader (see youtube_dl/downloader/common.py):
257 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
258 noresizebuffer, retries, continuedl, noprogress, consoletitle,
259 xattr_set_filesize, external_downloader_args.
261 The following options are used by the post processors:
262 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
263 otherwise prefer avconv.
264 pp_params: Extra parameters for external apps, like avconv.
270 _download_retcode
= None
271 _num_downloads
= None
274 def __init__(self
, params
=None, auto_init
=True):
275 """Create a FileDownloader object with the given options."""
279 self
._ies
_instances
= {}
281 self
._progress
_hooks
= []
282 self
._download
_retcode
= 0
283 self
._num
_downloads
= 0
284 self
._screen
_file
= [sys
.stdout
, sys
.stderr
][params
.get('logtostderr', False)]
285 self
._err
_file
= sys
.stderr
287 self
.cache
= Cache(self
)
289 if params
.get('bidi_workaround', False):
292 master
, slave
= pty
.openpty()
293 width
= compat_get_terminal_size().columns
297 width_args
= ['-w', str(width
)]
299 stdin
=subprocess
.PIPE
,
301 stderr
=self
._err
_file
)
303 self
._output
_process
= subprocess
.Popen(
304 ['bidiv'] + width_args
, **sp_kwargs
307 self
._output
_process
= subprocess
.Popen(
308 ['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
309 self
._output
_channel
= os
.fdopen(master
, 'rb')
310 except OSError as ose
:
312 self
.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
316 if (sys
.version_info
>= (3,) and sys
.platform
!= 'win32' and
317 sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
318 not params
.get('restrictfilenames', False)):
319 # On Python 3, the Unicode filesystem API will throw errors (#1474)
321 'Assuming --restrict-filenames since file system encoding '
322 'cannot encode all characters. '
323 'Set the LC_ALL environment variable to fix this.')
324 self
.params
['restrictfilenames'] = True
326 if isinstance(params
.get('outtmpl'), bytes):
328 'Parameter outtmpl is bytes, but should be a unicode string. '
329 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
334 self
.print_debug_header()
335 self
.add_default_info_extractors()
337 for pp_def_raw
in self
.params
.get('postprocessors', []):
338 pp_class
= get_postprocessor(pp_def_raw
['key'])
339 pp_def
= dict(pp_def_raw
)
341 pp
= pp_class(self
, **compat_kwargs(pp_def
))
342 self
.add_post_processor(pp
)
344 for ph
in self
.params
.get('progress_hooks', []):
345 self
.add_progress_hook(ph
)
347 def warn_if_short_id(self
, argv
):
348 # short YouTube ID starting with dash?
350 i
for i
, a
in enumerate(argv
)
351 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
355 [a
for i
, a
in enumerate(argv
) if i
not in idxs
] +
356 ['--'] + [argv
[i
] for i
in idxs
]
359 'Long argument string detected. '
360 'Use -- to separate parameters and URLs, like this:\n%s\n' %
361 args_to_str(correct_argv
))
363 def add_info_extractor(self
, ie
):
364 """Add an InfoExtractor object to the end of the list."""
366 self
._ies
_instances
[ie
.ie_key()] = ie
367 ie
.set_downloader(self
)
369 def get_info_extractor(self
, ie_key
):
371 Get an instance of an IE with name ie_key, it will try to get one from
372 the _ies list, if there's no instance it will create a new one and add
373 it to the extractor list.
375 ie
= self
._ies
_instances
.get(ie_key
)
377 ie
= get_info_extractor(ie_key
)()
378 self
.add_info_extractor(ie
)
381 def add_default_info_extractors(self
):
383 Add the InfoExtractors returned by gen_extractors to the end of the list
385 for ie
in gen_extractors():
386 self
.add_info_extractor(ie
)
388 def add_post_processor(self
, pp
):
389 """Add a PostProcessor object to the end of the chain."""
391 pp
.set_downloader(self
)
393 def add_progress_hook(self
, ph
):
394 """Add the progress hook (currently only for the file downloader)"""
395 self
._progress
_hooks
.append(ph
)
397 def _bidi_workaround(self
, message
):
398 if not hasattr(self
, '_output_channel'):
401 assert hasattr(self
, '_output_process')
402 assert isinstance(message
, compat_str
)
403 line_count
= message
.count('\n') + 1
404 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
405 self
._output
_process
.stdin
.flush()
406 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
407 for _
in range(line_count
))
408 return res
[:-len('\n')]
410 def to_screen(self
, message
, skip_eol
=False):
411 """Print message to stdout if not in quiet mode."""
412 return self
.to_stdout(message
, skip_eol
, check_quiet
=True)
414 def _write_string(self
, s
, out
=None):
415 write_string(s
, out
=out
, encoding
=self
.params
.get('encoding'))
417 def to_stdout(self
, message
, skip_eol
=False, check_quiet
=False):
418 """Print message to stdout if not in quiet mode."""
419 if self
.params
.get('logger'):
420 self
.params
['logger'].debug(message
)
421 elif not check_quiet
or not self
.params
.get('quiet', False):
422 message
= self
._bidi
_workaround
(message
)
423 terminator
= ['\n', ''][skip_eol
]
424 output
= message
+ terminator
426 self
._write
_string
(output
, self
._screen
_file
)
428 def to_stderr(self
, message
):
429 """Print message to stderr."""
430 assert isinstance(message
, compat_str
)
431 if self
.params
.get('logger'):
432 self
.params
['logger'].error(message
)
434 message
= self
._bidi
_workaround
(message
)
435 output
= message
+ '\n'
436 self
._write
_string
(output
, self
._err
_file
)
438 def to_console_title(self
, message
):
439 if not self
.params
.get('consoletitle', False):
441 if os
.name
== 'nt' and ctypes
.windll
.kernel32
.GetConsoleWindow():
442 # c_wchar_p() might not be necessary if `message` is
443 # already of type unicode()
444 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
445 elif 'TERM' in os
.environ
:
446 self
._write
_string
('\033]0;%s\007' % message
, self
._screen
_file
)
448 def save_console_title(self
):
449 if not self
.params
.get('consoletitle', False):
451 if 'TERM' in os
.environ
:
452 # Save the title on stack
453 self
._write
_string
('\033[22;0t', self
._screen
_file
)
455 def restore_console_title(self
):
456 if not self
.params
.get('consoletitle', False):
458 if 'TERM' in os
.environ
:
459 # Restore the title from stack
460 self
._write
_string
('\033[23;0t', self
._screen
_file
)
463 self
.save_console_title()
466 def __exit__(self
, *args
):
467 self
.restore_console_title()
469 if self
.params
.get('cookiefile') is not None:
470 self
.cookiejar
.save()
472 def trouble(self
, message
=None, tb
=None):
473 """Determine action to take when a download problem appears.
475 Depending on if the downloader has been configured to ignore
476 download errors or not, this method may throw an exception or
477 not when errors are found, after printing the message.
479 tb, if given, is additional traceback information.
481 if message
is not None:
482 self
.to_stderr(message
)
483 if self
.params
.get('verbose'):
485 if sys
.exc_info()[0]: # if .trouble has been called from an except block
487 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
488 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
489 tb
+= compat_str(traceback
.format_exc())
491 tb_data
= traceback
.format_list(traceback
.extract_stack())
492 tb
= ''.join(tb_data
)
494 if not self
.params
.get('ignoreerrors', False):
495 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
496 exc_info
= sys
.exc_info()[1].exc_info
498 exc_info
= sys
.exc_info()
499 raise DownloadError(message
, exc_info
)
500 self
._download
_retcode
= 1
502 def report_warning(self
, message
):
504 Print the message to stderr, it will be prefixed with 'WARNING:'
505 If stderr is a tty file the 'WARNING:' will be colored
507 if self
.params
.get('logger') is not None:
508 self
.params
['logger'].warning(message
)
510 if self
.params
.get('no_warnings'):
512 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and os
.name
!= 'nt':
513 _msg_header
= '\033[0;33mWARNING:\033[0m'
515 _msg_header
= 'WARNING:'
516 warning_message
= '%s %s' % (_msg_header
, message
)
517 self
.to_stderr(warning_message
)
519 def report_error(self
, message
, tb
=None):
521 Do the same as trouble, but prefixes the message with 'ERROR:', colored
522 in red if stderr is a tty file.
524 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and os
.name
!= 'nt':
525 _msg_header
= '\033[0;31mERROR:\033[0m'
527 _msg_header
= 'ERROR:'
528 error_message
= '%s %s' % (_msg_header
, message
)
529 self
.trouble(error_message
, tb
)
531 def report_file_already_downloaded(self
, file_name
):
532 """Report file has already been fully downloaded."""
534 self
.to_screen('[download] %s has already been downloaded' % file_name
)
535 except UnicodeEncodeError:
536 self
.to_screen('[download] The file has already been downloaded')
538 def prepare_filename(self
, info_dict
):
539 """Generate the output filename."""
541 template_dict
= dict(info_dict
)
543 template_dict
['epoch'] = int(time
.time())
544 autonumber_size
= self
.params
.get('autonumber_size')
545 if autonumber_size
is None:
547 autonumber_templ
= '%0' + str(autonumber_size
) + 'd'
548 template_dict
['autonumber'] = autonumber_templ
% self
._num
_downloads
549 if template_dict
.get('playlist_index') is not None:
550 template_dict
['playlist_index'] = '%0*d' % (len(str(template_dict
['n_entries'])), template_dict
['playlist_index'])
551 if template_dict
.get('resolution') is None:
552 if template_dict
.get('width') and template_dict
.get('height'):
553 template_dict
['resolution'] = '%dx%d' % (template_dict
['width'], template_dict
['height'])
554 elif template_dict
.get('height'):
555 template_dict
['resolution'] = '%sp' % template_dict
['height']
556 elif template_dict
.get('width'):
557 template_dict
['resolution'] = '?x%d' % template_dict
['width']
559 sanitize
= lambda k
, v
: sanitize_filename(
561 restricted
=self
.params
.get('restrictfilenames'),
563 template_dict
= dict((k
, sanitize(k
, v
))
564 for k
, v
in template_dict
.items()
566 template_dict
= collections
.defaultdict(lambda: 'NA', template_dict
)
568 outtmpl
= sanitize_path(self
.params
.get('outtmpl', DEFAULT_OUTTMPL
))
569 tmpl
= compat_expanduser(outtmpl
)
570 filename
= tmpl
% template_dict
571 # Temporary fix for #4787
572 # 'Treat' all problem characters by passing filename through preferredencoding
573 # to workaround encoding issues with subprocess on python2 @ Windows
574 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
575 filename
= encodeFilename(filename
, True).decode(preferredencoding())
577 except ValueError as err
:
578 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
581 def _match_entry(self
, info_dict
, incomplete
):
582 """ Returns None iff the file should be downloaded """
584 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
585 if 'title' in info_dict
:
586 # This can happen when we're just evaluating the playlist
587 title
= info_dict
['title']
588 matchtitle
= self
.params
.get('matchtitle', False)
590 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
591 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
592 rejecttitle
= self
.params
.get('rejecttitle', False)
594 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
595 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
596 date
= info_dict
.get('upload_date', None)
598 dateRange
= self
.params
.get('daterange', DateRange())
599 if date
not in dateRange
:
600 return '%s upload date is not in range %s' % (date_from_str(date
).isoformat(), dateRange
)
601 view_count
= info_dict
.get('view_count', None)
602 if view_count
is not None:
603 min_views
= self
.params
.get('min_views')
604 if min_views
is not None and view_count
< min_views
:
605 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
606 max_views
= self
.params
.get('max_views')
607 if max_views
is not None and view_count
> max_views
:
608 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
609 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
610 return 'Skipping "%s" because it is age restricted' % video_title
611 if self
.in_download_archive(info_dict
):
612 return '%s has already been recorded in archive' % video_title
615 match_filter
= self
.params
.get('match_filter')
616 if match_filter
is not None:
617 ret
= match_filter(info_dict
)
624 def add_extra_info(info_dict
, extra_info
):
625 '''Set the keys from extra_info in info dict if they are missing'''
626 for key
, value
in extra_info
.items():
627 info_dict
.setdefault(key
, value
)
629 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
={},
632 Returns a list with a dictionary for each video we find.
633 If 'download', also downloads the videos.
634 extra_info is a dict containing the extra values to add to each result
638 ies
= [self
.get_info_extractor(ie_key
)]
643 if not ie
.suitable(url
):
647 self
.report_warning('The program functionality for this site has been marked as broken, '
648 'and will probably not work.')
651 ie_result
= ie
.extract(url
)
652 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
654 if isinstance(ie_result
, list):
655 # Backwards compatibility: old IE result format
657 '_type': 'compat_list',
658 'entries': ie_result
,
660 self
.add_default_extra_info(ie_result
, ie
, url
)
662 return self
.process_ie_result(ie_result
, download
, extra_info
)
665 except ExtractorError
as de
: # An error we somewhat expected
666 self
.report_error(compat_str(de
), de
.format_traceback())
668 except MaxDownloadsReached
:
670 except Exception as e
:
671 if self
.params
.get('ignoreerrors', False):
672 self
.report_error(compat_str(e
), tb
=compat_str(traceback
.format_exc()))
677 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
679 def add_default_extra_info(self
, ie_result
, ie
, url
):
680 self
.add_extra_info(ie_result
, {
681 'extractor': ie
.IE_NAME
,
683 'webpage_url_basename': url_basename(url
),
684 'extractor_key': ie
.ie_key(),
687 def process_ie_result(self
, ie_result
, download
=True, extra_info
={}):
689 Take the result of the ie(may be modified) and resolve all unresolved
690 references (URLs, playlist items).
692 It will also download the videos if 'download'.
693 Returns the resolved ie_result.
696 result_type
= ie_result
.get('_type', 'video')
698 if result_type
in ('url', 'url_transparent'):
699 extract_flat
= self
.params
.get('extract_flat', False)
700 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
) or
701 extract_flat
is True):
702 if self
.params
.get('forcejson', False):
703 self
.to_stdout(json
.dumps(ie_result
))
706 if result_type
== 'video':
707 self
.add_extra_info(ie_result
, extra_info
)
708 return self
.process_video_result(ie_result
, download
=download
)
709 elif result_type
== 'url':
710 # We have to add extra_info to the results because it may be
711 # contained in a playlist
712 return self
.extract_info(ie_result
['url'],
714 ie_key
=ie_result
.get('ie_key'),
715 extra_info
=extra_info
)
716 elif result_type
== 'url_transparent':
717 # Use the information from the embedding page
718 info
= self
.extract_info(
719 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
720 extra_info
=extra_info
, download
=False, process
=False)
722 force_properties
= dict(
723 (k
, v
) for k
, v
in ie_result
.items() if v
is not None)
724 for f
in ('_type', 'url'):
725 if f
in force_properties
:
726 del force_properties
[f
]
727 new_result
= info
.copy()
728 new_result
.update(force_properties
)
730 assert new_result
.get('_type') != 'url_transparent'
732 return self
.process_ie_result(
733 new_result
, download
=download
, extra_info
=extra_info
)
734 elif result_type
== 'playlist' or result_type
== 'multi_video':
735 # We process each entry in the playlist
736 playlist
= ie_result
.get('title', None) or ie_result
.get('id', None)
737 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
739 playlist_results
= []
741 playliststart
= self
.params
.get('playliststart', 1) - 1
742 playlistend
= self
.params
.get('playlistend', None)
743 # For backwards compatibility, interpret -1 as whole list
744 if playlistend
== -1:
747 playlistitems_str
= self
.params
.get('playlist_items', None)
749 if playlistitems_str
is not None:
750 def iter_playlistitems(format
):
751 for string_segment
in format
.split(','):
752 if '-' in string_segment
:
753 start
, end
= string_segment
.split('-')
754 for item
in range(int(start
), int(end
) + 1):
757 yield int(string_segment
)
758 playlistitems
= iter_playlistitems(playlistitems_str
)
760 ie_entries
= ie_result
['entries']
761 if isinstance(ie_entries
, list):
762 n_all_entries
= len(ie_entries
)
765 ie_entries
[i
- 1] for i
in playlistitems
766 if -n_all_entries
<= i
- 1 < n_all_entries
]
768 entries
= ie_entries
[playliststart
:playlistend
]
769 n_entries
= len(entries
)
771 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
772 (ie_result
['extractor'], playlist
, n_all_entries
, n_entries
))
773 elif isinstance(ie_entries
, PagedList
):
776 for item
in playlistitems
:
777 entries
.extend(ie_entries
.getslice(
781 entries
= ie_entries
.getslice(
782 playliststart
, playlistend
)
783 n_entries
= len(entries
)
785 "[%s] playlist %s: Downloading %d videos" %
786 (ie_result
['extractor'], playlist
, n_entries
))
789 entry_list
= list(ie_entries
)
790 entries
= [entry_list
[i
- 1] for i
in playlistitems
]
792 entries
= list(itertools
.islice(
793 ie_entries
, playliststart
, playlistend
))
794 n_entries
= len(entries
)
796 "[%s] playlist %s: Downloading %d videos" %
797 (ie_result
['extractor'], playlist
, n_entries
))
799 if self
.params
.get('playlistreverse', False):
800 entries
= entries
[::-1]
802 for i
, entry
in enumerate(entries
, 1):
803 self
.to_screen('[download] Downloading video %s of %s' % (i
, n_entries
))
805 'n_entries': n_entries
,
806 'playlist': playlist
,
807 'playlist_id': ie_result
.get('id'),
808 'playlist_title': ie_result
.get('title'),
809 'playlist_index': i
+ playliststart
,
810 'extractor': ie_result
['extractor'],
811 'webpage_url': ie_result
['webpage_url'],
812 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
813 'extractor_key': ie_result
['extractor_key'],
816 reason
= self
._match
_entry
(entry
, incomplete
=True)
817 if reason
is not None:
818 self
.to_screen('[download] ' + reason
)
821 entry_result
= self
.process_ie_result(entry
,
824 playlist_results
.append(entry_result
)
825 ie_result
['entries'] = playlist_results
827 elif result_type
== 'compat_list':
829 'Extractor %s returned a compat_list result. '
830 'It needs to be updated.' % ie_result
.get('extractor'))
836 'extractor': ie_result
['extractor'],
837 'webpage_url': ie_result
['webpage_url'],
838 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
839 'extractor_key': ie_result
['extractor_key'],
843 ie_result
['entries'] = [
844 self
.process_ie_result(_fixup(r
), download
, extra_info
)
845 for r
in ie_result
['entries']
849 raise Exception('Invalid result type: %s' % result_type
)
851 def _apply_format_filter(self
, format_spec
, available_formats
):
852 " Returns a tuple of the remaining format_spec and filtered formats "
862 operator_rex
= re
.compile(r
'''(?x)\s*\[
863 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
864 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
865 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
867 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
868 m
= operator_rex
.search(format_spec
)
871 comparison_value
= int(m
.group('value'))
873 comparison_value
= parse_filesize(m
.group('value'))
874 if comparison_value
is None:
875 comparison_value
= parse_filesize(m
.group('value') + 'B')
876 if comparison_value
is None:
878 'Invalid value %r in format specification %r' % (
879 m
.group('value'), format_spec
))
880 op
= OPERATORS
[m
.group('op')]
887 str_operator_rex
= re
.compile(r
'''(?x)\s*\[
888 \s*(?P<key>ext|acodec|vcodec|container|protocol)
889 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
890 \s*(?P<value>[a-zA-Z0-9_-]+)
892 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
893 m
= str_operator_rex
.search(format_spec
)
895 comparison_value
= m
.group('value')
896 op
= STR_OPERATORS
[m
.group('op')]
899 raise ValueError('Invalid format specification %r' % format_spec
)
902 actual_value
= f
.get(m
.group('key'))
903 if actual_value
is None:
904 return m
.group('none_inclusive')
905 return op(actual_value
, comparison_value
)
906 new_formats
= [f
for f
in available_formats
if _filter(f
)]
908 new_format_spec
= format_spec
[:-len(m
.group(0))]
909 if not new_format_spec
:
910 new_format_spec
= 'best'
912 return (new_format_spec
, new_formats
)
914 def select_format(self
, format_spec
, available_formats
):
915 while format_spec
.endswith(']'):
916 format_spec
, available_formats
= self
._apply
_format
_filter
(
917 format_spec
, available_formats
)
918 if not available_formats
:
921 if format_spec
in ['best', 'worst', None]:
922 format_idx
= 0 if format_spec
== 'worst' else -1
923 audiovideo_formats
= [
924 f
for f
in available_formats
925 if f
.get('vcodec') != 'none' and f
.get('acodec') != 'none']
926 if audiovideo_formats
:
927 return audiovideo_formats
[format_idx
]
928 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
929 elif (all(f
.get('acodec') != 'none' for f
in available_formats
) or
930 all(f
.get('vcodec') != 'none' for f
in available_formats
)):
931 return available_formats
[format_idx
]
932 elif format_spec
== 'bestaudio':
934 f
for f
in available_formats
935 if f
.get('vcodec') == 'none']
937 return audio_formats
[-1]
938 elif format_spec
== 'worstaudio':
940 f
for f
in available_formats
941 if f
.get('vcodec') == 'none']
943 return audio_formats
[0]
944 elif format_spec
== 'bestvideo':
946 f
for f
in available_formats
947 if f
.get('acodec') == 'none']
949 return video_formats
[-1]
950 elif format_spec
== 'worstvideo':
952 f
for f
in available_formats
953 if f
.get('acodec') == 'none']
955 return video_formats
[0]
957 extensions
= ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
958 if format_spec
in extensions
:
959 filter_f
= lambda f
: f
['ext'] == format_spec
961 filter_f
= lambda f
: f
['format_id'] == format_spec
962 matches
= list(filter(filter_f
, available_formats
))
967 def _calc_headers(self
, info_dict
):
968 res
= std_headers
.copy()
970 add_headers
= info_dict
.get('http_headers')
972 res
.update(add_headers
)
974 cookies
= self
._calc
_cookies
(info_dict
)
976 res
['Cookie'] = cookies
980 def _calc_cookies(self
, info_dict
):
981 pr
= compat_urllib_request
.Request(info_dict
['url'])
982 self
.cookiejar
.add_cookie_header(pr
)
983 return pr
.get_header('Cookie')
985 def process_video_result(self
, info_dict
, download
=True):
986 assert info_dict
.get('_type', 'video') == 'video'
988 if 'id' not in info_dict
:
989 raise ExtractorError('Missing "id" field in extractor result')
990 if 'title' not in info_dict
:
991 raise ExtractorError('Missing "title" field in extractor result')
993 if 'playlist' not in info_dict
:
994 # It isn't part of a playlist
995 info_dict
['playlist'] = None
996 info_dict
['playlist_index'] = None
998 thumbnails
= info_dict
.get('thumbnails')
999 if thumbnails
is None:
1000 thumbnail
= info_dict
.get('thumbnail')
1002 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
1004 thumbnails
.sort(key
=lambda t
: (
1005 t
.get('preference'), t
.get('width'), t
.get('height'),
1006 t
.get('id'), t
.get('url')))
1007 for i
, t
in enumerate(thumbnails
):
1008 if 'width' in t
and 'height' in t
:
1009 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
1010 if t
.get('id') is None:
1013 if thumbnails
and 'thumbnail' not in info_dict
:
1014 info_dict
['thumbnail'] = thumbnails
[-1]['url']
1016 if 'display_id' not in info_dict
and 'id' in info_dict
:
1017 info_dict
['display_id'] = info_dict
['id']
1019 if info_dict
.get('upload_date') is None and info_dict
.get('timestamp') is not None:
1020 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1021 # see http://bugs.python.org/issue1646728)
1023 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
['timestamp'])
1024 info_dict
['upload_date'] = upload_date
.strftime('%Y%m%d')
1025 except (ValueError, OverflowError, OSError):
1028 if self
.params
.get('listsubtitles', False):
1029 if 'automatic_captions' in info_dict
:
1030 self
.list_subtitles(info_dict
['id'], info_dict
.get('automatic_captions'), 'automatic captions')
1031 self
.list_subtitles(info_dict
['id'], info_dict
.get('subtitles'), 'subtitles')
1033 info_dict
['requested_subtitles'] = self
.process_subtitles(
1034 info_dict
['id'], info_dict
.get('subtitles'),
1035 info_dict
.get('automatic_captions'))
1037 # This extractors handle format selection themselves
1038 if info_dict
['extractor'] in ['Youku']:
1040 self
.process_info(info_dict
)
1043 # We now pick which formats have to be downloaded
1044 if info_dict
.get('formats') is None:
1045 # There's only one format available
1046 formats
= [info_dict
]
1048 formats
= info_dict
['formats']
1051 raise ExtractorError('No video formats found!')
1055 # We check that all the formats have the format and format_id fields
1056 for i
, format
in enumerate(formats
):
1057 if 'url' not in format
:
1058 raise ExtractorError('Missing "url" key in result (index %d)' % i
)
1060 if format
.get('format_id') is None:
1061 format
['format_id'] = compat_str(i
)
1062 format_id
= format
['format_id']
1063 if format_id
not in formats_dict
:
1064 formats_dict
[format_id
] = []
1065 formats_dict
[format_id
].append(format
)
1067 # Make sure all formats have unique format_id
1068 for format_id
, ambiguous_formats
in formats_dict
.items():
1069 if len(ambiguous_formats
) > 1:
1070 for i
, format
in enumerate(ambiguous_formats
):
1071 format
['format_id'] = '%s-%d' % (format_id
, i
)
1073 for i
, format
in enumerate(formats
):
1074 if format
.get('format') is None:
1075 format
['format'] = '{id} - {res}{note}'.format(
1076 id=format
['format_id'],
1077 res
=self
.format_resolution(format
),
1078 note
=' ({0})'.format(format
['format_note']) if format
.get('format_note') is not None else '',
1080 # Automatically determine file extension if missing
1081 if 'ext' not in format
:
1082 format
['ext'] = determine_ext(format
['url']).lower()
1083 # Add HTTP headers, so that external programs can use them from the
1085 full_format_info
= info_dict
.copy()
1086 full_format_info
.update(format
)
1087 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
1089 # TODO Central sorting goes here
1091 if formats
[0] is not info_dict
:
1092 # only set the 'formats' fields if the original info_dict list them
1093 # otherwise we end up with a circular reference, the first (and unique)
1094 # element in the 'formats' field in info_dict is info_dict itself,
1095 # wich can't be exported to json
1096 info_dict
['formats'] = formats
1097 if self
.params
.get('listformats'):
1098 self
.list_formats(info_dict
)
1100 if self
.params
.get('list_thumbnails'):
1101 self
.list_thumbnails(info_dict
)
1104 req_format
= self
.params
.get('format')
1105 if req_format
is None:
1106 req_format_list
= []
1107 if (self
.params
.get('outtmpl', DEFAULT_OUTTMPL
) != '-' and
1108 info_dict
['extractor'] in ['youtube', 'ted']):
1109 merger
= FFmpegMergerPP(self
)
1110 if merger
.available
and merger
.can_merge():
1111 req_format_list
.append('bestvideo+bestaudio')
1112 req_format_list
.append('best')
1113 req_format
= '/'.join(req_format_list
)
1114 formats_to_download
= []
1115 if req_format
== 'all':
1116 formats_to_download
= formats
1118 for rfstr
in req_format
.split(','):
1119 # We can accept formats requested in the format: 34/5/best, we pick
1120 # the first that is available, starting from left
1121 req_formats
= rfstr
.split('/')
1122 for rf
in req_formats
:
1123 if re
.match(r
'.+?\+.+?', rf
) is not None:
1124 # Two formats have been requested like '137+139'
1125 format_1
, format_2
= rf
.split('+')
1126 formats_info
= (self
.select_format(format_1
, formats
),
1127 self
.select_format(format_2
, formats
))
1128 if all(formats_info
):
1129 # The first format must contain the video and the
1131 if formats_info
[0].get('vcodec') == 'none':
1132 self
.report_error('The first format must '
1133 'contain the video, try using '
1134 '"-f %s+%s"' % (format_2
, format_1
))
1137 formats_info
[0]['ext']
1138 if self
.params
.get('merge_output_format') is None
1139 else self
.params
['merge_output_format'])
1141 'requested_formats': formats_info
,
1142 'format': '%s+%s' % (formats_info
[0].get('format'),
1143 formats_info
[1].get('format')),
1144 'format_id': '%s+%s' % (formats_info
[0].get('format_id'),
1145 formats_info
[1].get('format_id')),
1146 'width': formats_info
[0].get('width'),
1147 'height': formats_info
[0].get('height'),
1148 'resolution': formats_info
[0].get('resolution'),
1149 'fps': formats_info
[0].get('fps'),
1150 'vcodec': formats_info
[0].get('vcodec'),
1151 'vbr': formats_info
[0].get('vbr'),
1152 'stretched_ratio': formats_info
[0].get('stretched_ratio'),
1153 'acodec': formats_info
[1].get('acodec'),
1154 'abr': formats_info
[1].get('abr'),
1158 selected_format
= None
1160 selected_format
= self
.select_format(rf
, formats
)
1161 if selected_format
is not None:
1162 formats_to_download
.append(selected_format
)
1164 if not formats_to_download
:
1165 raise ExtractorError('requested format not available',
1169 if len(formats_to_download
) > 1:
1170 self
.to_screen('[info] %s: downloading video in %s formats' % (info_dict
['id'], len(formats_to_download
)))
1171 for format
in formats_to_download
:
1172 new_info
= dict(info_dict
)
1173 new_info
.update(format
)
1174 self
.process_info(new_info
)
1175 # We update the info dict with the best quality format (backwards compatibility)
1176 info_dict
.update(formats_to_download
[-1])
1179 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
1180 """Select the requested subtitles and their format"""
1182 if normal_subtitles
and self
.params
.get('writesubtitles'):
1183 available_subs
.update(normal_subtitles
)
1184 if automatic_captions
and self
.params
.get('writeautomaticsub'):
1185 for lang
, cap_info
in automatic_captions
.items():
1186 if lang
not in available_subs
:
1187 available_subs
[lang
] = cap_info
1189 if (not self
.params
.get('writesubtitles') and not
1190 self
.params
.get('writeautomaticsub') or not
1194 if self
.params
.get('allsubtitles', False):
1195 requested_langs
= available_subs
.keys()
1197 if self
.params
.get('subtitleslangs', False):
1198 requested_langs
= self
.params
.get('subtitleslangs')
1199 elif 'en' in available_subs
:
1200 requested_langs
= ['en']
1202 requested_langs
= [list(available_subs
.keys())[0]]
1204 formats_query
= self
.params
.get('subtitlesformat', 'best')
1205 formats_preference
= formats_query
.split('/') if formats_query
else []
1207 for lang
in requested_langs
:
1208 formats
= available_subs
.get(lang
)
1210 self
.report_warning('%s subtitles not available for %s' % (lang
, video_id
))
1212 for ext
in formats_preference
:
1216 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
1222 self
.report_warning(
1223 'No subtitle format found matching "%s" for language %s, '
1224 'using %s' % (formats_query
, lang
, f
['ext']))
1228 def process_info(self
, info_dict
):
1229 """Process a single resolved IE result."""
1231 assert info_dict
.get('_type', 'video') == 'video'
1233 max_downloads
= self
.params
.get('max_downloads')
1234 if max_downloads
is not None:
1235 if self
._num
_downloads
>= int(max_downloads
):
1236 raise MaxDownloadsReached()
1238 info_dict
['fulltitle'] = info_dict
['title']
1239 if len(info_dict
['title']) > 200:
1240 info_dict
['title'] = info_dict
['title'][:197] + '...'
1242 if 'format' not in info_dict
:
1243 info_dict
['format'] = info_dict
['ext']
1245 reason
= self
._match
_entry
(info_dict
, incomplete
=False)
1246 if reason
is not None:
1247 self
.to_screen('[download] ' + reason
)
1250 self
._num
_downloads
+= 1
1252 info_dict
['_filename'] = filename
= self
.prepare_filename(info_dict
)
1255 if self
.params
.get('forcetitle', False):
1256 self
.to_stdout(info_dict
['fulltitle'])
1257 if self
.params
.get('forceid', False):
1258 self
.to_stdout(info_dict
['id'])
1259 if self
.params
.get('forceurl', False):
1260 if info_dict
.get('requested_formats') is not None:
1261 for f
in info_dict
['requested_formats']:
1262 self
.to_stdout(f
['url'] + f
.get('play_path', ''))
1264 # For RTMP URLs, also include the playpath
1265 self
.to_stdout(info_dict
['url'] + info_dict
.get('play_path', ''))
1266 if self
.params
.get('forcethumbnail', False) and info_dict
.get('thumbnail') is not None:
1267 self
.to_stdout(info_dict
['thumbnail'])
1268 if self
.params
.get('forcedescription', False) and info_dict
.get('description') is not None:
1269 self
.to_stdout(info_dict
['description'])
1270 if self
.params
.get('forcefilename', False) and filename
is not None:
1271 self
.to_stdout(filename
)
1272 if self
.params
.get('forceduration', False) and info_dict
.get('duration') is not None:
1273 self
.to_stdout(formatSeconds(info_dict
['duration']))
1274 if self
.params
.get('forceformat', False):
1275 self
.to_stdout(info_dict
['format'])
1276 if self
.params
.get('forcejson', False):
1277 self
.to_stdout(json
.dumps(info_dict
))
1279 # Do nothing else if in simulate mode
1280 if self
.params
.get('simulate', False):
1283 if filename
is None:
1287 dn
= os
.path
.dirname(sanitize_path(encodeFilename(filename
)))
1288 if dn
and not os
.path
.exists(dn
):
1290 except (OSError, IOError) as err
:
1291 self
.report_error('unable to create directory ' + compat_str(err
))
1294 if self
.params
.get('writedescription', False):
1295 descfn
= replace_extension(filename
, 'description', info_dict
.get('ext'))
1296 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(descfn
)):
1297 self
.to_screen('[info] Video description is already present')
1298 elif info_dict
.get('description') is None:
1299 self
.report_warning('There\'s no description to write.')
1302 self
.to_screen('[info] Writing video description to: ' + descfn
)
1303 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
1304 descfile
.write(info_dict
['description'])
1305 except (OSError, IOError):
1306 self
.report_error('Cannot write description file ' + descfn
)
1309 if self
.params
.get('writeannotations', False):
1310 annofn
= replace_extension(filename
, 'annotations.xml', info_dict
.get('ext'))
1311 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(annofn
)):
1312 self
.to_screen('[info] Video annotations are already present')
1315 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
1316 with io
.open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
1317 annofile
.write(info_dict
['annotations'])
1318 except (KeyError, TypeError):
1319 self
.report_warning('There are no annotations to write.')
1320 except (OSError, IOError):
1321 self
.report_error('Cannot write annotations file: ' + annofn
)
1324 subtitles_are_requested
= any([self
.params
.get('writesubtitles', False),
1325 self
.params
.get('writeautomaticsub')])
1327 if subtitles_are_requested
and info_dict
.get('requested_subtitles'):
1328 # subtitles download errors are already managed as troubles in relevant IE
1329 # that way it will silently go on when used with unsupporting IE
1330 subtitles
= info_dict
['requested_subtitles']
1331 ie
= self
.get_info_extractor(info_dict
['extractor_key'])
1332 for sub_lang
, sub_info
in subtitles
.items():
1333 sub_format
= sub_info
['ext']
1334 if sub_info
.get('data') is not None:
1335 sub_data
= sub_info
['data']
1338 sub_data
= ie
._download
_webpage
(
1339 sub_info
['url'], info_dict
['id'], note
=False)
1340 except ExtractorError
as err
:
1341 self
.report_warning('Unable to download subtitle for "%s": %s' %
1342 (sub_lang
, compat_str(err
.cause
)))
1345 sub_filename
= subtitles_filename(filename
, sub_lang
, sub_format
)
1346 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(sub_filename
)):
1347 self
.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang
, sub_format
))
1349 self
.to_screen('[info] Writing video subtitles to: ' + sub_filename
)
1350 with io
.open(encodeFilename(sub_filename
), 'w', encoding
='utf-8') as subfile
:
1351 subfile
.write(sub_data
)
1352 except (OSError, IOError):
1353 self
.report_error('Cannot write subtitles file ' + sub_filename
)
1356 if self
.params
.get('writeinfojson', False):
1357 infofn
= replace_extension(filename
, 'info.json', info_dict
.get('ext'))
1358 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(infofn
)):
1359 self
.to_screen('[info] Video description metadata is already present')
1361 self
.to_screen('[info] Writing video description metadata as JSON to: ' + infofn
)
1363 write_json_file(self
.filter_requested_info(info_dict
), infofn
)
1364 except (OSError, IOError):
1365 self
.report_error('Cannot write metadata to JSON file ' + infofn
)
1368 self
._write
_thumbnails
(info_dict
, filename
)
1370 if not self
.params
.get('skip_download', False):
1373 fd
= get_suitable_downloader(info
, self
.params
)(self
, self
.params
)
1374 for ph
in self
._progress
_hooks
:
1375 fd
.add_progress_hook(ph
)
1376 if self
.params
.get('verbose'):
1377 self
.to_stdout('[debug] Invoking downloader on %r' % info
.get('url'))
1378 return fd
.download(name
, info
)
1380 if info_dict
.get('requested_formats') is not None:
1383 merger
= FFmpegMergerPP(self
)
1384 if not merger
.available
:
1386 self
.report_warning('You have requested multiple '
1387 'formats but ffmpeg or avconv are not installed.'
1388 ' The formats won\'t be merged.')
1390 postprocessors
= [merger
]
1392 def compatible_formats(formats
):
1393 video
, audio
= formats
1395 video_ext
, audio_ext
= audio
.get('ext'), video
.get('ext')
1396 if video_ext
and audio_ext
:
1398 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1401 for exts
in COMPATIBLE_EXTS
:
1402 if video_ext
in exts
and audio_ext
in exts
:
1404 # TODO: Check acodec/vcodec
1407 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
1409 os
.path
.splitext(filename
)[0]
1410 if filename_real_ext
== info_dict
['ext']
1412 requested_formats
= info_dict
['requested_formats']
1413 if self
.params
.get('merge_output_format') is None and not compatible_formats(requested_formats
):
1414 info_dict
['ext'] = 'mkv'
1415 self
.report_warning(
1416 'Requested formats are incompatible for merge and will be merged into mkv.')
1417 # Ensure filename always has a correct extension for successful merge
1418 filename
= '%s.%s' % (filename_wo_ext
, info_dict
['ext'])
1419 if os
.path
.exists(encodeFilename(filename
)):
1421 '[download] %s has already been downloaded and '
1422 'merged' % filename
)
1424 for f
in requested_formats
:
1425 new_info
= dict(info_dict
)
1427 fname
= self
.prepare_filename(new_info
)
1428 fname
= prepend_extension(fname
, 'f%s' % f
['format_id'], new_info
['ext'])
1429 downloaded
.append(fname
)
1430 partial_success
= dl(fname
, new_info
)
1431 success
= success
and partial_success
1432 info_dict
['__postprocessors'] = postprocessors
1433 info_dict
['__files_to_merge'] = downloaded
1435 # Just a single file
1436 success
= dl(filename
, info_dict
)
1437 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
1438 self
.report_error('unable to download video data: %s' % str(err
))
1440 except (OSError, IOError) as err
:
1441 raise UnavailableVideoError(err
)
1442 except (ContentTooShortError
, ) as err
:
1443 self
.report_error('content too short (expected %s bytes and served %s)' % (err
.expected
, err
.downloaded
))
1448 fixup_policy
= self
.params
.get('fixup')
1449 if fixup_policy
is None:
1450 fixup_policy
= 'detect_or_warn'
1452 stretched_ratio
= info_dict
.get('stretched_ratio')
1453 if stretched_ratio
is not None and stretched_ratio
!= 1:
1454 if fixup_policy
== 'warn':
1455 self
.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1456 info_dict
['id'], stretched_ratio
))
1457 elif fixup_policy
== 'detect_or_warn':
1458 stretched_pp
= FFmpegFixupStretchedPP(self
)
1459 if stretched_pp
.available
:
1460 info_dict
.setdefault('__postprocessors', [])
1461 info_dict
['__postprocessors'].append(stretched_pp
)
1463 self
.report_warning(
1464 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1465 info_dict
['id'], stretched_ratio
))
1467 assert fixup_policy
in ('ignore', 'never')
1469 if info_dict
.get('requested_formats') is None and info_dict
.get('container') == 'm4a_dash':
1470 if fixup_policy
== 'warn':
1471 self
.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1473 elif fixup_policy
== 'detect_or_warn':
1474 fixup_pp
= FFmpegFixupM4aPP(self
)
1475 if fixup_pp
.available
:
1476 info_dict
.setdefault('__postprocessors', [])
1477 info_dict
['__postprocessors'].append(fixup_pp
)
1479 self
.report_warning(
1480 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1483 assert fixup_policy
in ('ignore', 'never')
1486 self
.post_process(filename
, info_dict
)
1487 except (PostProcessingError
) as err
:
1488 self
.report_error('postprocessing: %s' % str(err
))
1490 self
.record_download_archive(info_dict
)
1492 def download(self
, url_list
):
1493 """Download a given list of URLs."""
1494 outtmpl
= self
.params
.get('outtmpl', DEFAULT_OUTTMPL
)
1495 if (len(url_list
) > 1 and
1496 '%' not in outtmpl
and
1497 self
.params
.get('max_downloads') != 1):
1498 raise SameFileError(outtmpl
)
1500 for url
in url_list
:
1502 # It also downloads the videos
1503 res
= self
.extract_info(url
)
1504 except UnavailableVideoError
:
1505 self
.report_error('unable to download video')
1506 except MaxDownloadsReached
:
1507 self
.to_screen('[info] Maximum number of downloaded files reached.')
1510 if self
.params
.get('dump_single_json', False):
1511 self
.to_stdout(json
.dumps(res
))
1513 return self
._download
_retcode
1515 def download_with_info_file(self
, info_filename
):
1516 with contextlib
.closing(fileinput
.FileInput(
1517 [info_filename
], mode
='r',
1518 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
1519 # FileInput doesn't have a read method, we can't call json.load
1520 info
= self
.filter_requested_info(json
.loads('\n'.join(f
)))
1522 self
.process_ie_result(info
, download
=True)
1523 except DownloadError
:
1524 webpage_url
= info
.get('webpage_url')
1525 if webpage_url
is not None:
1526 self
.report_warning('The info failed to download, trying with "%s"' % webpage_url
)
1527 return self
.download([webpage_url
])
1530 return self
._download
_retcode
1533 def filter_requested_info(info_dict
):
1535 (k
, v
) for k
, v
in info_dict
.items()
1536 if k
not in ['requested_formats', 'requested_subtitles'])
1538 def post_process(self
, filename
, ie_info
):
1539 """Run all the postprocessors on the given file."""
1540 info
= dict(ie_info
)
1541 info
['filepath'] = filename
1543 if ie_info
.get('__postprocessors') is not None:
1544 pps_chain
.extend(ie_info
['__postprocessors'])
1545 pps_chain
.extend(self
._pps
)
1546 for pp
in pps_chain
:
1547 files_to_delete
= []
1549 files_to_delete
, info
= pp
.run(info
)
1550 except PostProcessingError
as e
:
1551 self
.report_error(e
.msg
)
1552 if files_to_delete
and not self
.params
.get('keepvideo', False):
1553 for old_filename
in files_to_delete
:
1554 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
1556 os
.remove(encodeFilename(old_filename
))
1557 except (IOError, OSError):
1558 self
.report_warning('Unable to remove downloaded original file')
1560 def _make_archive_id(self
, info_dict
):
1561 # Future-proof against any change in case
1562 # and backwards compatibility with prior versions
1563 extractor
= info_dict
.get('extractor_key')
1564 if extractor
is None:
1565 if 'id' in info_dict
:
1566 extractor
= info_dict
.get('ie_key') # key in a playlist
1567 if extractor
is None:
1568 return None # Incomplete video information
1569 return extractor
.lower() + ' ' + info_dict
['id']
1571 def in_download_archive(self
, info_dict
):
1572 fn
= self
.params
.get('download_archive')
1576 vid_id
= self
._make
_archive
_id
(info_dict
)
1578 return False # Incomplete video information
1581 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
1582 for line
in archive_file
:
1583 if line
.strip() == vid_id
:
1585 except IOError as ioe
:
1586 if ioe
.errno
!= errno
.ENOENT
:
1590 def record_download_archive(self
, info_dict
):
1591 fn
= self
.params
.get('download_archive')
1594 vid_id
= self
._make
_archive
_id
(info_dict
)
1596 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
1597 archive_file
.write(vid_id
+ '\n')
1600 def format_resolution(format
, default
='unknown'):
1601 if format
.get('vcodec') == 'none':
1603 if format
.get('resolution') is not None:
1604 return format
['resolution']
1605 if format
.get('height') is not None:
1606 if format
.get('width') is not None:
1607 res
= '%sx%s' % (format
['width'], format
['height'])
1609 res
= '%sp' % format
['height']
1610 elif format
.get('width') is not None:
1611 res
= '?x%d' % format
['width']
1616 def _format_note(self
, fdict
):
1618 if fdict
.get('ext') in ['f4f', 'f4m']:
1619 res
+= '(unsupported) '
1620 if fdict
.get('format_note') is not None:
1621 res
+= fdict
['format_note'] + ' '
1622 if fdict
.get('tbr') is not None:
1623 res
+= '%4dk ' % fdict
['tbr']
1624 if fdict
.get('container') is not None:
1627 res
+= '%s container' % fdict
['container']
1628 if (fdict
.get('vcodec') is not None and
1629 fdict
.get('vcodec') != 'none'):
1632 res
+= fdict
['vcodec']
1633 if fdict
.get('vbr') is not None:
1635 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
1637 if fdict
.get('vbr') is not None:
1638 res
+= '%4dk' % fdict
['vbr']
1639 if fdict
.get('fps') is not None:
1640 res
+= ', %sfps' % fdict
['fps']
1641 if fdict
.get('acodec') is not None:
1644 if fdict
['acodec'] == 'none':
1647 res
+= '%-5s' % fdict
['acodec']
1648 elif fdict
.get('abr') is not None:
1652 if fdict
.get('abr') is not None:
1653 res
+= '@%3dk' % fdict
['abr']
1654 if fdict
.get('asr') is not None:
1655 res
+= ' (%5dHz)' % fdict
['asr']
1656 if fdict
.get('filesize') is not None:
1659 res
+= format_bytes(fdict
['filesize'])
1660 elif fdict
.get('filesize_approx') is not None:
1663 res
+= '~' + format_bytes(fdict
['filesize_approx'])
1666 def list_formats(self
, info_dict
):
1667 formats
= info_dict
.get('formats', [info_dict
])
1669 [f
['format_id'], f
['ext'], self
.format_resolution(f
), self
._format
_note
(f
)]
1671 if f
.get('preference') is None or f
['preference'] >= -1000]
1672 if len(formats
) > 1:
1673 table
[-1][-1] += (' ' if table
[-1][-1] else '') + '(best)'
1675 header_line
= ['format code', 'extension', 'resolution', 'note']
1677 '[info] Available formats for %s:\n%s' %
1678 (info_dict
['id'], render_table(header_line
, table
)))
1680 def list_thumbnails(self
, info_dict
):
1681 thumbnails
= info_dict
.get('thumbnails')
1683 tn_url
= info_dict
.get('thumbnail')
1685 thumbnails
= [{'id': '0', 'url': tn_url}
]
1688 '[info] No thumbnails present for %s' % info_dict
['id'])
1692 '[info] Thumbnails for %s:' % info_dict
['id'])
1693 self
.to_screen(render_table(
1694 ['ID', 'width', 'height', 'URL'],
1695 [[t
['id'], t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
]))
1697 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
1699 self
.to_screen('%s has no %s' % (video_id
, name
))
1702 'Available %s for %s:' % (name
, video_id
))
1703 self
.to_screen(render_table(
1704 ['Language', 'formats'],
1705 [[lang
, ', '.join(f
['ext'] for f
in reversed(formats
))]
1706 for lang
, formats
in subtitles
.items()]))
1708 def urlopen(self
, req
):
1709 """ Start an HTTP download """
1711 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1712 # always respected by websites, some tend to give out URLs with non percent-encoded
1713 # non-ASCII characters (see telemb.py, ard.py [#3412])
1714 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1715 # To work around aforementioned issue we will replace request's original URL with
1716 # percent-encoded one
1717 req_is_string
= isinstance(req
, compat_basestring
)
1718 url
= req
if req_is_string
else req
.get_full_url()
1719 url_escaped
= escape_url(url
)
1721 # Substitute URL if any change after escaping
1722 if url
!= url_escaped
:
1726 req_type
= HEADRequest
if req
.get_method() == 'HEAD' else compat_urllib_request
.Request
1728 url_escaped
, data
=req
.data
, headers
=req
.headers
,
1729 origin_req_host
=req
.origin_req_host
, unverifiable
=req
.unverifiable
)
1731 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
1733 def print_debug_header(self
):
1734 if not self
.params
.get('verbose'):
1737 if type('') is not compat_str
:
1738 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1739 self
.report_warning(
1740 'Your Python is broken! Update to a newer and supported version')
1742 stdout_encoding
= getattr(
1743 sys
.stdout
, 'encoding', 'missing (%s)' % type(sys
.stdout
).__name
__)
1745 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1746 locale
.getpreferredencoding(),
1747 sys
.getfilesystemencoding(),
1749 self
.get_encoding()))
1750 write_string(encoding_str
, encoding
=None)
1752 self
._write
_string
('[debug] youtube-dl version ' + __version__
+ '\n')
1754 sp
= subprocess
.Popen(
1755 ['git', 'rev-parse', '--short', 'HEAD'],
1756 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
1757 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
1758 out
, err
= sp
.communicate()
1759 out
= out
.decode().strip()
1760 if re
.match('[0-9a-f]+', out
):
1761 self
._write
_string
('[debug] Git HEAD: ' + out
+ '\n')
1767 self
._write
_string
('[debug] Python version %s - %s\n' % (
1768 platform
.python_version(), platform_name()))
1770 exe_versions
= FFmpegPostProcessor
.get_versions(self
)
1771 exe_versions
['rtmpdump'] = rtmpdump_version()
1772 exe_str
= ', '.join(
1774 for exe
, v
in sorted(exe_versions
.items())
1779 self
._write
_string
('[debug] exe versions: %s\n' % exe_str
)
1782 for handler
in self
._opener
.handlers
:
1783 if hasattr(handler
, 'proxies'):
1784 proxy_map
.update(handler
.proxies
)
1785 self
._write
_string
('[debug] Proxy map: ' + compat_str(proxy_map
) + '\n')
1787 if self
.params
.get('call_home', False):
1788 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1789 self
._write
_string
('[debug] Public IP address: %s\n' % ipaddr
)
1790 latest_version
= self
.urlopen(
1791 'https://yt-dl.org/latest/version').read().decode('utf-8')
1792 if version_tuple(latest_version
) > version_tuple(__version__
):
1793 self
.report_warning(
1794 'You are using an outdated version (newest version: %s)! '
1795 'See https://yt-dl.org/update if you need help updating.' %
1798 def _setup_opener(self
):
1799 timeout_val
= self
.params
.get('socket_timeout')
1800 self
._socket
_timeout
= 600 if timeout_val
is None else float(timeout_val
)
1802 opts_cookiefile
= self
.params
.get('cookiefile')
1803 opts_proxy
= self
.params
.get('proxy')
1805 if opts_cookiefile
is None:
1806 self
.cookiejar
= compat_cookiejar
.CookieJar()
1808 self
.cookiejar
= compat_cookiejar
.MozillaCookieJar(
1810 if os
.access(opts_cookiefile
, os
.R_OK
):
1811 self
.cookiejar
.load()
1813 cookie_processor
= compat_urllib_request
.HTTPCookieProcessor(
1815 if opts_proxy
is not None:
1816 if opts_proxy
== '':
1819 proxies
= {'http': opts_proxy, 'https': opts_proxy}
1821 proxies
= compat_urllib_request
.getproxies()
1822 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1823 if 'http' in proxies
and 'https' not in proxies
:
1824 proxies
['https'] = proxies
['http']
1825 proxy_handler
= PerRequestProxyHandler(proxies
)
1827 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
1828 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
1829 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
1830 opener
= compat_urllib_request
.build_opener(
1831 proxy_handler
, https_handler
, cookie_processor
, ydlh
)
1833 # Delete the default user-agent header, which would otherwise apply in
1834 # cases where our custom HTTP handler doesn't come into play
1835 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1836 opener
.addheaders
= []
1837 self
._opener
= opener
1839 def encode(self
, s
):
1840 if isinstance(s
, bytes):
1841 return s
# Already encoded
1844 return s
.encode(self
.get_encoding())
1845 except UnicodeEncodeError as err
:
1846 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
1849 def get_encoding(self
):
1850 encoding
= self
.params
.get('encoding')
1851 if encoding
is None:
1852 encoding
= preferredencoding()
1855 def _write_thumbnails(self
, info_dict
, filename
):
1856 if self
.params
.get('writethumbnail', False):
1857 thumbnails
= info_dict
.get('thumbnails')
1859 thumbnails
= [thumbnails
[-1]]
1860 elif self
.params
.get('write_all_thumbnails', False):
1861 thumbnails
= info_dict
.get('thumbnails')
1866 # No thumbnails present, so return immediately
1869 for t
in thumbnails
:
1870 thumb_ext
= determine_ext(t
['url'], 'jpg')
1871 suffix
= '_%s' % t
['id'] if len(thumbnails
) > 1 else ''
1872 thumb_display_id
= '%s ' % t
['id'] if len(thumbnails
) > 1 else ''
1873 t
['filename'] = thumb_filename
= os
.path
.splitext(filename
)[0] + suffix
+ '.' + thumb_ext
1875 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(thumb_filename
)):
1876 self
.to_screen('[%s] %s: Thumbnail %sis already present' %
1877 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
1879 self
.to_screen('[%s] %s: Downloading thumbnail %s...' %
1880 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
1882 uf
= self
.urlopen(t
['url'])
1883 with open(thumb_filename
, 'wb') as thumbf
:
1884 shutil
.copyfileobj(uf
, thumbf
)
1885 self
.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1886 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
, thumb_filename
))
1887 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
1888 self
.report_warning('Unable to download thumbnail "%s": %s' %
1889 (t
['url'], compat_str(err
)))