4 from __future__
import absolute_import
, unicode_literals
29 from string
import ascii_letters
30 from zipimport
import zipimporter
35 compat_get_terminal_size
,
41 compat_tokenize_tokenize
,
43 compat_urllib_request
,
44 compat_urllib_request_DataHandler
,
56 DOT_DESKTOP_LINK_TEMPLATE
,
57 DOT_URL_LINK_TEMPLATE
,
58 DOT_WEBLOC_LINK_TEMPLATE
,
83 PerRequestProxyHandler
,
88 register_socks_protocols
,
102 UnavailableVideoError
,
108 YoutubeDLCookieProcessor
,
110 YoutubeDLRedirectHandler
,
111 process_communicate_or_kill
,
113 from .cache
import Cache
114 from .extractor
import get_info_extractor
, gen_extractor_classes
, _LAZY_LOADER
, _PLUGIN_CLASSES
115 from .extractor
.openload
import PhantomJSwrapper
116 from .downloader
import get_suitable_downloader
117 from .downloader
.rtmp
import rtmpdump_version
118 from .postprocessor
import (
121 FFmpegFixupStretchedPP
,
124 # FFmpegSubtitlesConvertorPP,
126 MoveFilesAfterDownloadPP
,
128 from .version
import __version__
130 if compat_os_name
== 'nt':
134 class YoutubeDL(object):
137 YoutubeDL objects are the ones responsible of downloading the
138 actual video file and writing it to disk if the user has requested
139 it, among some other tasks. In most cases there should be one per
140 program. As, given a video URL, the downloader doesn't know how to
141 extract all the needed information, task that InfoExtractors do, it
142 has to pass the URL to one of them.
144 For this, YoutubeDL objects have a method that allows
145 InfoExtractors to be registered in a given order. When it is passed
146 a URL, the YoutubeDL object handles it to the first InfoExtractor it
147 finds that reports being able to handle it. The InfoExtractor extracts
148 all the information about the video or videos the URL refers to, and
149 YoutubeDL process the extracted information, possibly using a File
150 Downloader to download the video.
152 YoutubeDL objects accept a lot of parameters. In order not to saturate
153 the object constructor with arguments, it receives a dictionary of
154 options instead. These options are available through the params
155 attribute for the InfoExtractors to use. The YoutubeDL also
156 registers itself as the downloader in charge for the InfoExtractors
157 that are added to it, so this is a "mutual registration".
161 username: Username for authentication purposes.
162 password: Password for authentication purposes.
163 videopassword: Password for accessing a video.
164 ap_mso: Adobe Pass multiple-system operator identifier.
165 ap_username: Multiple-system operator account username.
166 ap_password: Multiple-system operator account password.
167 usenetrc: Use netrc for authentication instead.
168 verbose: Print additional info to stdout.
169 quiet: Do not print messages to stdout.
170 no_warnings: Do not print out anything for warnings.
171 forceurl: Force printing final URL.
172 forcetitle: Force printing title.
173 forceid: Force printing ID.
174 forcethumbnail: Force printing thumbnail URL.
175 forcedescription: Force printing description.
176 forcefilename: Force printing final filename.
177 forceduration: Force printing duration.
178 forcejson: Force printing info_dict as JSON.
179 dump_single_json: Force printing the info_dict of the whole playlist
180 (or video) as a single JSON line.
181 force_write_download_archive: Force writing download archive regardless
182 of 'skip_download' or 'simulate'.
183 simulate: Do not download the video files.
184 format: Video format code. see "FORMAT SELECTION" for more details.
185 allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
186 format_sort: How to sort the video formats. see "Sorting Formats"
188 format_sort_force: Force the given format_sort. see "Sorting Formats"
190 allow_multiple_video_streams: Allow multiple video streams to be merged
192 allow_multiple_audio_streams: Allow multiple audio streams to be merged
194 paths: Dictionary of output paths. The allowed keys are 'home'
195 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
196 outtmpl: Dictionary of templates for output names. Allowed keys
197 are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
198 A string a also accepted for backward compatibility
199 outtmpl_na_placeholder: Placeholder for unavailable meta fields.
200 restrictfilenames: Do not allow "&" and spaces in file names
201 trim_file_name: Limit length of filename (extension excluded)
202 windowsfilenames: Force the filenames to be windows compatible
203 ignoreerrors: Do not stop on download errors
204 (Default True when running yt-dlp,
205 but False when directly accessing YoutubeDL class)
206 force_generic_extractor: Force downloader to use the generic extractor
207 overwrites: Overwrite all video and metadata files if True,
208 overwrite only non-video files if None
209 and don't overwrite any file if False
210 playliststart: Playlist item to start at.
211 playlistend: Playlist item to end at.
212 playlist_items: Specific indices of playlist to download.
213 playlistreverse: Download playlist items in reverse order.
214 playlistrandom: Download playlist items in random order.
215 matchtitle: Download only matching titles.
216 rejecttitle: Reject downloads for matching titles.
217 logger: Log messages to a logging.Logger instance.
218 logtostderr: Log messages to stderr instead of stdout.
219 writedescription: Write the video description to a .description file
220 writeinfojson: Write the video description to a .info.json file
221 clean_infojson: Remove private fields from the infojson
222 writecomments: Extract video comments. This will not be written to disk
223 unless writeinfojson is also given
224 writeannotations: Write the video annotations to a .annotations.xml file
225 writethumbnail: Write the thumbnail image to a file
226 allow_playlist_files: Whether to write playlists' description, infojson etc
227 also to disk when using the 'write*' options
228 write_all_thumbnails: Write all thumbnail formats to files
229 writelink: Write an internet shortcut file, depending on the
230 current platform (.url/.webloc/.desktop)
231 writeurllink: Write a Windows internet shortcut file (.url)
232 writewebloclink: Write a macOS internet shortcut file (.webloc)
233 writedesktoplink: Write a Linux internet shortcut file (.desktop)
234 writesubtitles: Write the video subtitles to a file
235 writeautomaticsub: Write the automatically generated subtitles to a file
236 allsubtitles: Downloads all the subtitles of the video
237 (requires writesubtitles or writeautomaticsub)
238 listsubtitles: Lists all available subtitles for the video
239 subtitlesformat: The format code for subtitles
240 subtitleslangs: List of languages of the subtitles to download
241 keepvideo: Keep the video file after post-processing
242 daterange: A DateRange object, download only if the upload_date is in the range.
243 skip_download: Skip the actual download of the video file
244 cachedir: Location of the cache files in the filesystem.
245 False to disable filesystem cache.
246 noplaylist: Download single video instead of a playlist if in doubt.
247 age_limit: An integer representing the user's age in years.
248 Unsuitable videos for the given age are skipped.
249 min_views: An integer representing the minimum view count the video
250 must have in order to not be skipped.
251 Videos without view count information are always
252 downloaded. None for no limit.
253 max_views: An integer representing the maximum view count.
254 Videos that are more popular than that are not
256 Videos without view count information are always
257 downloaded. None for no limit.
258 download_archive: File name of a file where all downloads are recorded.
259 Videos already present in the file are not downloaded
261 break_on_existing: Stop the download process after attempting to download a
262 file that is in the archive.
263 break_on_reject: Stop the download process when encountering a video that
264 has been filtered out.
265 cookiefile: File name where cookies should be read from and dumped to
266 nocheckcertificate:Do not verify SSL certificates
267 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
268 At the moment, this is only supported by YouTube.
269 proxy: URL of the proxy server to use
270 geo_verification_proxy: URL of the proxy to use for IP address verification
271 on geo-restricted sites.
272 socket_timeout: Time to wait for unresponsive hosts, in seconds
273 bidi_workaround: Work around buggy terminals without bidirectional text
274 support, using fridibi
275 debug_printtraffic:Print out sent and received HTTP traffic
276 include_ads: Download ads as well
277 default_search: Prepend this string if an input url is not valid.
278 'auto' for elaborate guessing
279 encoding: Use this encoding instead of the system-specified.
280 extract_flat: Do not resolve URLs, return the immediate result.
281 Pass in 'in_playlist' to only show this behavior for
283 postprocessors: A list of dictionaries, each with an entry
284 * key: The name of the postprocessor. See
285 yt_dlp/postprocessor/__init__.py for a list.
286 * _after_move: Optional. If True, run this post_processor
287 after 'MoveFilesAfterDownload'
288 as well as any further keyword arguments for the
290 post_hooks: A list of functions that get called as the final step
291 for each video file, after all postprocessors have been
292 called. The filename will be passed as the only argument.
293 progress_hooks: A list of functions that get called on download
294 progress, with a dictionary with the entries
295 * status: One of "downloading", "error", or "finished".
296 Check this first and ignore unknown values.
298 If status is one of "downloading", or "finished", the
299 following properties may also be present:
300 * filename: The final filename (always present)
301 * tmpfilename: The filename we're currently writing to
302 * downloaded_bytes: Bytes on disk
303 * total_bytes: Size of the whole file, None if unknown
304 * total_bytes_estimate: Guess of the eventual file size,
306 * elapsed: The number of seconds since download started.
307 * eta: The estimated time in seconds, None if unknown
308 * speed: The download speed in bytes/second, None if
310 * fragment_index: The counter of the currently
311 downloaded video fragment.
312 * fragment_count: The number of fragments (= individual
313 files that will be merged)
315 Progress hooks are guaranteed to be called at least once
316 (with status "finished") if the download is successful.
317 merge_output_format: Extension to use when merging formats.
318 final_ext: Expected final extension; used to detect when the file was
319 already downloaded and converted. "merge_output_format" is
320 replaced by this extension when given
321 fixup: Automatically correct known faults of the file.
323 - "never": do nothing
324 - "warn": only emit a warning
325 - "detect_or_warn": check whether we can do anything
326 about it, warn otherwise (default)
327 source_address: Client-side IP address to bind to.
328 call_home: Boolean, true iff we are allowed to contact the
329 yt-dlp servers for debugging. (BROKEN)
330 sleep_interval_requests: Number of seconds to sleep between requests
332 sleep_interval: Number of seconds to sleep before each download when
333 used alone or a lower bound of a range for randomized
334 sleep before each download (minimum possible number
335 of seconds to sleep) when used along with
337 max_sleep_interval:Upper bound of a range for randomized sleep before each
338 download (maximum possible number of seconds to sleep).
339 Must only be used along with sleep_interval.
340 Actual sleep time will be a random float from range
341 [sleep_interval; max_sleep_interval].
342 sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
343 listformats: Print an overview of available video formats and exit.
344 list_thumbnails: Print a table of all thumbnails and exit.
345 match_filter: A function that gets called with the info_dict of
347 If it returns a message, the video is ignored.
348 If it returns None, the video is downloaded.
349 match_filter_func in utils.py is one example for this.
350 no_color: Do not emit color codes in output.
351 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
354 Two-letter ISO 3166-2 country code that will be used for
355 explicit geographic restriction bypassing via faking
356 X-Forwarded-For HTTP header
358 IP range in CIDR notation that will be used similarly to
361 The following options determine which downloader is picked:
362 external_downloader: Executable of the external downloader to call.
363 None or unset for standard (built-in) downloader.
364 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
365 if True, otherwise use ffmpeg/avconv if False, otherwise
366 use downloader suggested by extractor if None.
368 The following parameters are not used by YoutubeDL itself, they are used by
369 the downloader (see yt_dlp/downloader/common.py):
370 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
371 noresizebuffer, retries, continuedl, noprogress, consoletitle,
372 xattr_set_filesize, external_downloader_args, hls_use_mpegts,
375 The following options are used by the post processors:
376 prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
377 otherwise prefer ffmpeg. (avconv support is deprecated)
378 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
379 to the binary or its containing directory.
380 postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
381 and a list of additional command-line arguments for the
382 postprocessor/executable. The dict can also have "PP+EXE" keys
383 which are used when the given exe is used by the given PP.
384 Use 'default' as the name for arguments to passed to all PP
386 The following options are used by the extractors:
387 extractor_retries: Number of times to retry for known errors
388 dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
389 hls_split_discontinuity: Split HLS playlists to different formats at
390 discontinuities such as ad breaks (default: False)
391 youtube_include_dash_manifest: If True (default), DASH manifests and related
392 data will be downloaded and processed by extractor.
393 You can reduce network I/O by disabling it if you don't
394 care about DASH. (only for youtube)
395 youtube_include_hls_manifest: If True (default), HLS manifests and related
396 data will be downloaded and processed by extractor.
397 You can reduce network I/O by disabling it if you don't
398 care about HLS. (only for youtube)
401 _NUMERIC_FIELDS
= set((
402 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
403 'timestamp', 'upload_year', 'upload_month', 'upload_day',
404 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
405 'average_rating', 'comment_count', 'age_limit',
406 'start_time', 'end_time',
407 'chapter_number', 'season_number', 'episode_number',
408 'track_number', 'disc_number', 'release_year',
414 _pps
= {'beforedl': [], 'aftermove': [], 'normal': []}
415 __prepare_filename_warned
= False
416 _first_webpage_request
= True
417 _download_retcode
= None
418 _num_downloads
= None
420 _playlist_urls
= set()
423 def __init__(self
, params
=None, auto_init
=True):
424 """Create a FileDownloader object with the given options."""
428 self
._ies
_instances
= {}
429 self
._pps
= {'beforedl': [], 'aftermove': [], 'normal': []}
430 self
.__prepare
_filename
_warned
= False
431 self
._first
_webpage
_request
= True
432 self
._post
_hooks
= []
433 self
._progress
_hooks
= []
434 self
._download
_retcode
= 0
435 self
._num
_downloads
= 0
436 self
._screen
_file
= [sys
.stdout
, sys
.stderr
][params
.get('logtostderr', False)]
437 self
._err
_file
= sys
.stderr
440 'nocheckcertificate': False,
442 self
.params
.update(params
)
443 self
.cache
= Cache(self
)
446 """Preload the archive, if any is specified"""
447 def preload_download_archive(self
):
448 fn
= self
.params
.get('download_archive')
452 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
453 for line
in archive_file
:
454 self
.archive
.add(line
.strip())
455 except IOError as ioe
:
456 if ioe
.errno
!= errno
.ENOENT
:
461 def check_deprecated(param
, option
, suggestion
):
462 if self
.params
.get(param
) is not None:
464 '%s is deprecated. Use %s instead.' % (option
, suggestion
))
468 if self
.params
.get('verbose'):
469 self
.to_stdout('[debug] Loading archive file %r' % self
.params
.get('download_archive'))
471 preload_download_archive(self
)
473 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
474 if self
.params
.get('geo_verification_proxy') is None:
475 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
477 if self
.params
.get('final_ext'):
478 if self
.params
.get('merge_output_format'):
479 self
.report_warning('--merge-output-format will be ignored since --remux-video or --recode-video is given')
480 self
.params
['merge_output_format'] = self
.params
['final_ext']
482 if 'overwrites' in self
.params
and self
.params
['overwrites'] is None:
483 del self
.params
['overwrites']
485 check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
486 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
487 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
489 if params
.get('bidi_workaround', False):
492 master
, slave
= pty
.openpty()
493 width
= compat_get_terminal_size().columns
497 width_args
= ['-w', str(width
)]
499 stdin
=subprocess
.PIPE
,
501 stderr
=self
._err
_file
)
503 self
._output
_process
= subprocess
.Popen(
504 ['bidiv'] + width_args
, **sp_kwargs
507 self
._output
_process
= subprocess
.Popen(
508 ['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
509 self
._output
_channel
= os
.fdopen(master
, 'rb')
510 except OSError as ose
:
511 if ose
.errno
== errno
.ENOENT
:
512 self
.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
516 if (sys
.platform
!= 'win32'
517 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
518 and not params
.get('restrictfilenames', False)):
519 # Unicode filesystem API will throw errors (#1474, #13027)
521 'Assuming --restrict-filenames since file system encoding '
522 'cannot encode all characters. '
523 'Set the LC_ALL environment variable to fix this.')
524 self
.params
['restrictfilenames'] = True
526 self
.outtmpl_dict
= self
.parse_outtmpl()
531 self
.print_debug_header()
532 self
.add_default_info_extractors()
534 for pp_def_raw
in self
.params
.get('postprocessors', []):
535 pp_class
= get_postprocessor(pp_def_raw
['key'])
536 pp_def
= dict(pp_def_raw
)
539 when
= pp_def
['when']
543 pp
= pp_class(self
, **compat_kwargs(pp_def
))
544 self
.add_post_processor(pp
, when
=when
)
546 for ph
in self
.params
.get('post_hooks', []):
547 self
.add_post_hook(ph
)
549 for ph
in self
.params
.get('progress_hooks', []):
550 self
.add_progress_hook(ph
)
552 register_socks_protocols()
554 def warn_if_short_id(self
, argv
):
555 # short YouTube ID starting with dash?
557 i
for i
, a
in enumerate(argv
)
558 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
562 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
563 + ['--'] + [argv
[i
] for i
in idxs
]
566 'Long argument string detected. '
567 'Use -- to separate parameters and URLs, like this:\n%s\n' %
568 args_to_str(correct_argv
))
570 def add_info_extractor(self
, ie
):
571 """Add an InfoExtractor object to the end of the list."""
573 if not isinstance(ie
, type):
574 self
._ies
_instances
[ie
.ie_key()] = ie
575 ie
.set_downloader(self
)
577 def get_info_extractor(self
, ie_key
):
579 Get an instance of an IE with name ie_key, it will try to get one from
580 the _ies list, if there's no instance it will create a new one and add
581 it to the extractor list.
583 ie
= self
._ies
_instances
.get(ie_key
)
585 ie
= get_info_extractor(ie_key
)()
586 self
.add_info_extractor(ie
)
589 def add_default_info_extractors(self
):
591 Add the InfoExtractors returned by gen_extractors to the end of the list
593 for ie
in gen_extractor_classes():
594 self
.add_info_extractor(ie
)
596 def add_post_processor(self
, pp
, when
='normal'):
597 """Add a PostProcessor object to the end of the chain."""
598 self
._pps
[when
].append(pp
)
599 pp
.set_downloader(self
)
601 def add_post_hook(self
, ph
):
602 """Add the post hook"""
603 self
._post
_hooks
.append(ph
)
605 def add_progress_hook(self
, ph
):
606 """Add the progress hook (currently only for the file downloader)"""
607 self
._progress
_hooks
.append(ph
)
609 def _bidi_workaround(self
, message
):
610 if not hasattr(self
, '_output_channel'):
613 assert hasattr(self
, '_output_process')
614 assert isinstance(message
, compat_str
)
615 line_count
= message
.count('\n') + 1
616 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
617 self
._output
_process
.stdin
.flush()
618 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
619 for _
in range(line_count
))
620 return res
[:-len('\n')]
622 def to_screen(self
, message
, skip_eol
=False):
623 """Print message to stdout if not in quiet mode."""
624 return self
.to_stdout(message
, skip_eol
, check_quiet
=True)
626 def _write_string(self
, s
, out
=None):
627 write_string(s
, out
=out
, encoding
=self
.params
.get('encoding'))
629 def to_stdout(self
, message
, skip_eol
=False, check_quiet
=False):
630 """Print message to stdout if not in quiet mode."""
631 if self
.params
.get('logger'):
632 self
.params
['logger'].debug(message
)
633 elif not check_quiet
or not self
.params
.get('quiet', False):
634 message
= self
._bidi
_workaround
(message
)
635 terminator
= ['\n', ''][skip_eol
]
636 output
= message
+ terminator
638 self
._write
_string
(output
, self
._screen
_file
)
640 def to_stderr(self
, message
):
641 """Print message to stderr."""
642 assert isinstance(message
, compat_str
)
643 if self
.params
.get('logger'):
644 self
.params
['logger'].error(message
)
646 message
= self
._bidi
_workaround
(message
)
647 output
= message
+ '\n'
648 self
._write
_string
(output
, self
._err
_file
)
650 def to_console_title(self
, message
):
651 if not self
.params
.get('consoletitle', False):
653 if compat_os_name
== 'nt':
654 if ctypes
.windll
.kernel32
.GetConsoleWindow():
655 # c_wchar_p() might not be necessary if `message` is
656 # already of type unicode()
657 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
658 elif 'TERM' in os
.environ
:
659 self
._write
_string
('\033]0;%s\007' % message
, self
._screen
_file
)
661 def save_console_title(self
):
662 if not self
.params
.get('consoletitle', False):
664 if self
.params
.get('simulate', False):
666 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
667 # Save the title on stack
668 self
._write
_string
('\033[22;0t', self
._screen
_file
)
670 def restore_console_title(self
):
671 if not self
.params
.get('consoletitle', False):
673 if self
.params
.get('simulate', False):
675 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
676 # Restore the title from stack
677 self
._write
_string
('\033[23;0t', self
._screen
_file
)
680 self
.save_console_title()
683 def __exit__(self
, *args
):
684 self
.restore_console_title()
686 if self
.params
.get('cookiefile') is not None:
687 self
.cookiejar
.save(ignore_discard
=True, ignore_expires
=True)
689 def trouble(self
, message
=None, tb
=None):
690 """Determine action to take when a download problem appears.
692 Depending on if the downloader has been configured to ignore
693 download errors or not, this method may throw an exception or
694 not when errors are found, after printing the message.
696 tb, if given, is additional traceback information.
698 if message
is not None:
699 self
.to_stderr(message
)
700 if self
.params
.get('verbose'):
702 if sys
.exc_info()[0]: # if .trouble has been called from an except block
704 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
705 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
706 tb
+= encode_compat_str(traceback
.format_exc())
708 tb_data
= traceback
.format_list(traceback
.extract_stack())
709 tb
= ''.join(tb_data
)
711 if not self
.params
.get('ignoreerrors', False):
712 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
713 exc_info
= sys
.exc_info()[1].exc_info
715 exc_info
= sys
.exc_info()
716 raise DownloadError(message
, exc_info
)
717 self
._download
_retcode
= 1
719 def report_warning(self
, message
):
721 Print the message to stderr, it will be prefixed with 'WARNING:'
722 If stderr is a tty file the 'WARNING:' will be colored
724 if self
.params
.get('logger') is not None:
725 self
.params
['logger'].warning(message
)
727 if self
.params
.get('no_warnings'):
729 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
730 _msg_header
= '\033[0;33mWARNING:\033[0m'
732 _msg_header
= 'WARNING:'
733 warning_message
= '%s %s' % (_msg_header
, message
)
734 self
.to_stderr(warning_message
)
736 def report_error(self
, message
, tb
=None):
738 Do the same as trouble, but prefixes the message with 'ERROR:', colored
739 in red if stderr is a tty file.
741 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
742 _msg_header
= '\033[0;31mERROR:\033[0m'
744 _msg_header
= 'ERROR:'
745 error_message
= '%s %s' % (_msg_header
, message
)
746 self
.trouble(error_message
, tb
)
748 def report_file_already_downloaded(self
, file_name
):
749 """Report file has already been fully downloaded."""
751 self
.to_screen('[download] %s has already been downloaded' % file_name
)
752 except UnicodeEncodeError:
753 self
.to_screen('[download] The file has already been downloaded')
755 def report_file_delete(self
, file_name
):
756 """Report that existing file will be deleted."""
758 self
.to_screen('Deleting existing file %s' % file_name
)
759 except UnicodeEncodeError:
760 self
.to_screen('Deleting existing file')
762 def parse_outtmpl(self
):
763 outtmpl_dict
= self
.params
.get('outtmpl', {})
764 if not isinstance(outtmpl_dict
, dict):
765 outtmpl_dict
= {'default': outtmpl_dict}
766 outtmpl_dict
.update({
767 k
: v
for k
, v
in DEFAULT_OUTTMPL
.items()
768 if not outtmpl_dict
.get(k
)})
769 for key
, val
in outtmpl_dict
.items():
770 if isinstance(val
, bytes):
772 'Parameter outtmpl is bytes, but should be a unicode string. '
773 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
776 def prepare_outtmpl(self
, outtmpl
, info_dict
, sanitize
=None):
777 """ Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
778 template_dict
= dict(info_dict
)
781 template_dict
['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
782 formatSeconds(info_dict
['duration'], '-')
783 if info_dict
.get('duration', None) is not None
787 template_dict
['epoch'] = int(time
.time())
790 autonumber_size
= self
.params
.get('autonumber_size')
791 if autonumber_size
is None:
793 template_dict
['autonumber'] = self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
795 # resolution if not defined
796 if template_dict
.get('resolution') is None:
797 if template_dict
.get('width') and template_dict
.get('height'):
798 template_dict
['resolution'] = '%dx%d' % (template_dict
['width'], template_dict
['height'])
799 elif template_dict
.get('height'):
800 template_dict
['resolution'] = '%sp' % template_dict
['height']
801 elif template_dict
.get('width'):
802 template_dict
['resolution'] = '%dx?' % template_dict
['width']
805 sanitize
= lambda k
, v
: v
806 template_dict
= dict((k
, v
if isinstance(v
, compat_numeric_types
) else sanitize(k
, v
))
807 for k
, v
in template_dict
.items()
808 if v
is not None and not isinstance(v
, (list, tuple, dict)))
809 na
= self
.params
.get('outtmpl_na_placeholder', 'NA')
810 template_dict
= collections
.defaultdict(lambda: na
, template_dict
)
812 # For fields playlist_index and autonumber convert all occurrences
813 # of %(field)s to %(field)0Nd for backward compatibility
814 field_size_compat_map
= {
815 'playlist_index': len(str(template_dict
['n_entries'])),
816 'autonumber': autonumber_size
,
818 FIELD_SIZE_COMPAT_RE
= r
'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
819 mobj
= re
.search(FIELD_SIZE_COMPAT_RE
, outtmpl
)
822 FIELD_SIZE_COMPAT_RE
,
823 r
'%%(\1)0%dd' % field_size_compat_map
[mobj
.group('field')],
826 numeric_fields
= list(self
._NUMERIC
_FIELDS
)
829 FORMAT_DATE_RE
= FORMAT_RE
.format(r
'(?P<key>(?P<field>\w+)>(?P<format>.+?))')
830 for mobj
in re
.finditer(FORMAT_DATE_RE
, outtmpl
):
831 conv_type
, field
, frmt
, key
= mobj
.group('type', 'field', 'format', 'key')
832 if key
in template_dict
:
834 value
= strftime_or_none(template_dict
.get(field
), frmt
, na
)
835 if conv_type
in 'crs': # string
836 value
= sanitize(field
, value
)
838 numeric_fields
.append(key
)
839 value
= float_or_none(value
, default
=None)
840 if value
is not None:
841 template_dict
[key
] = value
843 # Missing numeric fields used together with integer presentation types
844 # in format specification will break the argument substitution since
845 # string NA placeholder is returned for missing fields. We will patch
846 # output template for missing fields to meet string presentation type.
847 for numeric_field
in numeric_fields
:
848 if numeric_field
not in template_dict
:
850 FORMAT_RE
.format(re
.escape(numeric_field
)),
851 r
'%({0})s'.format(numeric_field
), outtmpl
)
853 return outtmpl
, template_dict
855 def _prepare_filename(self
, info_dict
, tmpl_type
='default'):
857 sanitize
= lambda k
, v
: sanitize_filename(
859 restricted
=self
.params
.get('restrictfilenames'),
860 is_id
=(k
== 'id' or k
.endswith('_id')))
861 outtmpl
= self
.outtmpl_dict
.get(tmpl_type
, self
.outtmpl_dict
['default'])
862 outtmpl
, template_dict
= self
.prepare_outtmpl(outtmpl
, info_dict
, sanitize
)
864 # expand_path translates '%%' into '%' and '$$' into '$'
865 # correspondingly that is not what we want since we need to keep
866 # '%%' intact for template dict substitution step. Working around
867 # with boundary-alike separator hack.
868 sep
= ''.join([random
.choice(ascii_letters
) for _
in range(32)])
869 outtmpl
= outtmpl
.replace('%%', '%{0}%'.format(sep
)).replace('$$', '${0}$'.format(sep
))
871 # outtmpl should be expand_path'ed before template dict substitution
872 # because meta fields may contain env variables we don't want to
873 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
874 # title "Hello $PATH", we don't want `$PATH` to be expanded.
875 filename
= expand_path(outtmpl
).replace(sep
, '') % template_dict
877 force_ext
= OUTTMPL_TYPES
.get(tmpl_type
)
878 if force_ext
is not None:
879 filename
= replace_extension(filename
, force_ext
, template_dict
.get('ext'))
881 # https://github.com/blackjack4494/youtube-dlc/issues/85
882 trim_file_name
= self
.params
.get('trim_file_name', False)
884 fn_groups
= filename
.rsplit('.')
887 if len(fn_groups
) > 2:
888 sub_ext
= fn_groups
[-2]
889 filename
= '.'.join(filter(None, [fn_groups
[0][:trim_file_name
], sub_ext
, ext
]))
892 except ValueError as err
:
893 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
896 def prepare_filename(self
, info_dict
, dir_type
='', warn
=False):
897 """Generate the output filename."""
898 paths
= self
.params
.get('paths', {})
899 assert isinstance(paths
, dict)
900 filename
= self
._prepare
_filename
(info_dict
, dir_type
or 'default')
902 if warn
and not self
.__prepare
_filename
_warned
:
905 elif filename
== '-':
906 self
.report_warning('--paths is ignored when an outputting to stdout')
907 elif os
.path
.isabs(filename
):
908 self
.report_warning('--paths is ignored since an absolute path is given in output template')
909 self
.__prepare
_filename
_warned
= True
910 if filename
== '-' or not filename
:
913 homepath
= expand_path(paths
.get('home', '').strip())
914 assert isinstance(homepath
, compat_str
)
915 subdir
= expand_path(paths
.get(dir_type
, '').strip()) if dir_type
else ''
916 assert isinstance(subdir
, compat_str
)
917 path
= os
.path
.join(homepath
, subdir
, filename
)
919 # Temporary fix for #4787
920 # 'Treat' all problem characters by passing filename through preferredencoding
921 # to workaround encoding issues with subprocess on python2 @ Windows
922 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
923 path
= encodeFilename(path
, True).decode(preferredencoding())
924 return sanitize_path(path
, force
=self
.params
.get('windowsfilenames'))
926 def _match_entry(self
, info_dict
, incomplete
):
927 """ Returns None if the file should be downloaded """
930 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
931 if 'title' in info_dict
:
932 # This can happen when we're just evaluating the playlist
933 title
= info_dict
['title']
934 matchtitle
= self
.params
.get('matchtitle', False)
936 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
937 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
938 rejecttitle
= self
.params
.get('rejecttitle', False)
940 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
941 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
942 date
= info_dict
.get('upload_date')
944 dateRange
= self
.params
.get('daterange', DateRange())
945 if date
not in dateRange
:
946 return '%s upload date is not in range %s' % (date_from_str(date
).isoformat(), dateRange
)
947 view_count
= info_dict
.get('view_count')
948 if view_count
is not None:
949 min_views
= self
.params
.get('min_views')
950 if min_views
is not None and view_count
< min_views
:
951 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
952 max_views
= self
.params
.get('max_views')
953 if max_views
is not None and view_count
> max_views
:
954 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
955 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
956 return 'Skipping "%s" because it is age restricted' % video_title
957 if self
.in_download_archive(info_dict
):
958 return '%s has already been recorded in archive' % video_title
961 match_filter
= self
.params
.get('match_filter')
962 if match_filter
is not None:
963 ret
= match_filter(info_dict
)
968 reason
= check_filter()
969 if reason
is not None:
970 self
.to_screen('[download] ' + reason
)
971 if reason
.endswith('has already been recorded in the archive') and self
.params
.get('break_on_existing', False):
972 raise ExistingVideoReached()
973 elif self
.params
.get('break_on_reject', False):
974 raise RejectedVideoReached()
978 def add_extra_info(info_dict
, extra_info
):
979 '''Set the keys from extra_info in info dict if they are missing'''
980 for key
, value
in extra_info
.items():
981 info_dict
.setdefault(key
, value
)
983 def extract_info(self
, url
, download
=True, ie_key
=None, info_dict
=None, extra_info
={},
984 process
=True, force_generic_extractor
=False):
986 Returns a list with a dictionary for each video we find.
987 If 'download', also downloads the videos.
988 extra_info is a dict containing the extra values to add to each result
991 if not ie_key
and force_generic_extractor
:
995 ies
= [self
.get_info_extractor(ie_key
)]
1000 if not ie
.suitable(url
):
1003 ie_key
= ie
.ie_key()
1004 ie
= self
.get_info_extractor(ie_key
)
1005 if not ie
.working():
1006 self
.report_warning('The program functionality for this site has been marked as broken, '
1007 'and will probably not work.')
1010 temp_id
= str_or_none(
1011 ie
.extract_id(url
) if callable(getattr(ie
, 'extract_id', None))
1012 else ie
._match
_id
(url
))
1013 except (AssertionError, IndexError, AttributeError):
1015 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': ie_key}
):
1016 self
.to_screen("[%s] %s: has already been recorded in archive" % (
1019 return self
.__extract
_info
(url
, ie
, download
, extra_info
, process
, info_dict
)
1021 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
1023 def __handle_extraction_exceptions(func
):
1024 def wrapper(self
, *args
, **kwargs
):
1026 return func(self
, *args
, **kwargs
)
1027 except GeoRestrictedError
as e
:
1030 msg
+= '\nThis video is available in %s.' % ', '.join(
1031 map(ISO3166Utils
.short2full
, e
.countries
))
1032 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
1033 self
.report_error(msg
)
1034 except ExtractorError
as e
: # An error we somewhat expected
1035 self
.report_error(compat_str(e
), e
.format_traceback())
1036 except (MaxDownloadsReached
, ExistingVideoReached
, RejectedVideoReached
):
1038 except Exception as e
:
1039 if self
.params
.get('ignoreerrors', False):
1040 self
.report_error(error_to_compat_str(e
), tb
=encode_compat_str(traceback
.format_exc()))
1045 @__handle_extraction_exceptions
1046 def __extract_info(self
, url
, ie
, download
, extra_info
, process
, info_dict
):
1047 ie_result
= ie
.extract(url
)
1048 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
1050 if isinstance(ie_result
, list):
1051 # Backwards compatibility: old IE result format
1053 '_type': 'compat_list',
1054 'entries': ie_result
,
1057 if info_dict
.get('id'):
1058 ie_result
['id'] = info_dict
['id']
1059 if info_dict
.get('title'):
1060 ie_result
['title'] = info_dict
['title']
1061 self
.add_default_extra_info(ie_result
, ie
, url
)
1063 return self
.process_ie_result(ie_result
, download
, extra_info
)
1067 def add_default_extra_info(self
, ie_result
, ie
, url
):
1068 self
.add_extra_info(ie_result
, {
1069 'extractor': ie
.IE_NAME
,
1071 'webpage_url_basename': url_basename(url
),
1072 'extractor_key': ie
.ie_key(),
1075 def process_ie_result(self
, ie_result
, download
=True, extra_info
={}):
1077 Take the result of the ie(may be modified) and resolve all unresolved
1078 references (URLs, playlist items).
1080 It will also download the videos if 'download'.
1081 Returns the resolved ie_result.
1083 result_type
= ie_result
.get('_type', 'video')
1085 if result_type
in ('url', 'url_transparent'):
1086 ie_result
['url'] = sanitize_url(ie_result
['url'])
1087 extract_flat
= self
.params
.get('extract_flat', False)
1088 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
1089 or extract_flat
is True):
1090 self
.__forced
_printings
(ie_result
, self
.prepare_filename(ie_result
), incomplete
=True)
1093 if result_type
== 'video':
1094 self
.add_extra_info(ie_result
, extra_info
)
1095 return self
.process_video_result(ie_result
, download
=download
)
1096 elif result_type
== 'url':
1097 # We have to add extra_info to the results because it may be
1098 # contained in a playlist
1099 return self
.extract_info(ie_result
['url'],
1100 download
, info_dict
=ie_result
,
1101 ie_key
=ie_result
.get('ie_key'),
1102 extra_info
=extra_info
)
1103 elif result_type
== 'url_transparent':
1104 # Use the information from the embedding page
1105 info
= self
.extract_info(
1106 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
1107 extra_info
=extra_info
, download
=False, process
=False)
1109 # extract_info may return None when ignoreerrors is enabled and
1110 # extraction failed with an error, don't crash and return early
1115 force_properties
= dict(
1116 (k
, v
) for k
, v
in ie_result
.items() if v
is not None)
1117 for f
in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
1118 if f
in force_properties
:
1119 del force_properties
[f
]
1120 new_result
= info
.copy()
1121 new_result
.update(force_properties
)
1123 # Extracted info may not be a video result (i.e.
1124 # info.get('_type', 'video') != video) but rather an url or
1125 # url_transparent. In such cases outer metadata (from ie_result)
1126 # should be propagated to inner one (info). For this to happen
1127 # _type of info should be overridden with url_transparent. This
1128 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
1129 if new_result
.get('_type') == 'url':
1130 new_result
['_type'] = 'url_transparent'
1132 return self
.process_ie_result(
1133 new_result
, download
=download
, extra_info
=extra_info
)
1134 elif result_type
in ('playlist', 'multi_video'):
1135 # Protect from infinite recursion due to recursively nested playlists
1136 # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
1137 webpage_url
= ie_result
['webpage_url']
1138 if webpage_url
in self
._playlist
_urls
:
1140 '[download] Skipping already downloaded playlist: %s'
1141 % ie_result
.get('title') or ie_result
.get('id'))
1144 self
._playlist
_level
+= 1
1145 self
._playlist
_urls
.add(webpage_url
)
1147 return self
.__process
_playlist
(ie_result
, download
)
1149 self
._playlist
_level
-= 1
1150 if not self
._playlist
_level
:
1151 self
._playlist
_urls
.clear()
1152 elif result_type
== 'compat_list':
1153 self
.report_warning(
1154 'Extractor %s returned a compat_list result. '
1155 'It needs to be updated.' % ie_result
.get('extractor'))
1158 self
.add_extra_info(
1161 'extractor': ie_result
['extractor'],
1162 'webpage_url': ie_result
['webpage_url'],
1163 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1164 'extractor_key': ie_result
['extractor_key'],
1168 ie_result
['entries'] = [
1169 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1170 for r
in ie_result
['entries']
1174 raise Exception('Invalid result type: %s' % result_type
)
1176 def _ensure_dir_exists(self
, path
):
1177 return make_dir(path
, self
.report_error
)
1179 def __process_playlist(self
, ie_result
, download
):
1180 # We process each entry in the playlist
1181 playlist
= ie_result
.get('title') or ie_result
.get('id')
1182 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
1184 if 'entries' not in ie_result
:
1185 raise EntryNotInPlaylist()
1186 incomplete_entries
= bool(ie_result
.get('requested_entries'))
1187 if incomplete_entries
:
1188 def fill_missing_entries(entries
, indexes
):
1189 ret
= [None] * max(*indexes
)
1190 for i
, entry
in zip(indexes
, entries
):
1193 ie_result
['entries'] = fill_missing_entries(ie_result
['entries'], ie_result
['requested_entries'])
1195 playlist_results
= []
1197 playliststart
= self
.params
.get('playliststart', 1) - 1
1198 playlistend
= self
.params
.get('playlistend')
1199 # For backwards compatibility, interpret -1 as whole list
1200 if playlistend
== -1:
1203 playlistitems_str
= self
.params
.get('playlist_items')
1204 playlistitems
= None
1205 if playlistitems_str
is not None:
1206 def iter_playlistitems(format
):
1207 for string_segment
in format
.split(','):
1208 if '-' in string_segment
:
1209 start
, end
= string_segment
.split('-')
1210 for item
in range(int(start
), int(end
) + 1):
1213 yield int(string_segment
)
1214 playlistitems
= orderedSet(iter_playlistitems(playlistitems_str
))
1216 ie_entries
= ie_result
['entries']
1218 def make_playlistitems_entries(list_ie_entries
):
1219 num_entries
= len(list_ie_entries
)
1220 for i
in playlistitems
:
1221 if -num_entries
< i
<= num_entries
:
1222 yield list_ie_entries
[i
- 1]
1223 elif incomplete_entries
:
1224 raise EntryNotInPlaylist()
1226 if isinstance(ie_entries
, list):
1227 n_all_entries
= len(ie_entries
)
1229 entries
= list(make_playlistitems_entries(ie_entries
))
1231 entries
= ie_entries
[playliststart
:playlistend
]
1232 n_entries
= len(entries
)
1233 msg
= 'Collected %d videos; downloading %d of them' % (n_all_entries
, n_entries
)
1234 elif isinstance(ie_entries
, PagedList
):
1237 for item
in playlistitems
:
1238 entries
.extend(ie_entries
.getslice(
1242 entries
= ie_entries
.getslice(
1243 playliststart
, playlistend
)
1244 n_entries
= len(entries
)
1245 msg
= 'Downloading %d videos' % n_entries
1248 entries
= list(make_playlistitems_entries(list(itertools
.islice(
1249 ie_entries
, 0, max(playlistitems
)))))
1251 entries
= list(itertools
.islice(
1252 ie_entries
, playliststart
, playlistend
))
1253 n_entries
= len(entries
)
1254 msg
= 'Downloading %d videos' % n_entries
1256 if any((entry
is None for entry
in entries
)):
1257 raise EntryNotInPlaylist()
1258 if not playlistitems
and (playliststart
or playlistend
):
1259 playlistitems
= list(range(1 + playliststart
, 1 + playliststart
+ len(entries
)))
1260 ie_result
['entries'] = entries
1261 ie_result
['requested_entries'] = playlistitems
1263 if self
.params
.get('allow_playlist_files', True):
1265 'playlist': playlist
,
1266 'playlist_id': ie_result
.get('id'),
1267 'playlist_title': ie_result
.get('title'),
1268 'playlist_uploader': ie_result
.get('uploader'),
1269 'playlist_uploader_id': ie_result
.get('uploader_id'),
1272 ie_copy
.update(dict(ie_result
))
1274 if self
.params
.get('writeinfojson', False):
1275 infofn
= self
.prepare_filename(ie_copy
, 'pl_infojson')
1276 if not self
._ensure
_dir
_exists
(encodeFilename(infofn
)):
1278 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(infofn
)):
1279 self
.to_screen('[info] Playlist metadata is already present')
1281 self
.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn
)
1283 write_json_file(self
.filter_requested_info(ie_result
, self
.params
.get('clean_infojson', True)), infofn
)
1284 except (OSError, IOError):
1285 self
.report_error('Cannot write playlist metadata to JSON file ' + infofn
)
1287 if self
.params
.get('writedescription', False):
1288 descfn
= self
.prepare_filename(ie_copy
, 'pl_description')
1289 if not self
._ensure
_dir
_exists
(encodeFilename(descfn
)):
1291 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(descfn
)):
1292 self
.to_screen('[info] Playlist description is already present')
1293 elif ie_result
.get('description') is None:
1294 self
.report_warning('There\'s no playlist description to write.')
1297 self
.to_screen('[info] Writing playlist description to: ' + descfn
)
1298 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
1299 descfile
.write(ie_result
['description'])
1300 except (OSError, IOError):
1301 self
.report_error('Cannot write playlist description file ' + descfn
)
1304 if self
.params
.get('playlistreverse', False):
1305 entries
= entries
[::-1]
1306 if self
.params
.get('playlistrandom', False):
1307 random
.shuffle(entries
)
1309 x_forwarded_for
= ie_result
.get('__x_forwarded_for_ip')
1311 self
.to_screen('[%s] playlist %s: %s' % (ie_result
['extractor'], playlist
, msg
))
1312 for i
, entry
in enumerate(entries
, 1):
1313 self
.to_screen('[download] Downloading video %s of %s' % (i
, n_entries
))
1314 # This __x_forwarded_for_ip thing is a bit ugly but requires
1317 entry
['__x_forwarded_for_ip'] = x_forwarded_for
1319 'n_entries': n_entries
,
1320 'playlist': playlist
,
1321 'playlist_id': ie_result
.get('id'),
1322 'playlist_title': ie_result
.get('title'),
1323 'playlist_uploader': ie_result
.get('uploader'),
1324 'playlist_uploader_id': ie_result
.get('uploader_id'),
1325 'playlist_index': playlistitems
[i
- 1] if playlistitems
else i
,
1326 'extractor': ie_result
['extractor'],
1327 'webpage_url': ie_result
['webpage_url'],
1328 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1329 'extractor_key': ie_result
['extractor_key'],
1332 if self
._match
_entry
(entry
, incomplete
=True) is not None:
1335 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, extra
)
1336 # TODO: skip failed (empty) entries?
1337 playlist_results
.append(entry_result
)
1338 ie_result
['entries'] = playlist_results
1339 self
.to_screen('[download] Finished downloading playlist: %s' % playlist
)
1342 @__handle_extraction_exceptions
1343 def __process_iterable_entry(self
, entry
, download
, extra_info
):
1344 return self
.process_ie_result(
1345 entry
, download
=download
, extra_info
=extra_info
)
1347 def _build_format_filter(self
, filter_spec
):
1348 " Returns a function to filter the formats according to the filter_spec "
1358 operator_rex
= re
.compile(r
'''(?x)\s*
1359 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
1360 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1361 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1363 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
1364 m
= operator_rex
.search(filter_spec
)
1367 comparison_value
= int(m
.group('value'))
1369 comparison_value
= parse_filesize(m
.group('value'))
1370 if comparison_value
is None:
1371 comparison_value
= parse_filesize(m
.group('value') + 'B')
1372 if comparison_value
is None:
1374 'Invalid value %r in format specification %r' % (
1375 m
.group('value'), filter_spec
))
1376 op
= OPERATORS
[m
.group('op')]
1381 '^=': lambda attr
, value
: attr
.startswith(value
),
1382 '$=': lambda attr
, value
: attr
.endswith(value
),
1383 '*=': lambda attr
, value
: value
in attr
,
1385 str_operator_rex
= re
.compile(r
'''(?x)
1386 \s*(?P<key>[a-zA-Z0-9._-]+)
1387 \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?
1388 \s*(?P<value>[a-zA-Z0-9._-]+)
1390 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
1391 m
= str_operator_rex
.search(filter_spec
)
1393 comparison_value
= m
.group('value')
1394 str_op
= STR_OPERATORS
[m
.group('op')]
1395 if m
.group('negation'):
1396 op
= lambda attr
, value
: not str_op(attr
, value
)
1401 raise ValueError('Invalid filter specification %r' % filter_spec
)
1404 actual_value
= f
.get(m
.group('key'))
1405 if actual_value
is None:
1406 return m
.group('none_inclusive')
1407 return op(actual_value
, comparison_value
)
1410 def _default_format_spec(self
, info_dict
, download
=True):
1413 merger
= FFmpegMergerPP(self
)
1414 return merger
.available
and merger
.can_merge()
1417 not self
.params
.get('simulate', False)
1421 or info_dict
.get('is_live', False)
1422 or self
.outtmpl_dict
['default'] == '-'))
1425 'best/bestvideo+bestaudio'
1427 else 'bestvideo*+bestaudio/best'
1428 if not self
.params
.get('allow_multiple_audio_streams', False)
1429 else 'bestvideo+bestaudio/best')
1431 def build_format_selector(self
, format_spec
):
1432 def syntax_error(note
, start
):
1434 'Invalid format specification: '
1435 '{0}\n\t{1}\n\t{2}^'.format(note
, format_spec
, ' ' * start
[1]))
1436 return SyntaxError(message
)
1438 PICKFIRST
= 'PICKFIRST'
1442 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1444 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
1445 'video': self
.params
.get('allow_multiple_video_streams', False)}
1447 def _parse_filter(tokens
):
1449 for type, string
, start
, _
, _
in tokens
:
1450 if type == tokenize
.OP
and string
== ']':
1451 return ''.join(filter_parts
)
1453 filter_parts
.append(string
)
1455 def _remove_unused_ops(tokens
):
1456 # Remove operators that we don't use and join them with the surrounding strings
1457 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1458 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
1459 last_string
, last_start
, last_end
, last_line
= None, None, None, None
1460 for type, string
, start
, end
, line
in tokens
:
1461 if type == tokenize
.OP
and string
== '[':
1463 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1465 yield type, string
, start
, end
, line
1466 # everything inside brackets will be handled by _parse_filter
1467 for type, string
, start
, end
, line
in tokens
:
1468 yield type, string
, start
, end
, line
1469 if type == tokenize
.OP
and string
== ']':
1471 elif type == tokenize
.OP
and string
in ALLOWED_OPS
:
1473 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1475 yield type, string
, start
, end
, line
1476 elif type in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
1478 last_string
= string
1482 last_string
+= string
1484 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1486 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
1488 current_selector
= None
1489 for type, string
, start
, _
, _
in tokens
:
1490 # ENCODING is only defined in python 3.x
1491 if type == getattr(tokenize
, 'ENCODING', None):
1493 elif type in [tokenize
.NAME
, tokenize
.NUMBER
]:
1494 current_selector
= FormatSelector(SINGLE
, string
, [])
1495 elif type == tokenize
.OP
:
1497 if not inside_group
:
1498 # ')' will be handled by the parentheses group
1499 tokens
.restore_last_token()
1501 elif inside_merge
and string
in ['/', ',']:
1502 tokens
.restore_last_token()
1504 elif inside_choice
and string
== ',':
1505 tokens
.restore_last_token()
1508 if not current_selector
:
1509 raise syntax_error('"," must follow a format selector', start
)
1510 selectors
.append(current_selector
)
1511 current_selector
= None
1513 if not current_selector
:
1514 raise syntax_error('"/" must follow a format selector', start
)
1515 first_choice
= current_selector
1516 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
1517 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
1519 if not current_selector
:
1520 current_selector
= FormatSelector(SINGLE
, 'best', [])
1521 format_filter
= _parse_filter(tokens
)
1522 current_selector
.filters
.append(format_filter
)
1524 if current_selector
:
1525 raise syntax_error('Unexpected "("', start
)
1526 group
= _parse_format_selection(tokens
, inside_group
=True)
1527 current_selector
= FormatSelector(GROUP
, group
, [])
1529 if not current_selector
:
1530 raise syntax_error('Unexpected "+"', start
)
1531 selector_1
= current_selector
1532 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
1534 raise syntax_error('Expected a selector', start
)
1535 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
1537 raise syntax_error('Operator not recognized: "{0}"'.format(string
), start
)
1538 elif type == tokenize
.ENDMARKER
:
1540 if current_selector
:
1541 selectors
.append(current_selector
)
1544 def _merge(formats_pair
):
1545 format_1
, format_2
= formats_pair
1548 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
1549 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
1551 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
1552 get_no_more
= {"video": False, "audio": False}
1553 for (i
, fmt_info
) in enumerate(formats_info
):
1554 for aud_vid
in ["audio", "video"]:
1555 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
1556 if get_no_more
[aud_vid
]:
1558 get_no_more
[aud_vid
] = True
1560 if len(formats_info
) == 1:
1561 return formats_info
[0]
1563 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
1564 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
1566 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
1567 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
1569 output_ext
= self
.params
.get('merge_output_format')
1572 output_ext
= the_only_video
['ext']
1573 elif the_only_audio
and not video_fmts
:
1574 output_ext
= the_only_audio
['ext']
1579 'requested_formats': formats_info
,
1580 'format': '+'.join(fmt_info
.get('format') for fmt_info
in formats_info
),
1581 'format_id': '+'.join(fmt_info
.get('format_id') for fmt_info
in formats_info
),
1587 'width': the_only_video
.get('width'),
1588 'height': the_only_video
.get('height'),
1589 'resolution': the_only_video
.get('resolution') or self
.format_resolution(the_only_video
),
1590 'fps': the_only_video
.get('fps'),
1591 'vcodec': the_only_video
.get('vcodec'),
1592 'vbr': the_only_video
.get('vbr'),
1593 'stretched_ratio': the_only_video
.get('stretched_ratio'),
1598 'acodec': the_only_audio
.get('acodec'),
1599 'abr': the_only_audio
.get('abr'),
1604 def _build_selector_function(selector
):
1605 if isinstance(selector
, list): # ,
1606 fs
= [_build_selector_function(s
) for s
in selector
]
1608 def selector_function(ctx
):
1610 for format
in f(ctx
):
1612 return selector_function
1614 elif selector
.type == GROUP
: # ()
1615 selector_function
= _build_selector_function(selector
.selector
)
1617 elif selector
.type == PICKFIRST
: # /
1618 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
1620 def selector_function(ctx
):
1622 picked_formats
= list(f(ctx
))
1624 return picked_formats
1627 elif selector
.type == SINGLE
: # atom
1628 format_spec
= (selector
.selector
if selector
.selector
is not None else 'best').lower()
1630 # TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
1631 if format_spec
== 'all':
1632 def selector_function(ctx
):
1633 formats
= list(ctx
['formats'])
1637 elif format_spec
== 'mergeall':
1638 def selector_function(ctx
):
1639 formats
= list(ctx
['formats'])
1640 merged_format
= formats
[0]
1641 for f
in formats
[1:]:
1642 merged_format
= _merge((merged_format
, f
))
1646 format_fallback
= False
1648 r
'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
1650 if mobj
is not None:
1651 format_idx
= int_or_none(mobj
.group('n'), default
=1)
1652 format_idx
= format_idx
- 1 if mobj
.group('bw')[0] == 'w' else -format_idx
1653 format_type
= (mobj
.group('type') or [None])[0]
1654 not_format_type
= {'v': 'a', 'a': 'v'}
.get(format_type
)
1655 format_modified
= mobj
.group('mod') is not None
1657 format_fallback
= not format_type
and not format_modified
# for b, w
1659 (lambda f
: f
.get('%scodec' % format_type
) != 'none')
1660 if format_type
and format_modified
# bv*, ba*, wv*, wa*
1661 else (lambda f
: f
.get('%scodec' % not_format_type
) == 'none')
1662 if format_type
# bv, ba, wv, wa
1663 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
1664 if not format_modified
# b, w
1668 filter_f
= ((lambda f
: f
.get('ext') == format_spec
)
1669 if format_spec
in ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav'] # extension
1670 else (lambda f
: f
.get('format_id') == format_spec
)) # id
1672 def selector_function(ctx
):
1673 formats
= list(ctx
['formats'])
1676 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
1678 if -n
<= format_idx
< n
:
1679 yield matches
[format_idx
]
1680 elif format_fallback
and ctx
['incomplete_formats']:
1681 # for extractors with incomplete formats (audio only (soundcloud)
1682 # or video only (imgur)) best/worst will fallback to
1683 # best/worst {video,audio}-only format
1685 if -n
<= format_idx
< n
:
1686 yield formats
[format_idx
]
1688 elif selector
.type == MERGE
: # +
1689 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
1691 def selector_function(ctx
):
1692 for pair
in itertools
.product(
1693 selector_1(copy
.deepcopy(ctx
)), selector_2(copy
.deepcopy(ctx
))):
1696 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
1698 def final_selector(ctx
):
1699 ctx_copy
= copy
.deepcopy(ctx
)
1700 for _filter
in filters
:
1701 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
1702 return selector_function(ctx_copy
)
1703 return final_selector
1705 stream
= io
.BytesIO(format_spec
.encode('utf-8'))
1707 tokens
= list(_remove_unused_ops(compat_tokenize_tokenize(stream
.readline
)))
1708 except tokenize
.TokenError
:
1709 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
1711 class TokenIterator(object):
1712 def __init__(self
, tokens
):
1713 self
.tokens
= tokens
1720 if self
.counter
>= len(self
.tokens
):
1721 raise StopIteration()
1722 value
= self
.tokens
[self
.counter
]
1728 def restore_last_token(self
):
1731 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
1732 return _build_selector_function(parsed_selector
)
1734 def _calc_headers(self
, info_dict
):
1735 res
= std_headers
.copy()
1737 add_headers
= info_dict
.get('http_headers')
1739 res
.update(add_headers
)
1741 cookies
= self
._calc
_cookies
(info_dict
)
1743 res
['Cookie'] = cookies
1745 if 'X-Forwarded-For' not in res
:
1746 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
1747 if x_forwarded_for_ip
:
1748 res
['X-Forwarded-For'] = x_forwarded_for_ip
1752 def _calc_cookies(self
, info_dict
):
1753 pr
= sanitized_Request(info_dict
['url'])
1754 self
.cookiejar
.add_cookie_header(pr
)
1755 return pr
.get_header('Cookie')
1757 def process_video_result(self
, info_dict
, download
=True):
1758 assert info_dict
.get('_type', 'video') == 'video'
1760 if 'id' not in info_dict
:
1761 raise ExtractorError('Missing "id" field in extractor result')
1762 if 'title' not in info_dict
:
1763 raise ExtractorError('Missing "title" field in extractor result')
1765 def report_force_conversion(field
, field_not
, conversion
):
1766 self
.report_warning(
1767 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1768 % (field
, field_not
, conversion
))
1770 def sanitize_string_field(info
, string_field
):
1771 field
= info
.get(string_field
)
1772 if field
is None or isinstance(field
, compat_str
):
1774 report_force_conversion(string_field
, 'a string', 'string')
1775 info
[string_field
] = compat_str(field
)
1777 def sanitize_numeric_fields(info
):
1778 for numeric_field
in self
._NUMERIC
_FIELDS
:
1779 field
= info
.get(numeric_field
)
1780 if field
is None or isinstance(field
, compat_numeric_types
):
1782 report_force_conversion(numeric_field
, 'numeric', 'int')
1783 info
[numeric_field
] = int_or_none(field
)
1785 sanitize_string_field(info_dict
, 'id')
1786 sanitize_numeric_fields(info_dict
)
1788 if 'playlist' not in info_dict
:
1789 # It isn't part of a playlist
1790 info_dict
['playlist'] = None
1791 info_dict
['playlist_index'] = None
1793 thumbnails
= info_dict
.get('thumbnails')
1794 if thumbnails
is None:
1795 thumbnail
= info_dict
.get('thumbnail')
1797 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
1799 thumbnails
.sort(key
=lambda t
: (
1800 t
.get('preference') if t
.get('preference') is not None else -1,
1801 t
.get('width') if t
.get('width') is not None else -1,
1802 t
.get('height') if t
.get('height') is not None else -1,
1803 t
.get('id') if t
.get('id') is not None else '', t
.get('url')))
1804 for i
, t
in enumerate(thumbnails
):
1805 t
['url'] = sanitize_url(t
['url'])
1806 if t
.get('width') and t
.get('height'):
1807 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
1808 if t
.get('id') is None:
1811 if self
.params
.get('list_thumbnails'):
1812 self
.list_thumbnails(info_dict
)
1815 thumbnail
= info_dict
.get('thumbnail')
1817 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
1819 info_dict
['thumbnail'] = thumbnails
[-1]['url']
1821 if 'display_id' not in info_dict
and 'id' in info_dict
:
1822 info_dict
['display_id'] = info_dict
['id']
1824 for ts_key
, date_key
in (
1825 ('timestamp', 'upload_date'),
1826 ('release_timestamp', 'release_date'),
1828 if info_dict
.get(date_key
) is None and info_dict
.get(ts_key
) is not None:
1829 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1830 # see http://bugs.python.org/issue1646728)
1832 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
[ts_key
])
1833 info_dict
[date_key
] = upload_date
.strftime('%Y%m%d')
1834 except (ValueError, OverflowError, OSError):
1837 # Auto generate title fields corresponding to the *_number fields when missing
1838 # in order to always have clean titles. This is very common for TV series.
1839 for field
in ('chapter', 'season', 'episode'):
1840 if info_dict
.get('%s_number' % field
) is not None and not info_dict
.get(field
):
1841 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
['%s_number' % field
])
1843 for cc_kind
in ('subtitles', 'automatic_captions'):
1844 cc
= info_dict
.get(cc_kind
)
1846 for _
, subtitle
in cc
.items():
1847 for subtitle_format
in subtitle
:
1848 if subtitle_format
.get('url'):
1849 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
1850 if subtitle_format
.get('ext') is None:
1851 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
1853 automatic_captions
= info_dict
.get('automatic_captions')
1854 subtitles
= info_dict
.get('subtitles')
1856 if self
.params
.get('listsubtitles', False):
1857 if 'automatic_captions' in info_dict
:
1858 self
.list_subtitles(
1859 info_dict
['id'], automatic_captions
, 'automatic captions')
1860 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
1863 info_dict
['requested_subtitles'] = self
.process_subtitles(
1864 info_dict
['id'], subtitles
, automatic_captions
)
1866 # We now pick which formats have to be downloaded
1867 if info_dict
.get('formats') is None:
1868 # There's only one format available
1869 formats
= [info_dict
]
1871 formats
= info_dict
['formats']
1874 raise ExtractorError('No video formats found!')
1876 def is_wellformed(f
):
1879 self
.report_warning(
1880 '"url" field is missing or empty - skipping format, '
1881 'there is an error in extractor')
1883 if isinstance(url
, bytes):
1884 sanitize_string_field(f
, 'url')
1887 # Filter out malformed formats for better extraction robustness
1888 formats
= list(filter(is_wellformed
, formats
))
1892 # We check that all the formats have the format and format_id fields
1893 for i
, format
in enumerate(formats
):
1894 sanitize_string_field(format
, 'format_id')
1895 sanitize_numeric_fields(format
)
1896 format
['url'] = sanitize_url(format
['url'])
1897 if not format
.get('format_id'):
1898 format
['format_id'] = compat_str(i
)
1900 # Sanitize format_id from characters used in format selector expression
1901 format
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', format
['format_id'])
1902 format_id
= format
['format_id']
1903 if format_id
not in formats_dict
:
1904 formats_dict
[format_id
] = []
1905 formats_dict
[format_id
].append(format
)
1907 # Make sure all formats have unique format_id
1908 for format_id
, ambiguous_formats
in formats_dict
.items():
1909 if len(ambiguous_formats
) > 1:
1910 for i
, format
in enumerate(ambiguous_formats
):
1911 format
['format_id'] = '%s-%d' % (format_id
, i
)
1913 for i
, format
in enumerate(formats
):
1914 if format
.get('format') is None:
1915 format
['format'] = '{id} - {res}{note}'.format(
1916 id=format
['format_id'],
1917 res
=self
.format_resolution(format
),
1918 note
=' ({0})'.format(format
['format_note']) if format
.get('format_note') is not None else '',
1920 # Automatically determine file extension if missing
1921 if format
.get('ext') is None:
1922 format
['ext'] = determine_ext(format
['url']).lower()
1923 # Automatically determine protocol if missing (useful for format
1924 # selection purposes)
1925 if format
.get('protocol') is None:
1926 format
['protocol'] = determine_protocol(format
)
1927 # Add HTTP headers, so that external programs can use them from the
1929 full_format_info
= info_dict
.copy()
1930 full_format_info
.update(format
)
1931 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
1932 # Remove private housekeeping stuff
1933 if '__x_forwarded_for_ip' in info_dict
:
1934 del info_dict
['__x_forwarded_for_ip']
1936 # TODO Central sorting goes here
1938 if formats
[0] is not info_dict
:
1939 # only set the 'formats' fields if the original info_dict list them
1940 # otherwise we end up with a circular reference, the first (and unique)
1941 # element in the 'formats' field in info_dict is info_dict itself,
1942 # which can't be exported to json
1943 info_dict
['formats'] = formats
1944 if self
.params
.get('listformats'):
1945 self
.list_formats(info_dict
)
1948 req_format
= self
.params
.get('format')
1949 if req_format
is None:
1950 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
1951 if self
.params
.get('verbose'):
1952 self
.to_screen('[debug] Default format spec: %s' % req_format
)
1954 format_selector
= self
.build_format_selector(req_format
)
1956 # While in format selection we may need to have an access to the original
1957 # format set in order to calculate some metrics or do some processing.
1958 # For now we need to be able to guess whether original formats provided
1959 # by extractor are incomplete or not (i.e. whether extractor provides only
1960 # video-only or audio-only formats) for proper formats selection for
1961 # extractors with such incomplete formats (see
1962 # https://github.com/ytdl-org/youtube-dl/pull/5556).
1963 # Since formats may be filtered during format selection and may not match
1964 # the original formats the results may be incorrect. Thus original formats
1965 # or pre-calculated metrics should be passed to format selection routines
1967 # We will pass a context object containing all necessary additional data
1968 # instead of just formats.
1969 # This fixes incorrect format selection issue (see
1970 # https://github.com/ytdl-org/youtube-dl/issues/10083).
1971 incomplete_formats
= (
1972 # All formats are video-only or
1973 all(f
.get('vcodec') != 'none' and f
.get('acodec') == 'none' for f
in formats
)
1974 # all formats are audio-only
1975 or all(f
.get('vcodec') == 'none' and f
.get('acodec') != 'none' for f
in formats
))
1979 'incomplete_formats': incomplete_formats
,
1982 formats_to_download
= list(format_selector(ctx
))
1983 if not formats_to_download
:
1984 raise ExtractorError('requested format not available',
1988 self
.to_screen('[info] Downloading format(s) %s' % ", ".join([f
['format_id'] for f
in formats_to_download
]))
1989 if len(formats_to_download
) > 1:
1990 self
.to_screen('[info] %s: downloading video in %s formats' % (info_dict
['id'], len(formats_to_download
)))
1991 for format
in formats_to_download
:
1992 new_info
= dict(info_dict
)
1993 new_info
.update(format
)
1994 self
.process_info(new_info
)
1995 # We update the info dict with the best quality format (backwards compatibility)
1996 info_dict
.update(formats_to_download
[-1])
1999 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
2000 """Select the requested subtitles and their format"""
2002 if normal_subtitles
and self
.params
.get('writesubtitles'):
2003 available_subs
.update(normal_subtitles
)
2004 if automatic_captions
and self
.params
.get('writeautomaticsub'):
2005 for lang
, cap_info
in automatic_captions
.items():
2006 if lang
not in available_subs
:
2007 available_subs
[lang
] = cap_info
2009 if (not self
.params
.get('writesubtitles') and not
2010 self
.params
.get('writeautomaticsub') or not
2014 if self
.params
.get('allsubtitles', False):
2015 requested_langs
= available_subs
.keys()
2017 if self
.params
.get('subtitleslangs', False):
2018 requested_langs
= self
.params
.get('subtitleslangs')
2019 elif 'en' in available_subs
:
2020 requested_langs
= ['en']
2022 requested_langs
= [list(available_subs
.keys())[0]]
2024 formats_query
= self
.params
.get('subtitlesformat', 'best')
2025 formats_preference
= formats_query
.split('/') if formats_query
else []
2027 for lang
in requested_langs
:
2028 formats
= available_subs
.get(lang
)
2030 self
.report_warning('%s subtitles not available for %s' % (lang
, video_id
))
2032 for ext
in formats_preference
:
2036 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
2042 self
.report_warning(
2043 'No subtitle format found matching "%s" for language %s, '
2044 'using %s' % (formats_query
, lang
, f
['ext']))
2048 def __forced_printings(self
, info_dict
, filename
, incomplete
):
2049 def print_mandatory(field
):
2050 if (self
.params
.get('force%s' % field
, False)
2051 and (not incomplete
or info_dict
.get(field
) is not None)):
2052 self
.to_stdout(info_dict
[field
])
2054 def print_optional(field
):
2055 if (self
.params
.get('force%s' % field
, False)
2056 and info_dict
.get(field
) is not None):
2057 self
.to_stdout(info_dict
[field
])
2059 print_mandatory('title')
2060 print_mandatory('id')
2061 if self
.params
.get('forceurl', False) and not incomplete
:
2062 if info_dict
.get('requested_formats') is not None:
2063 for f
in info_dict
['requested_formats']:
2064 self
.to_stdout(f
['url'] + f
.get('play_path', ''))
2066 # For RTMP URLs, also include the playpath
2067 self
.to_stdout(info_dict
['url'] + info_dict
.get('play_path', ''))
2068 print_optional('thumbnail')
2069 print_optional('description')
2070 if self
.params
.get('forcefilename', False) and filename
is not None:
2071 self
.to_stdout(filename
)
2072 if self
.params
.get('forceduration', False) and info_dict
.get('duration') is not None:
2073 self
.to_stdout(formatSeconds(info_dict
['duration']))
2074 print_mandatory('format')
2075 if self
.params
.get('forcejson', False):
2076 self
.post_extract(info_dict
)
2077 self
.to_stdout(json
.dumps(info_dict
, default
=repr))
2079 def process_info(self
, info_dict
):
2080 """Process a single resolved IE result."""
2082 assert info_dict
.get('_type', 'video') == 'video'
2084 info_dict
.setdefault('__postprocessors', [])
2086 max_downloads
= self
.params
.get('max_downloads')
2087 if max_downloads
is not None:
2088 if self
._num
_downloads
>= int(max_downloads
):
2089 raise MaxDownloadsReached()
2091 # TODO: backward compatibility, to be removed
2092 info_dict
['fulltitle'] = info_dict
['title']
2094 if 'format' not in info_dict
:
2095 info_dict
['format'] = info_dict
['ext']
2097 if self
._match
_entry
(info_dict
, incomplete
=False) is not None:
2100 self
.post_extract(info_dict
)
2101 self
._num
_downloads
+= 1
2103 info_dict
= self
.pre_process(info_dict
)
2105 # info_dict['_filename'] needs to be set for backward compatibility
2106 info_dict
['_filename'] = full_filename
= self
.prepare_filename(info_dict
, warn
=True)
2107 temp_filename
= self
.prepare_filename(info_dict
, 'temp')
2109 skip_dl
= self
.params
.get('skip_download', False)
2112 self
.__forced
_printings
(info_dict
, full_filename
, incomplete
=False)
2114 if self
.params
.get('simulate', False):
2115 if self
.params
.get('force_write_download_archive', False):
2116 self
.record_download_archive(info_dict
)
2118 # Do nothing else if in simulate mode
2121 if full_filename
is None:
2124 if not self
._ensure
_dir
_exists
(encodeFilename(full_filename
)):
2126 if not self
._ensure
_dir
_exists
(encodeFilename(temp_filename
)):
2129 if self
.params
.get('writedescription', False):
2130 descfn
= self
.prepare_filename(info_dict
, 'description')
2131 if not self
._ensure
_dir
_exists
(encodeFilename(descfn
)):
2133 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(descfn
)):
2134 self
.to_screen('[info] Video description is already present')
2135 elif info_dict
.get('description') is None:
2136 self
.report_warning('There\'s no description to write.')
2139 self
.to_screen('[info] Writing video description to: ' + descfn
)
2140 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
2141 descfile
.write(info_dict
['description'])
2142 except (OSError, IOError):
2143 self
.report_error('Cannot write description file ' + descfn
)
2146 if self
.params
.get('writeannotations', False):
2147 annofn
= self
.prepare_filename(info_dict
, 'annotation')
2148 if not self
._ensure
_dir
_exists
(encodeFilename(annofn
)):
2150 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(annofn
)):
2151 self
.to_screen('[info] Video annotations are already present')
2152 elif not info_dict
.get('annotations'):
2153 self
.report_warning('There are no annotations to write.')
2156 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
2157 with io
.open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
2158 annofile
.write(info_dict
['annotations'])
2159 except (KeyError, TypeError):
2160 self
.report_warning('There are no annotations to write.')
2161 except (OSError, IOError):
2162 self
.report_error('Cannot write annotations file: ' + annofn
)
2165 def dl(name
, info
, subtitle
=False):
2166 fd
= get_suitable_downloader(info
, self
.params
)(self
, self
.params
)
2167 for ph
in self
._progress
_hooks
:
2168 fd
.add_progress_hook(ph
)
2169 if self
.params
.get('verbose'):
2170 self
.to_screen('[debug] Invoking downloader on %r' % info
.get('url'))
2171 new_info
= dict(info
)
2172 if new_info
.get('http_headers') is None:
2173 new_info
['http_headers'] = self
._calc
_headers
(new_info
)
2174 return fd
.download(name
, new_info
, subtitle
)
2176 subtitles_are_requested
= any([self
.params
.get('writesubtitles', False),
2177 self
.params
.get('writeautomaticsub')])
2179 if subtitles_are_requested
and info_dict
.get('requested_subtitles'):
2180 # subtitles download errors are already managed as troubles in relevant IE
2181 # that way it will silently go on when used with unsupporting IE
2182 subtitles
= info_dict
['requested_subtitles']
2183 # ie = self.get_info_extractor(info_dict['extractor_key'])
2184 for sub_lang
, sub_info
in subtitles
.items():
2185 sub_format
= sub_info
['ext']
2186 sub_fn
= self
.prepare_filename(info_dict
, 'subtitle')
2187 sub_filename
= subtitles_filename(
2188 temp_filename
if not skip_dl
else sub_fn
,
2189 sub_lang
, sub_format
, info_dict
.get('ext'))
2190 sub_filename_final
= subtitles_filename(sub_fn
, sub_lang
, sub_format
, info_dict
.get('ext'))
2191 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(sub_filename
)):
2192 self
.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang
, sub_format
))
2193 sub_info
['filepath'] = sub_filename
2194 files_to_move
[sub_filename
] = sub_filename_final
2196 self
.to_screen('[info] Writing video subtitles to: ' + sub_filename
)
2197 if sub_info
.get('data') is not None:
2199 # Use newline='' to prevent conversion of newline characters
2200 # See https://github.com/ytdl-org/youtube-dl/issues/10268
2201 with io
.open(encodeFilename(sub_filename
), 'w', encoding
='utf-8', newline
='') as subfile
:
2202 subfile
.write(sub_info
['data'])
2203 sub_info
['filepath'] = sub_filename
2204 files_to_move
[sub_filename
] = sub_filename_final
2205 except (OSError, IOError):
2206 self
.report_error('Cannot write subtitles file ' + sub_filename
)
2210 dl(sub_filename
, sub_info
.copy(), subtitle
=True)
2211 sub_info
['filepath'] = sub_filename
2212 files_to_move
[sub_filename
] = sub_filename_final
2213 except (ExtractorError
, IOError, OSError, ValueError, compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
2214 self
.report_warning('Unable to download subtitle for "%s": %s' %
2215 (sub_lang
, error_to_compat_str(err
)))
2219 if self
.params
.get('convertsubtitles', False):
2220 # subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles'))
2221 filename_real_ext
= os
.path
.splitext(full_filename
)[1][1:]
2223 os
.path
.splitext(full_filename
)[0]
2224 if filename_real_ext
== info_dict
['ext']
2226 afilename
= '%s.%s' % (filename_wo_ext
, self
.params
.get('convertsubtitles'))
2227 # if subconv.available:
2228 # info_dict['__postprocessors'].append(subconv)
2229 if os
.path
.exists(encodeFilename(afilename
)):
2231 '[download] %s has already been downloaded and '
2232 'converted' % afilename
)
2235 self
.post_process(full_filename
, info_dict
, files_to_move
)
2236 except PostProcessingError
as err
:
2237 self
.report_error('Postprocessing: %s' % str(err
))
2240 if self
.params
.get('writeinfojson', False):
2241 infofn
= self
.prepare_filename(info_dict
, 'infojson')
2242 if not self
._ensure
_dir
_exists
(encodeFilename(infofn
)):
2244 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(infofn
)):
2245 self
.to_screen('[info] Video metadata is already present')
2247 self
.to_screen('[info] Writing video metadata as JSON to: ' + infofn
)
2249 write_json_file(self
.filter_requested_info(info_dict
, self
.params
.get('clean_infojson', True)), infofn
)
2250 except (OSError, IOError):
2251 self
.report_error('Cannot write video metadata to JSON file ' + infofn
)
2253 info_dict
['__infojson_filename'] = infofn
2255 thumbfn
= self
.prepare_filename(info_dict
, 'thumbnail')
2256 thumb_fn_temp
= temp_filename
if not skip_dl
else thumbfn
2257 for thumb_ext
in self
._write
_thumbnails
(info_dict
, thumb_fn_temp
):
2258 thumb_filename_temp
= replace_extension(thumb_fn_temp
, thumb_ext
, info_dict
.get('ext'))
2259 thumb_filename
= replace_extension(thumbfn
, thumb_ext
, info_dict
.get('ext'))
2260 files_to_move
[thumb_filename_temp
] = thumb_filename
2262 # Write internet shortcut files
2263 url_link
= webloc_link
= desktop_link
= False
2264 if self
.params
.get('writelink', False):
2265 if sys
.platform
== "darwin": # macOS.
2267 elif sys
.platform
.startswith("linux"):
2269 else: # if sys.platform in ['win32', 'cygwin']:
2271 if self
.params
.get('writeurllink', False):
2273 if self
.params
.get('writewebloclink', False):
2275 if self
.params
.get('writedesktoplink', False):
2278 if url_link
or webloc_link
or desktop_link
:
2279 if 'webpage_url' not in info_dict
:
2280 self
.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information')
2282 ascii_url
= iri_to_uri(info_dict
['webpage_url'])
2284 def _write_link_file(extension
, template
, newline
, embed_filename
):
2285 linkfn
= replace_extension(full_filename
, extension
, info_dict
.get('ext'))
2286 if self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(linkfn
)):
2287 self
.to_screen('[info] Internet shortcut is already present')
2290 self
.to_screen('[info] Writing internet shortcut to: ' + linkfn
)
2291 with io
.open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8', newline
=newline
) as linkfile
:
2292 template_vars
= {'url': ascii_url}
2294 template_vars
['filename'] = linkfn
[:-(len(extension
) + 1)]
2295 linkfile
.write(template
% template_vars
)
2296 except (OSError, IOError):
2297 self
.report_error('Cannot write internet shortcut ' + linkfn
)
2302 if not _write_link_file('url', DOT_URL_LINK_TEMPLATE
, '\r\n', embed_filename
=False):
2305 if not _write_link_file('webloc', DOT_WEBLOC_LINK_TEMPLATE
, '\n', embed_filename
=False):
2308 if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE
, '\n', embed_filename
=True):
2312 must_record_download_archive
= False
2316 def existing_file(*filepaths
):
2317 ext
= info_dict
.get('ext')
2318 final_ext
= self
.params
.get('final_ext', ext
)
2320 for file in orderedSet(filepaths
):
2321 if final_ext
!= ext
:
2322 converted
= replace_extension(file, final_ext
, ext
)
2323 if os
.path
.exists(encodeFilename(converted
)):
2324 existing_files
.append(converted
)
2325 if os
.path
.exists(encodeFilename(file)):
2326 existing_files
.append(file)
2328 if not existing_files
or self
.params
.get('overwrites', False):
2329 for file in orderedSet(existing_files
):
2330 self
.report_file_delete(file)
2331 os
.remove(encodeFilename(file))
2334 self
.report_file_already_downloaded(existing_files
[0])
2335 info_dict
['ext'] = os
.path
.splitext(existing_files
[0])[1][1:]
2336 return existing_files
[0]
2339 if info_dict
.get('requested_formats') is not None:
2341 merger
= FFmpegMergerPP(self
)
2342 if self
.params
.get('allow_unplayable_formats'):
2343 self
.report_warning(
2344 'You have requested merging of multiple formats '
2345 'while also allowing unplayable formats to be downloaded. '
2346 'The formats won\'t be merged to prevent data corruption.')
2347 elif not merger
.available
:
2348 self
.report_warning(
2349 'You have requested merging of multiple formats but ffmpeg is not installed. '
2350 'The formats won\'t be merged.')
2352 def compatible_formats(formats
):
2353 # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
2354 video_formats
= [format
for format
in formats
if format
.get('vcodec') != 'none']
2355 audio_formats
= [format
for format
in formats
if format
.get('acodec') != 'none']
2356 if len(video_formats
) > 2 or len(audio_formats
) > 2:
2360 exts
= set(format
.get('ext') for format
in formats
)
2362 set(('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma')),
2365 for ext_sets
in COMPATIBLE_EXTS
:
2366 if ext_sets
.issuperset(exts
):
2368 # TODO: Check acodec/vcodec
2371 requested_formats
= info_dict
['requested_formats']
2372 old_ext
= info_dict
['ext']
2373 if self
.params
.get('merge_output_format') is None:
2374 if not compatible_formats(requested_formats
):
2375 info_dict
['ext'] = 'mkv'
2376 self
.report_warning(
2377 'Requested formats are incompatible for merge and will be merged into mkv.')
2378 if (info_dict
['ext'] == 'webm'
2379 and self
.params
.get('writethumbnail', False)
2380 and info_dict
.get('thumbnails')):
2381 info_dict
['ext'] = 'mkv'
2382 self
.report_warning(
2383 'webm doesn\'t support embedding a thumbnail, mkv will be used.')
2385 def correct_ext(filename
):
2386 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
2388 os
.path
.splitext(filename
)[0]
2389 if filename_real_ext
== old_ext
2391 return '%s.%s' % (filename_wo_ext
, info_dict
['ext'])
2393 # Ensure filename always has a correct extension for successful merge
2394 full_filename
= correct_ext(full_filename
)
2395 temp_filename
= correct_ext(temp_filename
)
2396 dl_filename
= existing_file(full_filename
, temp_filename
)
2397 info_dict
['__real_download'] = False
2398 if dl_filename
is None:
2399 for f
in requested_formats
:
2400 new_info
= dict(info_dict
)
2402 fname
= prepend_extension(
2403 self
.prepare_filename(new_info
, 'temp'),
2404 'f%s' % f
['format_id'], new_info
['ext'])
2405 if not self
._ensure
_dir
_exists
(fname
):
2407 downloaded
.append(fname
)
2408 partial_success
, real_download
= dl(fname
, new_info
)
2409 info_dict
['__real_download'] = info_dict
['__real_download'] or real_download
2410 success
= success
and partial_success
2411 if merger
.available
and not self
.params
.get('allow_unplayable_formats'):
2412 info_dict
['__postprocessors'].append(merger
)
2413 info_dict
['__files_to_merge'] = downloaded
2414 # Even if there were no downloads, it is being merged only now
2415 info_dict
['__real_download'] = True
2417 for file in downloaded
:
2418 files_to_move
[file] = None
2420 # Just a single file
2421 dl_filename
= existing_file(full_filename
, temp_filename
)
2422 if dl_filename
is None:
2423 success
, real_download
= dl(temp_filename
, info_dict
)
2424 info_dict
['__real_download'] = real_download
2426 dl_filename
= dl_filename
or temp_filename
2427 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2429 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
2430 self
.report_error('unable to download video data: %s' % error_to_compat_str(err
))
2432 except (OSError, IOError) as err
:
2433 raise UnavailableVideoError(err
)
2434 except (ContentTooShortError
, ) as err
:
2435 self
.report_error('content too short (expected %s bytes and served %s)' % (err
.expected
, err
.downloaded
))
2438 if success
and full_filename
!= '-':
2440 fixup_policy
= self
.params
.get('fixup')
2441 if fixup_policy
is None:
2442 fixup_policy
= 'detect_or_warn'
2444 INSTALL_FFMPEG_MESSAGE
= 'Install ffmpeg to fix this automatically.'
2446 stretched_ratio
= info_dict
.get('stretched_ratio')
2447 if stretched_ratio
is not None and stretched_ratio
!= 1:
2448 if fixup_policy
== 'warn':
2449 self
.report_warning('%s: Non-uniform pixel ratio (%s)' % (
2450 info_dict
['id'], stretched_ratio
))
2451 elif fixup_policy
== 'detect_or_warn':
2452 stretched_pp
= FFmpegFixupStretchedPP(self
)
2453 if stretched_pp
.available
:
2454 info_dict
['__postprocessors'].append(stretched_pp
)
2456 self
.report_warning(
2457 '%s: Non-uniform pixel ratio (%s). %s'
2458 % (info_dict
['id'], stretched_ratio
, INSTALL_FFMPEG_MESSAGE
))
2460 assert fixup_policy
in ('ignore', 'never')
2462 if (info_dict
.get('requested_formats') is None
2463 and info_dict
.get('container') == 'm4a_dash'
2464 and info_dict
.get('ext') == 'm4a'):
2465 if fixup_policy
== 'warn':
2466 self
.report_warning(
2467 '%s: writing DASH m4a. '
2468 'Only some players support this container.'
2470 elif fixup_policy
== 'detect_or_warn':
2471 fixup_pp
= FFmpegFixupM4aPP(self
)
2472 if fixup_pp
.available
:
2473 info_dict
['__postprocessors'].append(fixup_pp
)
2475 self
.report_warning(
2476 '%s: writing DASH m4a. '
2477 'Only some players support this container. %s'
2478 % (info_dict
['id'], INSTALL_FFMPEG_MESSAGE
))
2480 assert fixup_policy
in ('ignore', 'never')
2482 if ('protocol' in info_dict
2483 and get_suitable_downloader(info_dict
, self
.params
).__name
__ == 'HlsFD'):
2484 if fixup_policy
== 'warn':
2485 self
.report_warning('%s: malformed AAC bitstream detected.' % (
2487 elif fixup_policy
== 'detect_or_warn':
2488 fixup_pp
= FFmpegFixupM3u8PP(self
)
2489 if fixup_pp
.available
:
2490 info_dict
['__postprocessors'].append(fixup_pp
)
2492 self
.report_warning(
2493 '%s: malformed AAC bitstream detected. %s'
2494 % (info_dict
['id'], INSTALL_FFMPEG_MESSAGE
))
2496 assert fixup_policy
in ('ignore', 'never')
2499 info_dict
= self
.post_process(dl_filename
, info_dict
, files_to_move
)
2500 except PostProcessingError
as err
:
2501 self
.report_error('Postprocessing: %s' % str(err
))
2504 for ph
in self
._post
_hooks
:
2505 ph(info_dict
['filepath'])
2506 except Exception as err
:
2507 self
.report_error('post hooks: %s' % str(err
))
2509 must_record_download_archive
= True
2511 if must_record_download_archive
or self
.params
.get('force_write_download_archive', False):
2512 self
.record_download_archive(info_dict
)
2513 max_downloads
= self
.params
.get('max_downloads')
2514 if max_downloads
is not None and self
._num
_downloads
>= int(max_downloads
):
2515 raise MaxDownloadsReached()
2517 def download(self
, url_list
):
2518 """Download a given list of URLs."""
2519 outtmpl
= self
.outtmpl_dict
['default']
2520 if (len(url_list
) > 1
2522 and '%' not in outtmpl
2523 and self
.params
.get('max_downloads') != 1):
2524 raise SameFileError(outtmpl
)
2526 for url
in url_list
:
2528 # It also downloads the videos
2529 res
= self
.extract_info(
2530 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
2531 except UnavailableVideoError
:
2532 self
.report_error('unable to download video')
2533 except MaxDownloadsReached
:
2534 self
.to_screen('[info] Maximum number of downloaded files reached')
2536 except ExistingVideoReached
:
2537 self
.to_screen('[info] Encountered a file that is already in the archive, stopping due to --break-on-existing')
2539 except RejectedVideoReached
:
2540 self
.to_screen('[info] Encountered a file that did not match filter, stopping due to --break-on-reject')
2543 if self
.params
.get('dump_single_json', False):
2544 self
.post_extract(res
)
2545 self
.to_stdout(json
.dumps(res
, default
=repr))
2547 return self
._download
_retcode
2549 def download_with_info_file(self
, info_filename
):
2550 with contextlib
.closing(fileinput
.FileInput(
2551 [info_filename
], mode
='r',
2552 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
2553 # FileInput doesn't have a read method, we can't call json.load
2554 info
= self
.filter_requested_info(json
.loads('\n'.join(f
)), self
.params
.get('clean_infojson', True))
2556 self
.process_ie_result(info
, download
=True)
2557 except (DownloadError
, EntryNotInPlaylist
):
2558 webpage_url
= info
.get('webpage_url')
2559 if webpage_url
is not None:
2560 self
.report_warning('The info failed to download, trying with "%s"' % webpage_url
)
2561 return self
.download([webpage_url
])
2564 return self
._download
_retcode
2567 def filter_requested_info(info_dict
, actually_filter
=True):
2568 if not actually_filter
:
2569 info_dict
['epoch'] = int(time
.time())
2572 'remove': ['requested_formats', 'requested_subtitles', 'requested_entries', 'filepath', 'entries'],
2575 keep_key
= lambda k
: k
in exceptions
['keep'] or not (k
.startswith('_') or k
in exceptions
['remove'])
2576 filter_fn
= lambda obj
: (
2577 list(map(filter_fn
, obj
)) if isinstance(obj
, (list, tuple))
2578 else obj
if not isinstance(obj
, dict)
2579 else dict((k
, filter_fn(v
)) for k
, v
in obj
.items() if keep_key(k
)))
2580 return filter_fn(info_dict
)
2582 def run_pp(self
, pp
, infodict
):
2583 files_to_delete
= []
2584 if '__files_to_move' not in infodict
:
2585 infodict
['__files_to_move'] = {}
2586 files_to_delete
, infodict
= pp
.run(infodict
)
2587 if not files_to_delete
:
2590 if self
.params
.get('keepvideo', False):
2591 for f
in files_to_delete
:
2592 infodict
['__files_to_move'].setdefault(f
, '')
2594 for old_filename
in set(files_to_delete
):
2595 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
2597 os
.remove(encodeFilename(old_filename
))
2598 except (IOError, OSError):
2599 self
.report_warning('Unable to remove downloaded original file')
2600 if old_filename
in infodict
['__files_to_move']:
2601 del infodict
['__files_to_move'][old_filename
]
2605 def post_extract(info_dict
):
2606 def actual_post_extract(info_dict
):
2607 if info_dict
.get('_type') in ('playlist', 'multi_video'):
2608 for video_dict
in info_dict
.get('entries', {}):
2609 actual_post_extract(video_dict
or {})
2612 if '__post_extractor' not in info_dict
:
2614 post_extractor
= info_dict
['__post_extractor']
2616 info_dict
.update(post_extractor().items())
2617 del info_dict
['__post_extractor']
2620 actual_post_extract(info_dict
or {})
2622 def pre_process(self
, ie_info
):
2623 info
= dict(ie_info
)
2624 for pp
in self
._pps
['beforedl']:
2625 info
= self
.run_pp(pp
, info
)
2628 def post_process(self
, filename
, ie_info
, files_to_move
=None):
2629 """Run all the postprocessors on the given file."""
2630 info
= dict(ie_info
)
2631 info
['filepath'] = filename
2632 info
['__files_to_move'] = files_to_move
or {}
2634 for pp
in ie_info
.get('__postprocessors', []) + self
._pps
['normal']:
2635 info
= self
.run_pp(pp
, info
)
2636 info
= self
.run_pp(MoveFilesAfterDownloadPP(self
), info
)
2637 del info
['__files_to_move']
2638 for pp
in self
._pps
['aftermove']:
2639 info
= self
.run_pp(pp
, info
)
2642 def _make_archive_id(self
, info_dict
):
2643 video_id
= info_dict
.get('id')
2646 # Future-proof against any change in case
2647 # and backwards compatibility with prior versions
2648 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
2649 if extractor
is None:
2650 url
= str_or_none(info_dict
.get('url'))
2653 # Try to find matching extractor for the URL and take its ie_key
2654 for ie
in self
._ies
:
2655 if ie
.suitable(url
):
2656 extractor
= ie
.ie_key()
2660 return '%s %s' % (extractor
.lower(), video_id
)
2662 def in_download_archive(self
, info_dict
):
2663 fn
= self
.params
.get('download_archive')
2667 vid_id
= self
._make
_archive
_id
(info_dict
)
2669 return False # Incomplete video information
2671 return vid_id
in self
.archive
2673 def record_download_archive(self
, info_dict
):
2674 fn
= self
.params
.get('download_archive')
2677 vid_id
= self
._make
_archive
_id
(info_dict
)
2679 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
2680 archive_file
.write(vid_id
+ '\n')
2681 self
.archive
.add(vid_id
)
2684 def format_resolution(format
, default
='unknown'):
2685 if format
.get('vcodec') == 'none':
2687 if format
.get('resolution') is not None:
2688 return format
['resolution']
2689 if format
.get('width') and format
.get('height'):
2690 res
= '%dx%d' % (format
['width'], format
['height'])
2691 elif format
.get('height'):
2692 res
= '%sp' % format
['height']
2693 elif format
.get('width'):
2694 res
= '%dx?' % format
['width']
2699 def _format_note(self
, fdict
):
2701 if fdict
.get('ext') in ['f4f', 'f4m']:
2702 res
+= '(unsupported) '
2703 if fdict
.get('language'):
2706 res
+= '[%s] ' % fdict
['language']
2707 if fdict
.get('format_note') is not None:
2708 res
+= fdict
['format_note'] + ' '
2709 if fdict
.get('tbr') is not None:
2710 res
+= '%4dk ' % fdict
['tbr']
2711 if fdict
.get('container') is not None:
2714 res
+= '%s container' % fdict
['container']
2715 if (fdict
.get('vcodec') is not None
2716 and fdict
.get('vcodec') != 'none'):
2719 res
+= fdict
['vcodec']
2720 if fdict
.get('vbr') is not None:
2722 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
2724 if fdict
.get('vbr') is not None:
2725 res
+= '%4dk' % fdict
['vbr']
2726 if fdict
.get('fps') is not None:
2729 res
+= '%sfps' % fdict
['fps']
2730 if fdict
.get('acodec') is not None:
2733 if fdict
['acodec'] == 'none':
2736 res
+= '%-5s' % fdict
['acodec']
2737 elif fdict
.get('abr') is not None:
2741 if fdict
.get('abr') is not None:
2742 res
+= '@%3dk' % fdict
['abr']
2743 if fdict
.get('asr') is not None:
2744 res
+= ' (%5dHz)' % fdict
['asr']
2745 if fdict
.get('filesize') is not None:
2748 res
+= format_bytes(fdict
['filesize'])
2749 elif fdict
.get('filesize_approx') is not None:
2752 res
+= '~' + format_bytes(fdict
['filesize_approx'])
2755 def _format_note_table(self
, f
):
2756 def join_fields(*vargs
):
2757 return ', '.join((val
for val
in vargs
if val
!= ''))
2760 'UNSUPPORTED' if f
.get('ext') in ('f4f', 'f4m') else '',
2761 format_field(f
, 'language', '[%s]'),
2762 format_field(f
, 'format_note'),
2763 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
2764 format_field(f
, 'asr', '%5dHz'))
2766 def list_formats(self
, info_dict
):
2767 formats
= info_dict
.get('formats', [info_dict
])
2768 new_format
= self
.params
.get('listformats_table', False)
2772 format_field(f
, 'format_id'),
2773 format_field(f
, 'ext'),
2774 self
.format_resolution(f
),
2775 format_field(f
, 'fps', '%d'),
2777 format_field(f
, 'filesize', ' %s', func
=format_bytes
) + format_field(f
, 'filesize_approx', '~%s', func
=format_bytes
),
2778 format_field(f
, 'tbr', '%4dk'),
2779 f
.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''),
2781 format_field(f
, 'vcodec', default
='unknown').replace('none', ''),
2782 format_field(f
, 'vbr', '%4dk'),
2783 format_field(f
, 'acodec', default
='unknown').replace('none', ''),
2784 format_field(f
, 'abr', '%3dk'),
2785 format_field(f
, 'asr', '%5dHz'),
2786 self
._format
_note
_table
(f
)]
2788 if f
.get('preference') is None or f
['preference'] >= -1000]
2789 header_line
= ['ID', 'EXT', 'RESOLUTION', 'FPS', '|', ' FILESIZE', ' TBR', 'PROTO',
2790 '|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'NOTE']
2794 format_field(f
, 'format_id'),
2795 format_field(f
, 'ext'),
2796 self
.format_resolution(f
),
2797 self
._format
_note
(f
)]
2799 if f
.get('preference') is None or f
['preference'] >= -1000]
2800 header_line
= ['format code', 'extension', 'resolution', 'note']
2803 '[info] Available formats for %s:\n%s' % (info_dict
['id'], render_table(
2807 extraGap
=(0 if new_format
else 1),
2808 hideEmpty
=new_format
)))
2810 def list_thumbnails(self
, info_dict
):
2811 thumbnails
= info_dict
.get('thumbnails')
2813 self
.to_screen('[info] No thumbnails present for %s' % info_dict
['id'])
2817 '[info] Thumbnails for %s:' % info_dict
['id'])
2818 self
.to_screen(render_table(
2819 ['ID', 'width', 'height', 'URL'],
2820 [[t
['id'], t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
]))
2822 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
2824 self
.to_screen('%s has no %s' % (video_id
, name
))
2827 'Available %s for %s:' % (name
, video_id
))
2828 self
.to_screen(render_table(
2829 ['Language', 'formats'],
2830 [[lang
, ', '.join(f
['ext'] for f
in reversed(formats
))]
2831 for lang
, formats
in subtitles
.items()]))
2833 def urlopen(self
, req
):
2834 """ Start an HTTP download """
2835 if isinstance(req
, compat_basestring
):
2836 req
= sanitized_Request(req
)
2837 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
2839 def print_debug_header(self
):
2840 if not self
.params
.get('verbose'):
2843 if type('') is not compat_str
:
2844 # Python 2.6 on SLES11 SP1 (https://github.com/ytdl-org/youtube-dl/issues/3326)
2845 self
.report_warning(
2846 'Your Python is broken! Update to a newer and supported version')
2848 stdout_encoding
= getattr(
2849 sys
.stdout
, 'encoding', 'missing (%s)' % type(sys
.stdout
).__name
__)
2851 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2852 locale
.getpreferredencoding(),
2853 sys
.getfilesystemencoding(),
2855 self
.get_encoding()))
2856 write_string(encoding_str
, encoding
=None)
2859 '(exe)' if hasattr(sys
, 'frozen')
2860 else '(zip)' if isinstance(globals().get('__loader__'), zipimporter
)
2861 else '(source)' if os
.path
.basename(sys
.argv
[0]) == '__main__.py'
2863 self
._write
_string
('[debug] yt-dlp version %s %s\n' % (__version__
, source
))
2865 self
._write
_string
('[debug] Lazy loading extractors enabled\n')
2868 '[debug] Plugin Extractors: %s\n' % [ie
.ie_key() for ie
in _PLUGIN_CLASSES
])
2870 sp
= subprocess
.Popen(
2871 ['git', 'rev-parse', '--short', 'HEAD'],
2872 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
2873 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
2874 out
, err
= process_communicate_or_kill(sp
)
2875 out
= out
.decode().strip()
2876 if re
.match('[0-9a-f]+', out
):
2877 self
._write
_string
('[debug] Git HEAD: %s\n' % out
)
2884 def python_implementation():
2885 impl_name
= platform
.python_implementation()
2886 if impl_name
== 'PyPy' and hasattr(sys
, 'pypy_version_info'):
2887 return impl_name
+ ' version %d.%d.%d' % sys
.pypy_version_info
[:3]
2890 self
._write
_string
('[debug] Python version %s (%s %s) - %s\n' % (
2891 platform
.python_version(),
2892 python_implementation(),
2893 platform
.architecture()[0],
2896 exe_versions
= FFmpegPostProcessor
.get_versions(self
)
2897 exe_versions
['rtmpdump'] = rtmpdump_version()
2898 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
2899 exe_str
= ', '.join(
2901 for exe
, v
in sorted(exe_versions
.items())
2906 self
._write
_string
('[debug] exe versions: %s\n' % exe_str
)
2909 for handler
in self
._opener
.handlers
:
2910 if hasattr(handler
, 'proxies'):
2911 proxy_map
.update(handler
.proxies
)
2912 self
._write
_string
('[debug] Proxy map: ' + compat_str(proxy_map
) + '\n')
2914 if self
.params
.get('call_home', False):
2915 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2916 self
._write
_string
('[debug] Public IP address: %s\n' % ipaddr
)
2918 latest_version
= self
.urlopen(
2919 'https://yt-dl.org/latest/version').read().decode('utf-8')
2920 if version_tuple(latest_version
) > version_tuple(__version__
):
2921 self
.report_warning(
2922 'You are using an outdated version (newest version: %s)! '
2923 'See https://yt-dl.org/update if you need help updating.' %
2926 def _setup_opener(self
):
2927 timeout_val
= self
.params
.get('socket_timeout')
2928 self
._socket
_timeout
= 600 if timeout_val
is None else float(timeout_val
)
2930 opts_cookiefile
= self
.params
.get('cookiefile')
2931 opts_proxy
= self
.params
.get('proxy')
2933 if opts_cookiefile
is None:
2934 self
.cookiejar
= compat_cookiejar
.CookieJar()
2936 opts_cookiefile
= expand_path(opts_cookiefile
)
2937 self
.cookiejar
= YoutubeDLCookieJar(opts_cookiefile
)
2938 if os
.access(opts_cookiefile
, os
.R_OK
):
2939 self
.cookiejar
.load(ignore_discard
=True, ignore_expires
=True)
2941 cookie_processor
= YoutubeDLCookieProcessor(self
.cookiejar
)
2942 if opts_proxy
is not None:
2943 if opts_proxy
== '':
2946 proxies
= {'http': opts_proxy, 'https': opts_proxy}
2948 proxies
= compat_urllib_request
.getproxies()
2949 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
2950 if 'http' in proxies
and 'https' not in proxies
:
2951 proxies
['https'] = proxies
['http']
2952 proxy_handler
= PerRequestProxyHandler(proxies
)
2954 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
2955 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
2956 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
2957 redirect_handler
= YoutubeDLRedirectHandler()
2958 data_handler
= compat_urllib_request_DataHandler()
2960 # When passing our own FileHandler instance, build_opener won't add the
2961 # default FileHandler and allows us to disable the file protocol, which
2962 # can be used for malicious purposes (see
2963 # https://github.com/ytdl-org/youtube-dl/issues/8227)
2964 file_handler
= compat_urllib_request
.FileHandler()
2966 def file_open(*args
, **kwargs
):
2967 raise compat_urllib_error
.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
2968 file_handler
.file_open
= file_open
2970 opener
= compat_urllib_request
.build_opener(
2971 proxy_handler
, https_handler
, cookie_processor
, ydlh
, redirect_handler
, data_handler
, file_handler
)
2973 # Delete the default user-agent header, which would otherwise apply in
2974 # cases where our custom HTTP handler doesn't come into play
2975 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
2976 opener
.addheaders
= []
2977 self
._opener
= opener
2979 def encode(self
, s
):
2980 if isinstance(s
, bytes):
2981 return s
# Already encoded
2984 return s
.encode(self
.get_encoding())
2985 except UnicodeEncodeError as err
:
2986 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
2989 def get_encoding(self
):
2990 encoding
= self
.params
.get('encoding')
2991 if encoding
is None:
2992 encoding
= preferredencoding()
2995 def _write_thumbnails(self
, info_dict
, filename
): # return the extensions
2996 write_all
= self
.params
.get('write_all_thumbnails', False)
2998 if write_all
or self
.params
.get('writethumbnail', False):
2999 thumbnails
= info_dict
.get('thumbnails') or []
3000 multiple
= write_all
and len(thumbnails
) > 1
3003 for t
in thumbnails
[::1 if write_all
else -1]:
3004 thumb_ext
= determine_ext(t
['url'], 'jpg')
3005 suffix
= '%s.' % t
['id'] if multiple
else ''
3006 thumb_display_id
= '%s ' % t
['id'] if multiple
else ''
3007 t
['filepath'] = thumb_filename
= replace_extension(filename
, suffix
+ thumb_ext
, info_dict
.get('ext'))
3009 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(thumb_filename
)):
3010 ret
.append(suffix
+ thumb_ext
)
3011 self
.to_screen('[%s] %s: Thumbnail %sis already present' %
3012 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
3014 self
.to_screen('[%s] %s: Downloading thumbnail %s ...' %
3015 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
3017 uf
= self
.urlopen(t
['url'])
3018 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
3019 shutil
.copyfileobj(uf
, thumbf
)
3020 ret
.append(suffix
+ thumb_ext
)
3021 self
.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
3022 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
, thumb_filename
))
3023 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
3024 self
.report_warning('Unable to download thumbnail "%s": %s' %
3025 (t
['url'], error_to_compat_str(err
)))
3026 if ret
and not write_all
: