4 from __future__
import absolute_import
, unicode_literals
32 from string
import ascii_letters
36 compat_get_terminal_size
,
43 compat_tokenize_tokenize
,
45 compat_urllib_request
,
46 compat_urllib_request_DataHandler
,
47 windows_enable_vt_mode
,
49 from .cookies
import load_cookies
89 PerRequestProxyHandler
,
95 register_socks_protocols
,
110 supports_terminal_sequences
,
115 UnavailableVideoError
,
121 YoutubeDLCookieProcessor
,
123 YoutubeDLRedirectHandler
,
125 from .cache
import Cache
126 from .minicurses
import format_text
127 from .extractor
import (
128 gen_extractor_classes
,
131 _PLUGIN_CLASSES
as plugin_extractors
133 from .extractor
.openload
import PhantomJSwrapper
134 from .downloader
import (
136 get_suitable_downloader
,
137 shorten_protocol_name
139 from .downloader
.rtmp
import rtmpdump_version
140 from .postprocessor
import (
143 FFmpegFixupDurationPP
,
146 FFmpegFixupStretchedPP
,
147 FFmpegFixupTimestampPP
,
150 MoveFilesAfterDownloadPP
,
151 _PLUGIN_CLASSES
as plugin_postprocessors
153 from .update
import detect_variant
154 from .version
import __version__
156 if compat_os_name
== 'nt':
160 class YoutubeDL(object):
163 YoutubeDL objects are the ones responsible of downloading the
164 actual video file and writing it to disk if the user has requested
165 it, among some other tasks. In most cases there should be one per
166 program. As, given a video URL, the downloader doesn't know how to
167 extract all the needed information, task that InfoExtractors do, it
168 has to pass the URL to one of them.
170 For this, YoutubeDL objects have a method that allows
171 InfoExtractors to be registered in a given order. When it is passed
172 a URL, the YoutubeDL object handles it to the first InfoExtractor it
173 finds that reports being able to handle it. The InfoExtractor extracts
174 all the information about the video or videos the URL refers to, and
175 YoutubeDL process the extracted information, possibly using a File
176 Downloader to download the video.
178 YoutubeDL objects accept a lot of parameters. In order not to saturate
179 the object constructor with arguments, it receives a dictionary of
180 options instead. These options are available through the params
181 attribute for the InfoExtractors to use. The YoutubeDL also
182 registers itself as the downloader in charge for the InfoExtractors
183 that are added to it, so this is a "mutual registration".
187 username: Username for authentication purposes.
188 password: Password for authentication purposes.
189 videopassword: Password for accessing a video.
190 ap_mso: Adobe Pass multiple-system operator identifier.
191 ap_username: Multiple-system operator account username.
192 ap_password: Multiple-system operator account password.
193 usenetrc: Use netrc for authentication instead.
194 verbose: Print additional info to stdout.
195 quiet: Do not print messages to stdout.
196 no_warnings: Do not print out anything for warnings.
197 forceprint: A list of templates to force print
198 forceurl: Force printing final URL. (Deprecated)
199 forcetitle: Force printing title. (Deprecated)
200 forceid: Force printing ID. (Deprecated)
201 forcethumbnail: Force printing thumbnail URL. (Deprecated)
202 forcedescription: Force printing description. (Deprecated)
203 forcefilename: Force printing final filename. (Deprecated)
204 forceduration: Force printing duration. (Deprecated)
205 forcejson: Force printing info_dict as JSON.
206 dump_single_json: Force printing the info_dict of the whole playlist
207 (or video) as a single JSON line.
208 force_write_download_archive: Force writing download archive regardless
209 of 'skip_download' or 'simulate'.
210 simulate: Do not download the video files. If unset (or None),
211 simulate only if listsubtitles, listformats or list_thumbnails is used
212 format: Video format code. see "FORMAT SELECTION" for more details.
213 allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
214 ignore_no_formats_error: Ignore "No video formats" error. Usefull for
215 extracting metadata even if the video is not actually
216 available for download (experimental)
217 format_sort: A list of fields by which to sort the video formats.
218 See "Sorting Formats" for more details.
219 format_sort_force: Force the given format_sort. see "Sorting Formats"
221 allow_multiple_video_streams: Allow multiple video streams to be merged
223 allow_multiple_audio_streams: Allow multiple audio streams to be merged
225 check_formats Whether to test if the formats are downloadable.
226 Can be True (check all), False (check none),
227 'selected' (check selected formats),
228 or None (check only if requested by extractor)
229 paths: Dictionary of output paths. The allowed keys are 'home'
230 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
231 outtmpl: Dictionary of templates for output names. Allowed keys
232 are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
233 For compatibility with youtube-dl, a single string can also be used
234 outtmpl_na_placeholder: Placeholder for unavailable meta fields.
235 restrictfilenames: Do not allow "&" and spaces in file names
236 trim_file_name: Limit length of filename (extension excluded)
237 windowsfilenames: Force the filenames to be windows compatible
238 ignoreerrors: Do not stop on download/postprocessing errors.
239 Can be 'only_download' to ignore only download errors.
240 Default is 'only_download' for CLI, but False for API
241 skip_playlist_after_errors: Number of allowed failures until the rest of
242 the playlist is skipped
243 force_generic_extractor: Force downloader to use the generic extractor
244 overwrites: Overwrite all video and metadata files if True,
245 overwrite only non-video files if None
246 and don't overwrite any file if False
247 For compatibility with youtube-dl,
248 "nooverwrites" may also be used instead
249 playliststart: Playlist item to start at.
250 playlistend: Playlist item to end at.
251 playlist_items: Specific indices of playlist to download.
252 playlistreverse: Download playlist items in reverse order.
253 playlistrandom: Download playlist items in random order.
254 matchtitle: Download only matching titles.
255 rejecttitle: Reject downloads for matching titles.
256 logger: Log messages to a logging.Logger instance.
257 logtostderr: Log messages to stderr instead of stdout.
258 consoletitle: Display progress in console window's titlebar.
259 writedescription: Write the video description to a .description file
260 writeinfojson: Write the video description to a .info.json file
261 clean_infojson: Remove private fields from the infojson
262 getcomments: Extract video comments. This will not be written to disk
263 unless writeinfojson is also given
264 writeannotations: Write the video annotations to a .annotations.xml file
265 writethumbnail: Write the thumbnail image to a file
266 allow_playlist_files: Whether to write playlists' description, infojson etc
267 also to disk when using the 'write*' options
268 write_all_thumbnails: Write all thumbnail formats to files
269 writelink: Write an internet shortcut file, depending on the
270 current platform (.url/.webloc/.desktop)
271 writeurllink: Write a Windows internet shortcut file (.url)
272 writewebloclink: Write a macOS internet shortcut file (.webloc)
273 writedesktoplink: Write a Linux internet shortcut file (.desktop)
274 writesubtitles: Write the video subtitles to a file
275 writeautomaticsub: Write the automatically generated subtitles to a file
276 allsubtitles: Deprecated - Use subtitleslangs = ['all']
277 Downloads all the subtitles of the video
278 (requires writesubtitles or writeautomaticsub)
279 listsubtitles: Lists all available subtitles for the video
280 subtitlesformat: The format code for subtitles
281 subtitleslangs: List of languages of the subtitles to download (can be regex).
282 The list may contain "all" to refer to all the available
283 subtitles. The language can be prefixed with a "-" to
284 exclude it from the requested languages. Eg: ['all', '-live_chat']
285 keepvideo: Keep the video file after post-processing
286 daterange: A DateRange object, download only if the upload_date is in the range.
287 skip_download: Skip the actual download of the video file
288 cachedir: Location of the cache files in the filesystem.
289 False to disable filesystem cache.
290 noplaylist: Download single video instead of a playlist if in doubt.
291 age_limit: An integer representing the user's age in years.
292 Unsuitable videos for the given age are skipped.
293 min_views: An integer representing the minimum view count the video
294 must have in order to not be skipped.
295 Videos without view count information are always
296 downloaded. None for no limit.
297 max_views: An integer representing the maximum view count.
298 Videos that are more popular than that are not
300 Videos without view count information are always
301 downloaded. None for no limit.
302 download_archive: File name of a file where all downloads are recorded.
303 Videos already present in the file are not downloaded
305 break_on_existing: Stop the download process after attempting to download a
306 file that is in the archive.
307 break_on_reject: Stop the download process when encountering a video that
308 has been filtered out.
309 cookiefile: File name where cookies should be read from and dumped to
310 cookiesfrombrowser: A tuple containing the name of the browser and the profile
311 name/path from where cookies are loaded.
312 Eg: ('chrome', ) or ('vivaldi', 'default')
313 nocheckcertificate:Do not verify SSL certificates
314 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
315 At the moment, this is only supported by YouTube.
316 proxy: URL of the proxy server to use
317 geo_verification_proxy: URL of the proxy to use for IP address verification
318 on geo-restricted sites.
319 socket_timeout: Time to wait for unresponsive hosts, in seconds
320 bidi_workaround: Work around buggy terminals without bidirectional text
321 support, using fridibi
322 debug_printtraffic:Print out sent and received HTTP traffic
323 include_ads: Download ads as well
324 default_search: Prepend this string if an input url is not valid.
325 'auto' for elaborate guessing
326 encoding: Use this encoding instead of the system-specified.
327 extract_flat: Do not resolve URLs, return the immediate result.
328 Pass in 'in_playlist' to only show this behavior for
330 postprocessors: A list of dictionaries, each with an entry
331 * key: The name of the postprocessor. See
332 yt_dlp/postprocessor/__init__.py for a list.
333 * when: When to run the postprocessor. Can be one of
334 pre_process|before_dl|post_process|after_move.
335 Assumed to be 'post_process' if not given
336 post_hooks: Deprecated - Register a custom postprocessor instead
337 A list of functions that get called as the final step
338 for each video file, after all postprocessors have been
339 called. The filename will be passed as the only argument.
340 progress_hooks: A list of functions that get called on download
341 progress, with a dictionary with the entries
342 * status: One of "downloading", "error", or "finished".
343 Check this first and ignore unknown values.
344 * info_dict: The extracted info_dict
346 If status is one of "downloading", or "finished", the
347 following properties may also be present:
348 * filename: The final filename (always present)
349 * tmpfilename: The filename we're currently writing to
350 * downloaded_bytes: Bytes on disk
351 * total_bytes: Size of the whole file, None if unknown
352 * total_bytes_estimate: Guess of the eventual file size,
354 * elapsed: The number of seconds since download started.
355 * eta: The estimated time in seconds, None if unknown
356 * speed: The download speed in bytes/second, None if
358 * fragment_index: The counter of the currently
359 downloaded video fragment.
360 * fragment_count: The number of fragments (= individual
361 files that will be merged)
363 Progress hooks are guaranteed to be called at least once
364 (with status "finished") if the download is successful.
365 postprocessor_hooks: A list of functions that get called on postprocessing
366 progress, with a dictionary with the entries
367 * status: One of "started", "processing", or "finished".
368 Check this first and ignore unknown values.
369 * postprocessor: Name of the postprocessor
370 * info_dict: The extracted info_dict
372 Progress hooks are guaranteed to be called at least twice
373 (with status "started" and "finished") if the processing is successful.
374 merge_output_format: Extension to use when merging formats.
375 final_ext: Expected final extension; used to detect when the file was
376 already downloaded and converted. "merge_output_format" is
377 replaced by this extension when given
378 fixup: Automatically correct known faults of the file.
380 - "never": do nothing
381 - "warn": only emit a warning
382 - "detect_or_warn": check whether we can do anything
383 about it, warn otherwise (default)
384 source_address: Client-side IP address to bind to.
385 call_home: Boolean, true iff we are allowed to contact the
386 yt-dlp servers for debugging. (BROKEN)
387 sleep_interval_requests: Number of seconds to sleep between requests
389 sleep_interval: Number of seconds to sleep before each download when
390 used alone or a lower bound of a range for randomized
391 sleep before each download (minimum possible number
392 of seconds to sleep) when used along with
394 max_sleep_interval:Upper bound of a range for randomized sleep before each
395 download (maximum possible number of seconds to sleep).
396 Must only be used along with sleep_interval.
397 Actual sleep time will be a random float from range
398 [sleep_interval; max_sleep_interval].
399 sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
400 listformats: Print an overview of available video formats and exit.
401 list_thumbnails: Print a table of all thumbnails and exit.
402 match_filter: A function that gets called with the info_dict of
404 If it returns a message, the video is ignored.
405 If it returns None, the video is downloaded.
406 match_filter_func in utils.py is one example for this.
407 no_color: Do not emit color codes in output.
408 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
411 Two-letter ISO 3166-2 country code that will be used for
412 explicit geographic restriction bypassing via faking
413 X-Forwarded-For HTTP header
415 IP range in CIDR notation that will be used similarly to
418 The following options determine which downloader is picked:
419 external_downloader: A dictionary of protocol keys and the executable of the
420 external downloader to use for it. The allowed protocols
421 are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
422 Set the value to 'native' to use the native downloader
423 hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
424 or {'m3u8': 'ffmpeg'} instead.
425 Use the native HLS downloader instead of ffmpeg/avconv
426 if True, otherwise use ffmpeg/avconv if False, otherwise
427 use downloader suggested by extractor if None.
428 compat_opts: Compatibility options. See "Differences in default behavior".
429 The following options do not work when used through the API:
430 filename, abort-on-error, multistreams, no-live-chat, format-sort
431 no-clean-infojson, no-playlist-metafiles, no-keep-subs.
432 Refer __init__.py for their implementation
433 progress_template: Dictionary of templates for progress outputs.
434 Allowed keys are 'download', 'postprocess',
435 'download-title' (console title) and 'postprocess-title'.
436 The template is mapped on a dictionary with keys 'progress' and 'info'
438 The following parameters are not used by YoutubeDL itself, they are used by
439 the downloader (see yt_dlp/downloader/common.py):
440 nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
441 max_filesize, test, noresizebuffer, retries, fragment_retries, continuedl,
442 noprogress, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
443 external_downloader_args.
445 The following options are used by the post processors:
446 prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
447 otherwise prefer ffmpeg. (avconv support is deprecated)
448 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
449 to the binary or its containing directory.
450 postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
451 and a list of additional command-line arguments for the
452 postprocessor/executable. The dict can also have "PP+EXE" keys
453 which are used when the given exe is used by the given PP.
454 Use 'default' as the name for arguments to passed to all PP
455 For compatibility with youtube-dl, a single list of args
458 The following options are used by the extractors:
459 extractor_retries: Number of times to retry for known errors
460 dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
461 hls_split_discontinuity: Split HLS playlists to different formats at
462 discontinuities such as ad breaks (default: False)
463 extractor_args: A dictionary of arguments to be passed to the extractors.
464 See "EXTRACTOR ARGUMENTS" for details.
465 Eg: {'youtube': {'skip': ['dash', 'hls']}}
466 youtube_include_dash_manifest: Deprecated - Use extractor_args instead.
467 If True (default), DASH manifests and related
468 data will be downloaded and processed by extractor.
469 You can reduce network I/O by disabling it if you don't
470 care about DASH. (only for youtube)
471 youtube_include_hls_manifest: Deprecated - Use extractor_args instead.
472 If True (default), HLS manifests and related
473 data will be downloaded and processed by extractor.
474 You can reduce network I/O by disabling it if you don't
475 care about HLS. (only for youtube)
478 _NUMERIC_FIELDS
= set((
479 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
480 'timestamp', 'release_timestamp',
481 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
482 'average_rating', 'comment_count', 'age_limit',
483 'start_time', 'end_time',
484 'chapter_number', 'season_number', 'episode_number',
485 'track_number', 'disc_number', 'release_year',
488 _format_selection_exts
= {
489 'audio': {'m4a', 'mp3', 'ogg', 'aac'}
,
490 'video': {'mp4', 'flv', 'webm', '3gp'}
,
491 'storyboards': {'mhtml'}
,
496 _pps
= {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
497 _printed_messages
= set()
498 _first_webpage_request
= True
499 _download_retcode
= None
500 _num_downloads
= None
502 _playlist_urls
= set()
505 def __init__(self
, params
=None, auto_init
=True):
506 """Create a FileDownloader object with the given options.
507 @param auto_init Whether to load the default extractors and print header (if verbose).
508 Set to 'no_verbose_header' to not print the header
513 self
._ies
_instances
= {}
514 self
._pps
= {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
515 self
._printed
_messages
= set()
516 self
._first
_webpage
_request
= True
517 self
._post
_hooks
= []
518 self
._progress
_hooks
= []
519 self
._postprocessor
_hooks
= []
520 self
._download
_retcode
= 0
521 self
._num
_downloads
= 0
522 self
._screen
_file
= [sys
.stdout
, sys
.stderr
][params
.get('logtostderr', False)]
523 self
._err
_file
= sys
.stderr
525 self
.cache
= Cache(self
)
527 windows_enable_vt_mode()
528 # FIXME: This will break if we ever print color to stdout
529 self
._allow
_colors
= {
530 'screen': not self
.params
.get('no_color') and supports_terminal_sequences(self
._screen
_file
),
531 'err': not self
.params
.get('no_color') and supports_terminal_sequences(self
._err
_file
),
534 if sys
.version_info
< (3, 6):
536 'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys
.version_info
[:2])
538 if self
.params
.get('allow_unplayable_formats'):
540 f
'You have asked for {self._format_err("UNPLAYABLE", self.Styles.EMPHASIS)} formats to be listed/downloaded. '
541 'This is a developer option intended for debugging. \n'
542 ' If you experience any issues while using this option, '
543 f
'{self._format_err("DO NOT", self.Styles.ERROR)} open a bug report')
545 def check_deprecated(param
, option
, suggestion
):
546 if self
.params
.get(param
) is not None:
547 self
.report_warning('%s is deprecated. Use %s instead' % (option
, suggestion
))
551 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
552 if self
.params
.get('geo_verification_proxy') is None:
553 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
555 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
556 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
557 check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
559 for msg
in self
.params
.get('_warnings', []):
560 self
.report_warning(msg
)
562 if 'list-formats' in self
.params
.get('compat_opts', []):
563 self
.params
['listformats_table'] = False
565 if 'overwrites' not in self
.params
and self
.params
.get('nooverwrites') is not None:
566 # nooverwrites was unnecessarily changed to overwrites
567 # in 0c3d0f51778b153f65c21906031c2e091fcfb641
568 # This ensures compatibility with both keys
569 self
.params
['overwrites'] = not self
.params
['nooverwrites']
570 elif self
.params
.get('overwrites') is None:
571 self
.params
.pop('overwrites', None)
573 self
.params
['nooverwrites'] = not self
.params
['overwrites']
575 if params
.get('bidi_workaround', False):
578 master
, slave
= pty
.openpty()
579 width
= compat_get_terminal_size().columns
583 width_args
= ['-w', str(width
)]
585 stdin
=subprocess
.PIPE
,
587 stderr
=self
._err
_file
)
589 self
._output
_process
= Popen(['bidiv'] + width_args
, **sp_kwargs
)
591 self
._output
_process
= Popen(['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
592 self
._output
_channel
= os
.fdopen(master
, 'rb')
593 except OSError as ose
:
594 if ose
.errno
== errno
.ENOENT
:
596 'Could not find fribidi executable, ignoring --bidi-workaround. '
597 'Make sure that fribidi is an executable file in one of the directories in your $PATH.')
601 if (sys
.platform
!= 'win32'
602 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
603 and not params
.get('restrictfilenames', False)):
604 # Unicode filesystem API will throw errors (#1474, #13027)
606 'Assuming --restrict-filenames since file system encoding '
607 'cannot encode all characters. '
608 'Set the LC_ALL environment variable to fix this.')
609 self
.params
['restrictfilenames'] = True
611 self
.outtmpl_dict
= self
.parse_outtmpl()
613 # Creating format selector here allows us to catch syntax errors before the extraction
614 self
.format_selector
= (
615 None if self
.params
.get('format') is None
616 else self
.build_format_selector(self
.params
['format']))
621 if auto_init
!= 'no_verbose_header':
622 self
.print_debug_header()
623 self
.add_default_info_extractors()
625 for pp_def_raw
in self
.params
.get('postprocessors', []):
626 pp_def
= dict(pp_def_raw
)
627 when
= pp_def
.pop('when', 'post_process')
628 pp_class
= get_postprocessor(pp_def
.pop('key'))
629 pp
= pp_class(self
, **compat_kwargs(pp_def
))
630 self
.add_post_processor(pp
, when
=when
)
632 for ph
in self
.params
.get('post_hooks', []):
633 self
.add_post_hook(ph
)
635 for ph
in self
.params
.get('progress_hooks', []):
636 self
.add_progress_hook(ph
)
638 register_socks_protocols()
640 def preload_download_archive(fn
):
641 """Preload the archive, if any is specified"""
644 self
.write_debug(f
'Loading archive file {fn!r}')
646 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
647 for line
in archive_file
:
648 self
.archive
.add(line
.strip())
649 except IOError as ioe
:
650 if ioe
.errno
!= errno
.ENOENT
:
656 preload_download_archive(self
.params
.get('download_archive'))
658 def warn_if_short_id(self
, argv
):
659 # short YouTube ID starting with dash?
661 i
for i
, a
in enumerate(argv
)
662 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
666 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
667 + ['--'] + [argv
[i
] for i
in idxs
]
670 'Long argument string detected. '
671 'Use -- to separate parameters and URLs, like this:\n%s' %
672 args_to_str(correct_argv
))
674 def add_info_extractor(self
, ie
):
675 """Add an InfoExtractor object to the end of the list."""
677 self
._ies
[ie_key
] = ie
678 if not isinstance(ie
, type):
679 self
._ies
_instances
[ie_key
] = ie
680 ie
.set_downloader(self
)
682 def _get_info_extractor_class(self
, ie_key
):
683 ie
= self
._ies
.get(ie_key
)
685 ie
= get_info_extractor(ie_key
)
686 self
.add_info_extractor(ie
)
689 def get_info_extractor(self
, ie_key
):
691 Get an instance of an IE with name ie_key, it will try to get one from
692 the _ies list, if there's no instance it will create a new one and add
693 it to the extractor list.
695 ie
= self
._ies
_instances
.get(ie_key
)
697 ie
= get_info_extractor(ie_key
)()
698 self
.add_info_extractor(ie
)
701 def add_default_info_extractors(self
):
703 Add the InfoExtractors returned by gen_extractors to the end of the list
705 for ie
in gen_extractor_classes():
706 self
.add_info_extractor(ie
)
708 def add_post_processor(self
, pp
, when
='post_process'):
709 """Add a PostProcessor object to the end of the chain."""
710 self
._pps
[when
].append(pp
)
711 pp
.set_downloader(self
)
713 def add_post_hook(self
, ph
):
714 """Add the post hook"""
715 self
._post
_hooks
.append(ph
)
717 def add_progress_hook(self
, ph
):
718 """Add the download progress hook"""
719 self
._progress
_hooks
.append(ph
)
721 def add_postprocessor_hook(self
, ph
):
722 """Add the postprocessing progress hook"""
723 self
._postprocessor
_hooks
.append(ph
)
725 def _bidi_workaround(self
, message
):
726 if not hasattr(self
, '_output_channel'):
729 assert hasattr(self
, '_output_process')
730 assert isinstance(message
, compat_str
)
731 line_count
= message
.count('\n') + 1
732 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
733 self
._output
_process
.stdin
.flush()
734 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
735 for _
in range(line_count
))
736 return res
[:-len('\n')]
738 def _write_string(self
, message
, out
=None, only_once
=False):
740 if message
in self
._printed
_messages
:
742 self
._printed
_messages
.add(message
)
743 write_string(message
, out
=out
, encoding
=self
.params
.get('encoding'))
745 def to_stdout(self
, message
, skip_eol
=False, quiet
=False):
746 """Print message to stdout"""
747 if self
.params
.get('logger'):
748 self
.params
['logger'].debug(message
)
749 elif not quiet
or self
.params
.get('verbose'):
751 '%s%s' % (self
._bidi
_workaround
(message
), ('' if skip_eol
else '\n')),
752 self
._err
_file
if quiet
else self
._screen
_file
)
754 def to_stderr(self
, message
, only_once
=False):
755 """Print message to stderr"""
756 assert isinstance(message
, compat_str
)
757 if self
.params
.get('logger'):
758 self
.params
['logger'].error(message
)
760 self
._write
_string
('%s\n' % self
._bidi
_workaround
(message
), self
._err
_file
, only_once
=only_once
)
762 def to_console_title(self
, message
):
763 if not self
.params
.get('consoletitle', False):
765 if compat_os_name
== 'nt':
766 if ctypes
.windll
.kernel32
.GetConsoleWindow():
767 # c_wchar_p() might not be necessary if `message` is
768 # already of type unicode()
769 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
770 elif 'TERM' in os
.environ
:
771 self
._write
_string
('\033]0;%s\007' % message
, self
._screen
_file
)
773 def save_console_title(self
):
774 if not self
.params
.get('consoletitle', False):
776 if self
.params
.get('simulate'):
778 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
779 # Save the title on stack
780 self
._write
_string
('\033[22;0t', self
._screen
_file
)
782 def restore_console_title(self
):
783 if not self
.params
.get('consoletitle', False):
785 if self
.params
.get('simulate'):
787 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
788 # Restore the title from stack
789 self
._write
_string
('\033[23;0t', self
._screen
_file
)
792 self
.save_console_title()
795 def __exit__(self
, *args
):
796 self
.restore_console_title()
798 if self
.params
.get('cookiefile') is not None:
799 self
.cookiejar
.save(ignore_discard
=True, ignore_expires
=True)
801 def trouble(self
, message
=None, tb
=None):
802 """Determine action to take when a download problem appears.
804 Depending on if the downloader has been configured to ignore
805 download errors or not, this method may throw an exception or
806 not when errors are found, after printing the message.
808 tb, if given, is additional traceback information.
810 if message
is not None:
811 self
.to_stderr(message
)
812 if self
.params
.get('verbose'):
814 if sys
.exc_info()[0]: # if .trouble has been called from an except block
816 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
817 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
818 tb
+= encode_compat_str(traceback
.format_exc())
820 tb_data
= traceback
.format_list(traceback
.extract_stack())
821 tb
= ''.join(tb_data
)
824 if not self
.params
.get('ignoreerrors'):
825 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
826 exc_info
= sys
.exc_info()[1].exc_info
828 exc_info
= sys
.exc_info()
829 raise DownloadError(message
, exc_info
)
830 self
._download
_retcode
= 1
832 def to_screen(self
, message
, skip_eol
=False):
833 """Print message to stdout if not in quiet mode"""
835 message
, skip_eol
, quiet
=self
.params
.get('quiet', False))
845 def __format_text(self
, out
, text
, f
, fallback
=None, *, test_encoding
=False):
846 assert out
in ('screen', 'err')
849 handle
= self
._screen
_file
if out
== 'screen' else self
._err
_file
850 encoding
= self
.params
.get('encoding') or getattr(handle
, 'encoding', 'ascii')
851 text
= text
.encode(encoding
, 'ignore').decode(encoding
)
852 if fallback
is not None and text
!= original_text
:
854 if isinstance(f
, self
.Styles
):
856 return format_text(text
, f
) if self
._allow
_colors
[out
] else text
if fallback
is None else fallback
858 def _format_screen(self
, *args
, **kwargs
):
859 return self
.__format
_text
('screen', *args
, **kwargs
)
861 def _format_err(self
, *args
, **kwargs
):
862 return self
.__format
_text
('err', *args
, **kwargs
)
864 def report_warning(self
, message
, only_once
=False):
866 Print the message to stderr, it will be prefixed with 'WARNING:'
867 If stderr is a tty file the 'WARNING:' will be colored
869 if self
.params
.get('logger') is not None:
870 self
.params
['logger'].warning(message
)
872 if self
.params
.get('no_warnings'):
874 self
.to_stderr(f
'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once
)
876 def report_error(self
, message
, tb
=None):
878 Do the same as trouble, but prefixes the message with 'ERROR:', colored
879 in red if stderr is a tty file.
881 self
.trouble(f
'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', tb
)
883 def write_debug(self
, message
, only_once
=False):
884 '''Log debug message or Print message to stderr'''
885 if not self
.params
.get('verbose', False):
887 message
= '[debug] %s' % message
888 if self
.params
.get('logger'):
889 self
.params
['logger'].debug(message
)
891 self
.to_stderr(message
, only_once
)
893 def report_file_already_downloaded(self
, file_name
):
894 """Report file has already been fully downloaded."""
896 self
.to_screen('[download] %s has already been downloaded' % file_name
)
897 except UnicodeEncodeError:
898 self
.to_screen('[download] The file has already been downloaded')
900 def report_file_delete(self
, file_name
):
901 """Report that existing file will be deleted."""
903 self
.to_screen('Deleting existing file %s' % file_name
)
904 except UnicodeEncodeError:
905 self
.to_screen('Deleting existing file')
907 def raise_no_formats(self
, info
, forced
=False):
908 has_drm
= info
.get('__has_drm')
909 msg
= 'This video is DRM protected' if has_drm
else 'No video formats found!'
910 expected
= self
.params
.get('ignore_no_formats_error')
911 if forced
or not expected
:
912 raise ExtractorError(msg
, video_id
=info
['id'], ie
=info
['extractor'],
913 expected
=has_drm
or expected
)
915 self
.report_warning(msg
)
917 def parse_outtmpl(self
):
918 outtmpl_dict
= self
.params
.get('outtmpl', {})
919 if not isinstance(outtmpl_dict
, dict):
920 outtmpl_dict
= {'default': outtmpl_dict}
921 # Remove spaces in the default template
922 if self
.params
.get('restrictfilenames'):
923 sanitize
= lambda x
: x
.replace(' - ', ' ').replace(' ', '-')
925 sanitize
= lambda x
: x
926 outtmpl_dict
.update({
927 k
: sanitize(v
) for k
, v
in DEFAULT_OUTTMPL
.items()
928 if outtmpl_dict
.get(k
) is None})
929 for key
, val
in outtmpl_dict
.items():
930 if isinstance(val
, bytes):
932 'Parameter outtmpl is bytes, but should be a unicode string. '
933 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
936 def get_output_path(self
, dir_type
='', filename
=None):
937 paths
= self
.params
.get('paths', {})
938 assert isinstance(paths
, dict)
940 expand_path(paths
.get('home', '').strip()),
941 expand_path(paths
.get(dir_type
, '').strip()) if dir_type
else '',
944 # Temporary fix for #4787
945 # 'Treat' all problem characters by passing filename through preferredencoding
946 # to workaround encoding issues with subprocess on python2 @ Windows
947 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
948 path
= encodeFilename(path
, True).decode(preferredencoding())
949 return sanitize_path(path
, force
=self
.params
.get('windowsfilenames'))
952 def _outtmpl_expandpath(outtmpl
):
953 # expand_path translates '%%' into '%' and '$$' into '$'
954 # correspondingly that is not what we want since we need to keep
955 # '%%' intact for template dict substitution step. Working around
956 # with boundary-alike separator hack.
957 sep
= ''.join([random
.choice(ascii_letters
) for _
in range(32)])
958 outtmpl
= outtmpl
.replace('%%', '%{0}%'.format(sep
)).replace('$$', '${0}$'.format(sep
))
960 # outtmpl should be expand_path'ed before template dict substitution
961 # because meta fields may contain env variables we don't want to
962 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
963 # title "Hello $PATH", we don't want `$PATH` to be expanded.
964 return expand_path(outtmpl
).replace(sep
, '')
967 def escape_outtmpl(outtmpl
):
968 ''' Escape any remaining strings like %s, %abc% etc. '''
970 STR_FORMAT_RE_TMPL
.format('', '(?![%(\0])'),
971 lambda mobj
: ('' if mobj
.group('has_key') else '%') + mobj
.group(0),
975 def validate_outtmpl(cls
, outtmpl
):
976 ''' @return None or Exception object '''
978 STR_FORMAT_RE_TMPL
.format('[^)]*', '[ljqBU]'),
979 lambda mobj
: f
'{mobj.group(0)[:-1]}s',
980 cls
._outtmpl
_expandpath
(outtmpl
))
982 cls
.escape_outtmpl(outtmpl
) % collections
.defaultdict(int)
984 except ValueError as err
:
988 def _copy_infodict(info_dict
):
989 info_dict
= dict(info_dict
)
990 for key
in ('__original_infodict', '__postprocessors'):
991 info_dict
.pop(key
, None)
994 def prepare_outtmpl(self
, outtmpl
, info_dict
, sanitize
=None):
995 """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
996 info_dict
.setdefault('epoch', int(time
.time())) # keep epoch consistent once set
998 info_dict
= self
._copy
_infodict
(info_dict
)
999 info_dict
['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
1000 formatSeconds(info_dict
['duration'], '-' if sanitize
else ':')
1001 if info_dict
.get('duration', None) is not None
1003 info_dict
['autonumber'] = self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
1004 if info_dict
.get('resolution') is None:
1005 info_dict
['resolution'] = self
.format_resolution(info_dict
, default
=None)
1007 # For fields playlist_index, playlist_autonumber and autonumber convert all occurrences
1008 # of %(field)s to %(field)0Nd for backward compatibility
1009 field_size_compat_map
= {
1010 'playlist_index': number_of_digits(info_dict
.get('_last_playlist_index') or 0),
1011 'playlist_autonumber': number_of_digits(info_dict
.get('n_entries') or 0),
1012 'autonumber': self
.params
.get('autonumber_size') or 5,
1016 EXTERNAL_FORMAT_RE
= re
.compile(STR_FORMAT_RE_TMPL
.format('[^)]*', f
'[{STR_FORMAT_TYPES}ljqBU]'))
1021 # Field is of the form key1.key2...
1022 # where keys (except first) can be string, int or slice
1023 FIELD_RE
= r
'\w*(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num
=r
'(?:-?\d+)')
1024 MATH_FIELD_RE
= r
'''{field}|{num}'''.format(field
=FIELD_RE
, num
=r
'-?\d+(?:.\d+)?')
1025 MATH_OPERATORS_RE
= r
'(?:%s)' % '|'.join(map(re
.escape
, MATH_FUNCTIONS
.keys()))
1026 INTERNAL_FORMAT_RE
= re
.compile(r
'''(?x)
1029 (?P<maths>(?:{math_op}{math_field})*)
1030 (?:>(?P<strf_format>.+?))?
1031 (?P<alternate>(?<!\\),[^|)]+)?
1032 (?:\|(?P<default>.*?))?
1033 $'''.format(field
=FIELD_RE
, math_op
=MATH_OPERATORS_RE
, math_field
=MATH_FIELD_RE
))
1035 def _traverse_infodict(k
):
1039 return traverse_obj(info_dict
, k
, is_user_input
=True, traverse_string
=True)
1041 def get_value(mdict
):
1043 value
= _traverse_infodict(mdict
['fields'])
1046 value
= float_or_none(value
)
1047 if value
is not None:
1050 offset_key
= mdict
['maths']
1052 value
= float_or_none(value
)
1056 MATH_FIELD_RE
if operator
else MATH_OPERATORS_RE
,
1057 offset_key
).group(0)
1058 offset_key
= offset_key
[len(item
):]
1059 if operator
is None:
1060 operator
= MATH_FUNCTIONS
[item
]
1062 item
, multiplier
= (item
[1:], -1) if item
[0] == '-' else (item
, 1)
1063 offset
= float_or_none(item
)
1065 offset
= float_or_none(_traverse_infodict(item
))
1067 value
= operator(value
, multiplier
* offset
)
1068 except (TypeError, ZeroDivisionError):
1071 # Datetime formatting
1072 if mdict
['strf_format']:
1073 value
= strftime_or_none(value
, mdict
['strf_format'].replace('\\,', ','))
1077 na
= self
.params
.get('outtmpl_na_placeholder', 'NA')
1079 def _dumpjson_default(obj
):
1080 if isinstance(obj
, (set, LazyList
)):
1082 raise TypeError(f
'Object of type {type(obj).__name__} is not JSON serializable')
1084 def create_key(outer_mobj
):
1085 if not outer_mobj
.group('has_key'):
1086 return outer_mobj
.group(0)
1087 key
= outer_mobj
.group('key')
1088 mobj
= re
.match(INTERNAL_FORMAT_RE
, key
)
1089 initial_field
= mobj
.group('fields').split('.')[-1] if mobj
else ''
1090 value
, default
= None, na
1092 mobj
= mobj
.groupdict()
1093 default
= mobj
['default'] if mobj
['default'] is not None else default
1094 value
= get_value(mobj
)
1095 if value
is None and mobj
['alternate']:
1096 mobj
= re
.match(INTERNAL_FORMAT_RE
, mobj
['alternate'][1:])
1100 fmt
= outer_mobj
.group('format')
1101 if fmt
== 's' and value
is not None and key
in field_size_compat_map
.keys():
1102 fmt
= '0{:d}d'.format(field_size_compat_map
[key
])
1104 value
= default
if value
is None else value
1106 str_fmt
= f
'{fmt[:-1]}s'
1107 if fmt
[-1] == 'l': # list
1108 delim
= '\n' if '#' in (outer_mobj
.group('conversion') or '') else ', '
1109 value
, fmt
= delim
.join(variadic(value
)), str_fmt
1110 elif fmt
[-1] == 'j': # json
1111 value
, fmt
= json
.dumps(value
, default
=_dumpjson_default
), str_fmt
1112 elif fmt
[-1] == 'q': # quoted
1113 value
, fmt
= compat_shlex_quote(str(value
)), str_fmt
1114 elif fmt
[-1] == 'B': # bytes
1115 value
= f
'%{str_fmt}'.encode('utf-8') % str(value
).encode('utf-8')
1116 value
, fmt
= value
.decode('utf-8', 'ignore'), 's'
1117 elif fmt
[-1] == 'U': # unicode normalized
1118 opts
= outer_mobj
.group('conversion') or ''
1119 value
, fmt
= unicodedata
.normalize(
1120 # "+" = compatibility equivalence, "#" = NFD
1121 'NF%s%s' % ('K' if '+' in opts
else '', 'D' if '#' in opts
else 'C'),
1123 elif fmt
[-1] == 'c':
1125 value
= str(value
)[0]
1128 elif fmt
[-1] not in 'rs': # numeric
1129 value
= float_or_none(value
)
1131 value
, fmt
= default
, 's'
1135 # If value is an object, sanitize might convert it to a string
1136 # So we convert it to repr first
1137 value
, fmt
= repr(value
), str_fmt
1138 if fmt
[-1] in 'csr':
1139 value
= sanitize(initial_field
, value
)
1141 key
= '%s\0%s' % (key
.replace('%', '%\0'), outer_mobj
.group('format'))
1142 TMPL_DICT
[key
] = value
1143 return '{prefix}%({key}){fmt}'.format(key
=key
, fmt
=fmt
, prefix
=outer_mobj
.group('prefix'))
1145 return EXTERNAL_FORMAT_RE
.sub(create_key
, outtmpl
), TMPL_DICT
1147 def evaluate_outtmpl(self
, outtmpl
, info_dict
, *args
, **kwargs
):
1148 outtmpl
, info_dict
= self
.prepare_outtmpl(outtmpl
, info_dict
, *args
, **kwargs
)
1149 return self
.escape_outtmpl(outtmpl
) % info_dict
1151 def _prepare_filename(self
, info_dict
, tmpl_type
='default'):
1153 sanitize
= lambda k
, v
: sanitize_filename(
1155 restricted
=self
.params
.get('restrictfilenames'),
1156 is_id
=(k
== 'id' or k
.endswith('_id')))
1157 outtmpl
= self
._outtmpl
_expandpath
(self
.outtmpl_dict
.get(tmpl_type
, self
.outtmpl_dict
['default']))
1158 filename
= self
.evaluate_outtmpl(outtmpl
, info_dict
, sanitize
)
1160 force_ext
= OUTTMPL_TYPES
.get(tmpl_type
)
1161 if filename
and force_ext
is not None:
1162 filename
= replace_extension(filename
, force_ext
, info_dict
.get('ext'))
1164 # https://github.com/blackjack4494/youtube-dlc/issues/85
1165 trim_file_name
= self
.params
.get('trim_file_name', False)
1167 fn_groups
= filename
.rsplit('.')
1170 if len(fn_groups
) > 2:
1171 sub_ext
= fn_groups
[-2]
1172 filename
= '.'.join(filter(None, [fn_groups
[0][:trim_file_name
], sub_ext
, ext
]))
1175 except ValueError as err
:
1176 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
1179 def prepare_filename(self
, info_dict
, dir_type
='', warn
=False):
1180 """Generate the output filename."""
1182 filename
= self
._prepare
_filename
(info_dict
, dir_type
or 'default')
1183 if not filename
and dir_type
not in ('', 'temp'):
1187 if not self
.params
.get('paths'):
1189 elif filename
== '-':
1190 self
.report_warning('--paths is ignored when an outputting to stdout', only_once
=True)
1191 elif os
.path
.isabs(filename
):
1192 self
.report_warning('--paths is ignored since an absolute path is given in output template', only_once
=True)
1193 if filename
== '-' or not filename
:
1196 return self
.get_output_path(dir_type
, filename
)
1198 def _match_entry(self
, info_dict
, incomplete
=False, silent
=False):
1199 """ Returns None if the file should be downloaded """
1201 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
1204 if 'title' in info_dict
:
1205 # This can happen when we're just evaluating the playlist
1206 title
= info_dict
['title']
1207 matchtitle
= self
.params
.get('matchtitle', False)
1209 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
1210 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
1211 rejecttitle
= self
.params
.get('rejecttitle', False)
1213 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
1214 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
1215 date
= info_dict
.get('upload_date')
1216 if date
is not None:
1217 dateRange
= self
.params
.get('daterange', DateRange())
1218 if date
not in dateRange
:
1219 return '%s upload date is not in range %s' % (date_from_str(date
).isoformat(), dateRange
)
1220 view_count
= info_dict
.get('view_count')
1221 if view_count
is not None:
1222 min_views
= self
.params
.get('min_views')
1223 if min_views
is not None and view_count
< min_views
:
1224 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
1225 max_views
= self
.params
.get('max_views')
1226 if max_views
is not None and view_count
> max_views
:
1227 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
1228 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
1229 return 'Skipping "%s" because it is age restricted' % video_title
1231 match_filter
= self
.params
.get('match_filter')
1232 if match_filter
is not None:
1234 ret
= match_filter(info_dict
, incomplete
=incomplete
)
1236 # For backward compatibility
1237 ret
= None if incomplete
else match_filter(info_dict
)
1242 if self
.in_download_archive(info_dict
):
1243 reason
= '%s has already been recorded in the archive' % video_title
1244 break_opt
, break_err
= 'break_on_existing', ExistingVideoReached
1246 reason
= check_filter()
1247 break_opt
, break_err
= 'break_on_reject', RejectedVideoReached
1248 if reason
is not None:
1250 self
.to_screen('[download] ' + reason
)
1251 if self
.params
.get(break_opt
, False):
1256 def add_extra_info(info_dict
, extra_info
):
1257 '''Set the keys from extra_info in info dict if they are missing'''
1258 for key
, value
in extra_info
.items():
1259 info_dict
.setdefault(key
, value
)
1261 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
=None,
1262 process
=True, force_generic_extractor
=False):
1264 Return a list with a dictionary for each video extracted.
1267 url -- URL to extract
1270 download -- whether to download videos during extraction
1271 ie_key -- extractor key hint
1272 extra_info -- dictionary containing the extra values to add to each result
1273 process -- whether to resolve all unresolved references (URLs, playlist items),
1274 must be True for download to work.
1275 force_generic_extractor -- force using the generic extractor
1278 if extra_info
is None:
1281 if not ie_key
and force_generic_extractor
:
1285 ies
= {ie_key: self._get_info_extractor_class(ie_key)}
1289 for ie_key
, ie
in ies
.items():
1290 if not ie
.suitable(url
):
1293 if not ie
.working():
1294 self
.report_warning('The program functionality for this site has been marked as broken, '
1295 'and will probably not work.')
1297 temp_id
= ie
.get_temp_id(url
)
1298 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': ie_key}
):
1299 self
.to_screen("[%s] %s: has already been recorded in archive" % (
1302 return self
.__extract
_info
(url
, self
.get_info_extractor(ie_key
), download
, extra_info
, process
)
1304 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
1306 def __handle_extraction_exceptions(func
):
1307 @functools.wraps(func
)
1308 def wrapper(self
, *args
, **kwargs
):
1310 return func(self
, *args
, **kwargs
)
1311 except GeoRestrictedError
as e
:
1314 msg
+= '\nThis video is available in %s.' % ', '.join(
1315 map(ISO3166Utils
.short2full
, e
.countries
))
1316 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
1317 self
.report_error(msg
)
1318 except ExtractorError
as e
: # An error we somewhat expected
1319 self
.report_error(compat_str(e
), e
.format_traceback())
1320 except ThrottledDownload
:
1321 self
.to_stderr('\r')
1322 self
.report_warning('The download speed is below throttle limit. Re-extracting data')
1323 return wrapper(self
, *args
, **kwargs
)
1324 except (DownloadCancelled
, LazyList
.IndexError):
1326 except Exception as e
:
1327 if self
.params
.get('ignoreerrors'):
1328 self
.report_error(error_to_compat_str(e
), tb
=encode_compat_str(traceback
.format_exc()))
1333 @__handle_extraction_exceptions
1334 def __extract_info(self
, url
, ie
, download
, extra_info
, process
):
1335 ie_result
= ie
.extract(url
)
1336 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
1338 if isinstance(ie_result
, list):
1339 # Backwards compatibility: old IE result format
1341 '_type': 'compat_list',
1342 'entries': ie_result
,
1344 if extra_info
.get('original_url'):
1345 ie_result
.setdefault('original_url', extra_info
['original_url'])
1346 self
.add_default_extra_info(ie_result
, ie
, url
)
1348 return self
.process_ie_result(ie_result
, download
, extra_info
)
1352 def add_default_extra_info(self
, ie_result
, ie
, url
):
1354 self
.add_extra_info(ie_result
, {
1356 'original_url': url
,
1357 'webpage_url_basename': url_basename(url
),
1360 self
.add_extra_info(ie_result
, {
1361 'extractor': ie
.IE_NAME
,
1362 'extractor_key': ie
.ie_key(),
1365 def process_ie_result(self
, ie_result
, download
=True, extra_info
=None):
1367 Take the result of the ie(may be modified) and resolve all unresolved
1368 references (URLs, playlist items).
1370 It will also download the videos if 'download'.
1371 Returns the resolved ie_result.
1373 if extra_info
is None:
1375 result_type
= ie_result
.get('_type', 'video')
1377 if result_type
in ('url', 'url_transparent'):
1378 ie_result
['url'] = sanitize_url(ie_result
['url'])
1379 if ie_result
.get('original_url'):
1380 extra_info
.setdefault('original_url', ie_result
['original_url'])
1382 extract_flat
= self
.params
.get('extract_flat', False)
1383 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
1384 or extract_flat
is True):
1385 info_copy
= ie_result
.copy()
1386 ie
= try_get(ie_result
.get('ie_key'), self
.get_info_extractor
)
1387 if ie
and not ie_result
.get('id'):
1388 info_copy
['id'] = ie
.get_temp_id(ie_result
['url'])
1389 self
.add_default_extra_info(info_copy
, ie
, ie_result
['url'])
1390 self
.add_extra_info(info_copy
, extra_info
)
1391 self
.__forced
_printings
(info_copy
, self
.prepare_filename(info_copy
), incomplete
=True)
1392 if self
.params
.get('force_write_download_archive', False):
1393 self
.record_download_archive(info_copy
)
1396 if result_type
== 'video':
1397 self
.add_extra_info(ie_result
, extra_info
)
1398 ie_result
= self
.process_video_result(ie_result
, download
=download
)
1399 additional_urls
= (ie_result
or {}).get('additional_urls')
1401 # TODO: Improve MetadataParserPP to allow setting a list
1402 if isinstance(additional_urls
, compat_str
):
1403 additional_urls
= [additional_urls
]
1405 '[info] %s: %d additional URL(s) requested' % (ie_result
['id'], len(additional_urls
)))
1406 self
.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls
))
1407 ie_result
['additional_entries'] = [
1409 url
, download
, extra_info
,
1410 force_generic_extractor
=self
.params
.get('force_generic_extractor'))
1411 for url
in additional_urls
1414 elif result_type
== 'url':
1415 # We have to add extra_info to the results because it may be
1416 # contained in a playlist
1417 return self
.extract_info(
1418 ie_result
['url'], download
,
1419 ie_key
=ie_result
.get('ie_key'),
1420 extra_info
=extra_info
)
1421 elif result_type
== 'url_transparent':
1422 # Use the information from the embedding page
1423 info
= self
.extract_info(
1424 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
1425 extra_info
=extra_info
, download
=False, process
=False)
1427 # extract_info may return None when ignoreerrors is enabled and
1428 # extraction failed with an error, don't crash and return early
1433 force_properties
= dict(
1434 (k
, v
) for k
, v
in ie_result
.items() if v
is not None)
1435 for f
in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
1436 if f
in force_properties
:
1437 del force_properties
[f
]
1438 new_result
= info
.copy()
1439 new_result
.update(force_properties
)
1441 # Extracted info may not be a video result (i.e.
1442 # info.get('_type', 'video') != video) but rather an url or
1443 # url_transparent. In such cases outer metadata (from ie_result)
1444 # should be propagated to inner one (info). For this to happen
1445 # _type of info should be overridden with url_transparent. This
1446 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
1447 if new_result
.get('_type') == 'url':
1448 new_result
['_type'] = 'url_transparent'
1450 return self
.process_ie_result(
1451 new_result
, download
=download
, extra_info
=extra_info
)
1452 elif result_type
in ('playlist', 'multi_video'):
1453 # Protect from infinite recursion due to recursively nested playlists
1454 # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
1455 webpage_url
= ie_result
['webpage_url']
1456 if webpage_url
in self
._playlist
_urls
:
1458 '[download] Skipping already downloaded playlist: %s'
1459 % ie_result
.get('title') or ie_result
.get('id'))
1462 self
._playlist
_level
+= 1
1463 self
._playlist
_urls
.add(webpage_url
)
1464 self
._sanitize
_thumbnails
(ie_result
)
1466 return self
.__process
_playlist
(ie_result
, download
)
1468 self
._playlist
_level
-= 1
1469 if not self
._playlist
_level
:
1470 self
._playlist
_urls
.clear()
1471 elif result_type
== 'compat_list':
1472 self
.report_warning(
1473 'Extractor %s returned a compat_list result. '
1474 'It needs to be updated.' % ie_result
.get('extractor'))
1477 self
.add_extra_info(r
, {
1478 'extractor': ie_result
['extractor'],
1479 'webpage_url': ie_result
['webpage_url'],
1480 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1481 'extractor_key': ie_result
['extractor_key'],
1484 ie_result
['entries'] = [
1485 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1486 for r
in ie_result
['entries']
1490 raise Exception('Invalid result type: %s' % result_type
)
1492 def _ensure_dir_exists(self
, path
):
1493 return make_dir(path
, self
.report_error
)
1495 def __process_playlist(self
, ie_result
, download
):
1496 # We process each entry in the playlist
1497 playlist
= ie_result
.get('title') or ie_result
.get('id')
1498 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
1500 if 'entries' not in ie_result
:
1501 raise EntryNotInPlaylist()
1502 incomplete_entries
= bool(ie_result
.get('requested_entries'))
1503 if incomplete_entries
:
1504 def fill_missing_entries(entries
, indexes
):
1505 ret
= [None] * max(*indexes
)
1506 for i
, entry
in zip(indexes
, entries
):
1509 ie_result
['entries'] = fill_missing_entries(ie_result
['entries'], ie_result
['requested_entries'])
1511 playlist_results
= []
1513 playliststart
= self
.params
.get('playliststart', 1)
1514 playlistend
= self
.params
.get('playlistend')
1515 # For backwards compatibility, interpret -1 as whole list
1516 if playlistend
== -1:
1519 playlistitems_str
= self
.params
.get('playlist_items')
1520 playlistitems
= None
1521 if playlistitems_str
is not None:
1522 def iter_playlistitems(format
):
1523 for string_segment
in format
.split(','):
1524 if '-' in string_segment
:
1525 start
, end
= string_segment
.split('-')
1526 for item
in range(int(start
), int(end
) + 1):
1529 yield int(string_segment
)
1530 playlistitems
= orderedSet(iter_playlistitems(playlistitems_str
))
1532 ie_entries
= ie_result
['entries']
1534 'Downloading %d videos' if not isinstance(ie_entries
, list)
1535 else 'Collected %d videos; downloading %%d of them' % len(ie_entries
))
1537 if isinstance(ie_entries
, list):
1539 return ie_entries
[i
- 1]
1541 if not isinstance(ie_entries
, PagedList
):
1542 ie_entries
= LazyList(ie_entries
)
1545 return YoutubeDL
.__handle
_extraction
_exceptions
(
1546 lambda self
, i
: ie_entries
[i
- 1]
1550 items
= playlistitems
if playlistitems
is not None else itertools
.count(playliststart
)
1554 if playlistitems
is None and playlistend
is not None and playlistend
< i
:
1558 entry
= get_entry(i
)
1560 raise EntryNotInPlaylist()
1561 except (IndexError, EntryNotInPlaylist
):
1562 if incomplete_entries
:
1563 raise EntryNotInPlaylist()
1564 elif not playlistitems
:
1566 entries
.append(entry
)
1568 if entry
is not None:
1569 self
._match
_entry
(entry
, incomplete
=True, silent
=True)
1570 except (ExistingVideoReached
, RejectedVideoReached
):
1572 ie_result
['entries'] = entries
1574 # Save playlist_index before re-ordering
1576 ((playlistitems
[i
- 1] if playlistitems
else i
+ playliststart
- 1), entry
)
1577 for i
, entry
in enumerate(entries
, 1)
1578 if entry
is not None]
1579 n_entries
= len(entries
)
1581 if not playlistitems
and (playliststart
or playlistend
):
1582 playlistitems
= list(range(playliststart
, playliststart
+ n_entries
))
1583 ie_result
['requested_entries'] = playlistitems
1585 if not self
.params
.get('simulate') and self
.params
.get('allow_playlist_files', True):
1587 'playlist': playlist
,
1588 'playlist_id': ie_result
.get('id'),
1589 'playlist_title': ie_result
.get('title'),
1590 'playlist_uploader': ie_result
.get('uploader'),
1591 'playlist_uploader_id': ie_result
.get('uploader_id'),
1592 'playlist_index': 0,
1593 'n_entries': n_entries
,
1595 ie_copy
.update(dict(ie_result
))
1597 if self
._write
_info
_json
('playlist', ie_result
,
1598 self
.prepare_filename(ie_copy
, 'pl_infojson')) is None:
1600 if self
._write
_description
('playlist', ie_result
,
1601 self
.prepare_filename(ie_copy
, 'pl_description')) is None:
1603 # TODO: This should be passed to ThumbnailsConvertor if necessary
1604 self
._write
_thumbnails
('playlist', ie_copy
, self
.prepare_filename(ie_copy
, 'pl_thumbnail'))
1606 if self
.params
.get('playlistreverse', False):
1607 entries
= entries
[::-1]
1608 if self
.params
.get('playlistrandom', False):
1609 random
.shuffle(entries
)
1611 x_forwarded_for
= ie_result
.get('__x_forwarded_for_ip')
1613 self
.to_screen('[%s] playlist %s: %s' % (ie_result
['extractor'], playlist
, msg
% n_entries
))
1615 max_failures
= self
.params
.get('skip_playlist_after_errors') or float('inf')
1616 for i
, entry_tuple
in enumerate(entries
, 1):
1617 playlist_index
, entry
= entry_tuple
1618 if 'playlist-index' in self
.params
.get('compat_opts', []):
1619 playlist_index
= playlistitems
[i
- 1] if playlistitems
else i
+ playliststart
- 1
1620 self
.to_screen('[download] Downloading video %s of %s' % (i
, n_entries
))
1621 # This __x_forwarded_for_ip thing is a bit ugly but requires
1624 entry
['__x_forwarded_for_ip'] = x_forwarded_for
1626 'n_entries': n_entries
,
1627 '_last_playlist_index': max(playlistitems
) if playlistitems
else (playlistend
or n_entries
),
1628 'playlist_index': playlist_index
,
1629 'playlist_autonumber': i
,
1630 'playlist': playlist
,
1631 'playlist_id': ie_result
.get('id'),
1632 'playlist_title': ie_result
.get('title'),
1633 'playlist_uploader': ie_result
.get('uploader'),
1634 'playlist_uploader_id': ie_result
.get('uploader_id'),
1635 'extractor': ie_result
['extractor'],
1636 'webpage_url': ie_result
['webpage_url'],
1637 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1638 'extractor_key': ie_result
['extractor_key'],
1641 if self
._match
_entry
(entry
, incomplete
=True) is not None:
1644 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, extra
)
1645 if not entry_result
:
1647 if failures
>= max_failures
:
1649 'Skipping the remaining entries in playlist "%s" since %d items failed extraction' % (playlist
, failures
))
1651 # TODO: skip failed (empty) entries?
1652 playlist_results
.append(entry_result
)
1653 ie_result
['entries'] = playlist_results
1654 self
.to_screen('[download] Finished downloading playlist: %s' % playlist
)
1657 @__handle_extraction_exceptions
1658 def __process_iterable_entry(self
, entry
, download
, extra_info
):
1659 return self
.process_ie_result(
1660 entry
, download
=download
, extra_info
=extra_info
)
1662 def _build_format_filter(self
, filter_spec
):
1663 " Returns a function to filter the formats according to the filter_spec "
1673 operator_rex
= re
.compile(r
'''(?x)\s*
1674 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)\s*
1675 (?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1676 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s*
1677 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
1678 m
= operator_rex
.fullmatch(filter_spec
)
1681 comparison_value
= int(m
.group('value'))
1683 comparison_value
= parse_filesize(m
.group('value'))
1684 if comparison_value
is None:
1685 comparison_value
= parse_filesize(m
.group('value') + 'B')
1686 if comparison_value
is None:
1688 'Invalid value %r in format specification %r' % (
1689 m
.group('value'), filter_spec
))
1690 op
= OPERATORS
[m
.group('op')]
1695 '^=': lambda attr
, value
: attr
.startswith(value
),
1696 '$=': lambda attr
, value
: attr
.endswith(value
),
1697 '*=': lambda attr
, value
: value
in attr
,
1699 str_operator_rex
= re
.compile(r
'''(?x)\s*
1700 (?P<key>[a-zA-Z0-9._-]+)\s*
1701 (?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1702 (?P<value>[a-zA-Z0-9._-]+)\s*
1703 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
1704 m
= str_operator_rex
.fullmatch(filter_spec
)
1706 comparison_value
= m
.group('value')
1707 str_op
= STR_OPERATORS
[m
.group('op')]
1708 if m
.group('negation'):
1709 op
= lambda attr
, value
: not str_op(attr
, value
)
1714 raise SyntaxError('Invalid filter specification %r' % filter_spec
)
1717 actual_value
= f
.get(m
.group('key'))
1718 if actual_value
is None:
1719 return m
.group('none_inclusive')
1720 return op(actual_value
, comparison_value
)
1723 def _check_formats(self
, formats
):
1725 self
.to_screen('[info] Testing format %s' % f
['format_id'])
1726 temp_file
= tempfile
.NamedTemporaryFile(
1727 suffix
='.tmp', delete
=False,
1728 dir=self
.get_output_path('temp') or None)
1731 success
, _
= self
.dl(temp_file
.name
, f
, test
=True)
1732 except (DownloadError
, IOError, OSError, ValueError) + network_exceptions
:
1735 if os
.path
.exists(temp_file
.name
):
1737 os
.remove(temp_file
.name
)
1739 self
.report_warning('Unable to delete temporary file "%s"' % temp_file
.name
)
1743 self
.to_screen('[info] Unable to download format %s. Skipping...' % f
['format_id'])
1745 def _default_format_spec(self
, info_dict
, download
=True):
1748 merger
= FFmpegMergerPP(self
)
1749 return merger
.available
and merger
.can_merge()
1752 not self
.params
.get('simulate')
1756 or info_dict
.get('is_live', False)
1757 or self
.outtmpl_dict
['default'] == '-'))
1760 or self
.params
.get('allow_multiple_audio_streams', False)
1761 or 'format-spec' in self
.params
.get('compat_opts', []))
1764 'best/bestvideo+bestaudio' if prefer_best
1765 else 'bestvideo*+bestaudio/best' if not compat
1766 else 'bestvideo+bestaudio/best')
1768 def build_format_selector(self
, format_spec
):
1769 def syntax_error(note
, start
):
1771 'Invalid format specification: '
1772 '{0}\n\t{1}\n\t{2}^'.format(note
, format_spec
, ' ' * start
[1]))
1773 return SyntaxError(message
)
1775 PICKFIRST
= 'PICKFIRST'
1779 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1781 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
1782 'video': self
.params
.get('allow_multiple_video_streams', False)}
1784 check_formats
= self
.params
.get('check_formats') == 'selected'
1786 def _parse_filter(tokens
):
1788 for type, string
, start
, _
, _
in tokens
:
1789 if type == tokenize
.OP
and string
== ']':
1790 return ''.join(filter_parts
)
1792 filter_parts
.append(string
)
1794 def _remove_unused_ops(tokens
):
1795 # Remove operators that we don't use and join them with the surrounding strings
1796 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1797 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
1798 last_string
, last_start
, last_end
, last_line
= None, None, None, None
1799 for type, string
, start
, end
, line
in tokens
:
1800 if type == tokenize
.OP
and string
== '[':
1802 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1804 yield type, string
, start
, end
, line
1805 # everything inside brackets will be handled by _parse_filter
1806 for type, string
, start
, end
, line
in tokens
:
1807 yield type, string
, start
, end
, line
1808 if type == tokenize
.OP
and string
== ']':
1810 elif type == tokenize
.OP
and string
in ALLOWED_OPS
:
1812 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1814 yield type, string
, start
, end
, line
1815 elif type in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
1817 last_string
= string
1821 last_string
+= string
1823 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1825 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
1827 current_selector
= None
1828 for type, string
, start
, _
, _
in tokens
:
1829 # ENCODING is only defined in python 3.x
1830 if type == getattr(tokenize
, 'ENCODING', None):
1832 elif type in [tokenize
.NAME
, tokenize
.NUMBER
]:
1833 current_selector
= FormatSelector(SINGLE
, string
, [])
1834 elif type == tokenize
.OP
:
1836 if not inside_group
:
1837 # ')' will be handled by the parentheses group
1838 tokens
.restore_last_token()
1840 elif inside_merge
and string
in ['/', ',']:
1841 tokens
.restore_last_token()
1843 elif inside_choice
and string
== ',':
1844 tokens
.restore_last_token()
1847 if not current_selector
:
1848 raise syntax_error('"," must follow a format selector', start
)
1849 selectors
.append(current_selector
)
1850 current_selector
= None
1852 if not current_selector
:
1853 raise syntax_error('"/" must follow a format selector', start
)
1854 first_choice
= current_selector
1855 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
1856 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
1858 if not current_selector
:
1859 current_selector
= FormatSelector(SINGLE
, 'best', [])
1860 format_filter
= _parse_filter(tokens
)
1861 current_selector
.filters
.append(format_filter
)
1863 if current_selector
:
1864 raise syntax_error('Unexpected "("', start
)
1865 group
= _parse_format_selection(tokens
, inside_group
=True)
1866 current_selector
= FormatSelector(GROUP
, group
, [])
1868 if not current_selector
:
1869 raise syntax_error('Unexpected "+"', start
)
1870 selector_1
= current_selector
1871 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
1873 raise syntax_error('Expected a selector', start
)
1874 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
1876 raise syntax_error('Operator not recognized: "{0}"'.format(string
), start
)
1877 elif type == tokenize
.ENDMARKER
:
1879 if current_selector
:
1880 selectors
.append(current_selector
)
1883 def _merge(formats_pair
):
1884 format_1
, format_2
= formats_pair
1887 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
1888 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
1890 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
1891 get_no_more
= {'video': False, 'audio': False}
1892 for (i
, fmt_info
) in enumerate(formats_info
):
1893 if fmt_info
.get('acodec') == fmt_info
.get('vcodec') == 'none':
1896 for aud_vid
in ['audio', 'video']:
1897 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
1898 if get_no_more
[aud_vid
]:
1901 get_no_more
[aud_vid
] = True
1903 if len(formats_info
) == 1:
1904 return formats_info
[0]
1906 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
1907 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
1909 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
1910 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
1912 output_ext
= self
.params
.get('merge_output_format')
1915 output_ext
= the_only_video
['ext']
1916 elif the_only_audio
and not video_fmts
:
1917 output_ext
= the_only_audio
['ext']
1921 filtered
= lambda *keys
: filter(None, (traverse_obj(fmt
, *keys
) for fmt
in formats_info
))
1924 'requested_formats': formats_info
,
1925 'format': '+'.join(filtered('format')),
1926 'format_id': '+'.join(filtered('format_id')),
1928 'protocol': '+'.join(map(determine_protocol
, formats_info
)),
1929 'language': '+'.join(orderedSet(filtered('language'))),
1930 'format_note': '+'.join(orderedSet(filtered('format_note'))),
1931 'filesize_approx': sum(filtered('filesize', 'filesize_approx')),
1932 'tbr': sum(filtered('tbr', 'vbr', 'abr')),
1937 'width': the_only_video
.get('width'),
1938 'height': the_only_video
.get('height'),
1939 'resolution': the_only_video
.get('resolution') or self
.format_resolution(the_only_video
),
1940 'fps': the_only_video
.get('fps'),
1941 'dynamic_range': the_only_video
.get('dynamic_range'),
1942 'vcodec': the_only_video
.get('vcodec'),
1943 'vbr': the_only_video
.get('vbr'),
1944 'stretched_ratio': the_only_video
.get('stretched_ratio'),
1949 'acodec': the_only_audio
.get('acodec'),
1950 'abr': the_only_audio
.get('abr'),
1951 'asr': the_only_audio
.get('asr'),
1956 def _check_formats(formats
):
1957 if not check_formats
:
1960 yield from self
._check
_formats
(formats
)
1962 def _build_selector_function(selector
):
1963 if isinstance(selector
, list): # ,
1964 fs
= [_build_selector_function(s
) for s
in selector
]
1966 def selector_function(ctx
):
1969 return selector_function
1971 elif selector
.type == GROUP
: # ()
1972 selector_function
= _build_selector_function(selector
.selector
)
1974 elif selector
.type == PICKFIRST
: # /
1975 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
1977 def selector_function(ctx
):
1979 picked_formats
= list(f(ctx
))
1981 return picked_formats
1984 elif selector
.type == MERGE
: # +
1985 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
1987 def selector_function(ctx
):
1988 for pair
in itertools
.product(
1989 selector_1(copy
.deepcopy(ctx
)), selector_2(copy
.deepcopy(ctx
))):
1992 elif selector
.type == SINGLE
: # atom
1993 format_spec
= selector
.selector
or 'best'
1995 # TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
1996 if format_spec
== 'all':
1997 def selector_function(ctx
):
1998 yield from _check_formats(ctx
['formats'])
1999 elif format_spec
== 'mergeall':
2000 def selector_function(ctx
):
2001 formats
= list(_check_formats(ctx
['formats']))
2004 merged_format
= formats
[-1]
2005 for f
in formats
[-2::-1]:
2006 merged_format
= _merge((merged_format
, f
))
2010 format_fallback
, format_reverse
, format_idx
= False, True, 1
2012 r
'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
2014 if mobj
is not None:
2015 format_idx
= int_or_none(mobj
.group('n'), default
=1)
2016 format_reverse
= mobj
.group('bw')[0] == 'b'
2017 format_type
= (mobj
.group('type') or [None])[0]
2018 not_format_type
= {'v': 'a', 'a': 'v'}
.get(format_type
)
2019 format_modified
= mobj
.group('mod') is not None
2021 format_fallback
= not format_type
and not format_modified
# for b, w
2023 (lambda f
: f
.get('%scodec' % format_type
) != 'none')
2024 if format_type
and format_modified
# bv*, ba*, wv*, wa*
2025 else (lambda f
: f
.get('%scodec' % not_format_type
) == 'none')
2026 if format_type
# bv, ba, wv, wa
2027 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
2028 if not format_modified
# b, w
2029 else lambda f
: True) # b*, w*
2030 filter_f
= lambda f
: _filter_f(f
) and (
2031 f
.get('vcodec') != 'none' or f
.get('acodec') != 'none')
2033 if format_spec
in self
._format
_selection
_exts
['audio']:
2034 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none'
2035 elif format_spec
in self
._format
_selection
_exts
['video']:
2036 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none' and f
.get('vcodec') != 'none'
2037 elif format_spec
in self
._format
_selection
_exts
['storyboards']:
2038 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') == 'none' and f
.get('vcodec') == 'none'
2040 filter_f
= lambda f
: f
.get('format_id') == format_spec
# id
2042 def selector_function(ctx
):
2043 formats
= list(ctx
['formats'])
2044 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
2045 if format_fallback
and ctx
['incomplete_formats'] and not matches
:
2046 # for extractors with incomplete formats (audio only (soundcloud)
2047 # or video only (imgur)) best/worst will fallback to
2048 # best/worst {video,audio}-only format
2050 matches
= LazyList(_check_formats(matches
[::-1 if format_reverse
else 1]))
2052 yield matches
[format_idx
- 1]
2056 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
2058 def final_selector(ctx
):
2059 ctx_copy
= copy
.deepcopy(ctx
)
2060 for _filter
in filters
:
2061 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
2062 return selector_function(ctx_copy
)
2063 return final_selector
2065 stream
= io
.BytesIO(format_spec
.encode('utf-8'))
2067 tokens
= list(_remove_unused_ops(compat_tokenize_tokenize(stream
.readline
)))
2068 except tokenize
.TokenError
:
2069 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
2071 class TokenIterator(object):
2072 def __init__(self
, tokens
):
2073 self
.tokens
= tokens
2080 if self
.counter
>= len(self
.tokens
):
2081 raise StopIteration()
2082 value
= self
.tokens
[self
.counter
]
2088 def restore_last_token(self
):
2091 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
2092 return _build_selector_function(parsed_selector
)
2094 def _calc_headers(self
, info_dict
):
2095 res
= std_headers
.copy()
2097 add_headers
= info_dict
.get('http_headers')
2099 res
.update(add_headers
)
2101 cookies
= self
._calc
_cookies
(info_dict
)
2103 res
['Cookie'] = cookies
2105 if 'X-Forwarded-For' not in res
:
2106 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
2107 if x_forwarded_for_ip
:
2108 res
['X-Forwarded-For'] = x_forwarded_for_ip
2112 def _calc_cookies(self
, info_dict
):
2113 pr
= sanitized_Request(info_dict
['url'])
2114 self
.cookiejar
.add_cookie_header(pr
)
2115 return pr
.get_header('Cookie')
2117 def _sort_thumbnails(self
, thumbnails
):
2118 thumbnails
.sort(key
=lambda t
: (
2119 t
.get('preference') if t
.get('preference') is not None else -1,
2120 t
.get('width') if t
.get('width') is not None else -1,
2121 t
.get('height') if t
.get('height') is not None else -1,
2122 t
.get('id') if t
.get('id') is not None else '',
2125 def _sanitize_thumbnails(self
, info_dict
):
2126 thumbnails
= info_dict
.get('thumbnails')
2127 if thumbnails
is None:
2128 thumbnail
= info_dict
.get('thumbnail')
2130 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
2134 def check_thumbnails(thumbnails
):
2135 for t
in thumbnails
:
2136 self
.to_screen(f
'[info] Testing thumbnail {t["id"]}')
2138 self
.urlopen(HEADRequest(t
['url']))
2139 except network_exceptions
as err
:
2140 self
.to_screen(f
'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...')
2144 self
._sort
_thumbnails
(thumbnails
)
2145 for i
, t
in enumerate(thumbnails
):
2146 if t
.get('id') is None:
2148 if t
.get('width') and t
.get('height'):
2149 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
2150 t
['url'] = sanitize_url(t
['url'])
2152 if self
.params
.get('check_formats') is True:
2153 info_dict
['thumbnails'] = LazyList(check_thumbnails(thumbnails
[::-1])).reverse()
2155 info_dict
['thumbnails'] = thumbnails
2157 def process_video_result(self
, info_dict
, download
=True):
2158 assert info_dict
.get('_type', 'video') == 'video'
2160 if 'id' not in info_dict
:
2161 raise ExtractorError('Missing "id" field in extractor result')
2162 if 'title' not in info_dict
:
2163 raise ExtractorError('Missing "title" field in extractor result',
2164 video_id
=info_dict
['id'], ie
=info_dict
['extractor'])
2166 def report_force_conversion(field
, field_not
, conversion
):
2167 self
.report_warning(
2168 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
2169 % (field
, field_not
, conversion
))
2171 def sanitize_string_field(info
, string_field
):
2172 field
= info
.get(string_field
)
2173 if field
is None or isinstance(field
, compat_str
):
2175 report_force_conversion(string_field
, 'a string', 'string')
2176 info
[string_field
] = compat_str(field
)
2178 def sanitize_numeric_fields(info
):
2179 for numeric_field
in self
._NUMERIC
_FIELDS
:
2180 field
= info
.get(numeric_field
)
2181 if field
is None or isinstance(field
, compat_numeric_types
):
2183 report_force_conversion(numeric_field
, 'numeric', 'int')
2184 info
[numeric_field
] = int_or_none(field
)
2186 sanitize_string_field(info_dict
, 'id')
2187 sanitize_numeric_fields(info_dict
)
2189 if 'playlist' not in info_dict
:
2190 # It isn't part of a playlist
2191 info_dict
['playlist'] = None
2192 info_dict
['playlist_index'] = None
2194 self
._sanitize
_thumbnails
(info_dict
)
2196 thumbnail
= info_dict
.get('thumbnail')
2197 thumbnails
= info_dict
.get('thumbnails')
2199 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
2201 info_dict
['thumbnail'] = thumbnails
[-1]['url']
2203 if info_dict
.get('display_id') is None and 'id' in info_dict
:
2204 info_dict
['display_id'] = info_dict
['id']
2206 if info_dict
.get('duration') is not None:
2207 info_dict
['duration_string'] = formatSeconds(info_dict
['duration'])
2209 for ts_key
, date_key
in (
2210 ('timestamp', 'upload_date'),
2211 ('release_timestamp', 'release_date'),
2213 if info_dict
.get(date_key
) is None and info_dict
.get(ts_key
) is not None:
2214 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
2215 # see http://bugs.python.org/issue1646728)
2217 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
[ts_key
])
2218 info_dict
[date_key
] = upload_date
.strftime('%Y%m%d')
2219 except (ValueError, OverflowError, OSError):
2222 live_keys
= ('is_live', 'was_live')
2223 live_status
= info_dict
.get('live_status')
2224 if live_status
is None:
2225 for key
in live_keys
:
2226 if info_dict
.get(key
) is False:
2228 if info_dict
.get(key
):
2231 if all(info_dict
.get(key
) is False for key
in live_keys
):
2232 live_status
= 'not_live'
2234 info_dict
['live_status'] = live_status
2235 for key
in live_keys
:
2236 if info_dict
.get(key
) is None:
2237 info_dict
[key
] = (live_status
== key
)
2239 # Auto generate title fields corresponding to the *_number fields when missing
2240 # in order to always have clean titles. This is very common for TV series.
2241 for field
in ('chapter', 'season', 'episode'):
2242 if info_dict
.get('%s_number' % field
) is not None and not info_dict
.get(field
):
2243 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
['%s_number' % field
])
2245 for cc_kind
in ('subtitles', 'automatic_captions'):
2246 cc
= info_dict
.get(cc_kind
)
2248 for _
, subtitle
in cc
.items():
2249 for subtitle_format
in subtitle
:
2250 if subtitle_format
.get('url'):
2251 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
2252 if subtitle_format
.get('ext') is None:
2253 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
2255 automatic_captions
= info_dict
.get('automatic_captions')
2256 subtitles
= info_dict
.get('subtitles')
2258 info_dict
['requested_subtitles'] = self
.process_subtitles(
2259 info_dict
['id'], subtitles
, automatic_captions
)
2261 if info_dict
.get('formats') is None:
2262 # There's only one format available
2263 formats
= [info_dict
]
2265 formats
= info_dict
['formats']
2267 info_dict
['__has_drm'] = any(f
.get('has_drm') for f
in formats
)
2268 if not self
.params
.get('allow_unplayable_formats'):
2269 formats
= [f
for f
in formats
if not f
.get('has_drm')]
2272 self
.raise_no_formats(info_dict
)
2274 def is_wellformed(f
):
2277 self
.report_warning(
2278 '"url" field is missing or empty - skipping format, '
2279 'there is an error in extractor')
2281 if isinstance(url
, bytes):
2282 sanitize_string_field(f
, 'url')
2285 # Filter out malformed formats for better extraction robustness
2286 formats
= list(filter(is_wellformed
, formats
))
2290 # We check that all the formats have the format and format_id fields
2291 for i
, format
in enumerate(formats
):
2292 sanitize_string_field(format
, 'format_id')
2293 sanitize_numeric_fields(format
)
2294 format
['url'] = sanitize_url(format
['url'])
2295 if not format
.get('format_id'):
2296 format
['format_id'] = compat_str(i
)
2298 # Sanitize format_id from characters used in format selector expression
2299 format
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', format
['format_id'])
2300 format_id
= format
['format_id']
2301 if format_id
not in formats_dict
:
2302 formats_dict
[format_id
] = []
2303 formats_dict
[format_id
].append(format
)
2305 # Make sure all formats have unique format_id
2306 common_exts
= set(itertools
.chain(*self
._format
_selection
_exts
.values()))
2307 for format_id
, ambiguous_formats
in formats_dict
.items():
2308 ambigious_id
= len(ambiguous_formats
) > 1
2309 for i
, format
in enumerate(ambiguous_formats
):
2311 format
['format_id'] = '%s-%d' % (format_id
, i
)
2312 if format
.get('ext') is None:
2313 format
['ext'] = determine_ext(format
['url']).lower()
2314 # Ensure there is no conflict between id and ext in format selection
2315 # See https://github.com/yt-dlp/yt-dlp/issues/1282
2316 if format
['format_id'] != format
['ext'] and format
['format_id'] in common_exts
:
2317 format
['format_id'] = 'f%s' % format
['format_id']
2319 for i
, format
in enumerate(formats
):
2320 if format
.get('format') is None:
2321 format
['format'] = '{id} - {res}{note}'.format(
2322 id=format
['format_id'],
2323 res
=self
.format_resolution(format
),
2324 note
=format_field(format
, 'format_note', ' (%s)'),
2326 if format
.get('protocol') is None:
2327 format
['protocol'] = determine_protocol(format
)
2328 if format
.get('resolution') is None:
2329 format
['resolution'] = self
.format_resolution(format
, default
=None)
2330 if format
.get('dynamic_range') is None and format
.get('vcodec') != 'none':
2331 format
['dynamic_range'] = 'SDR'
2332 if (info_dict
.get('duration') and format
.get('tbr')
2333 and not format
.get('filesize') and not format
.get('filesize_approx')):
2334 format
['filesize_approx'] = info_dict
['duration'] * format
['tbr'] * (1024 / 8)
2336 # Add HTTP headers, so that external programs can use them from the
2338 full_format_info
= info_dict
.copy()
2339 full_format_info
.update(format
)
2340 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
2341 # Remove private housekeeping stuff
2342 if '__x_forwarded_for_ip' in info_dict
:
2343 del info_dict
['__x_forwarded_for_ip']
2345 # TODO Central sorting goes here
2347 if self
.params
.get('check_formats') is True:
2348 formats
= LazyList(self
._check
_formats
(formats
[::-1])).reverse()
2350 if not formats
or formats
[0] is not info_dict
:
2351 # only set the 'formats' fields if the original info_dict list them
2352 # otherwise we end up with a circular reference, the first (and unique)
2353 # element in the 'formats' field in info_dict is info_dict itself,
2354 # which can't be exported to json
2355 info_dict
['formats'] = formats
2357 info_dict
, _
= self
.pre_process(info_dict
)
2359 if self
.params
.get('list_thumbnails'):
2360 self
.list_thumbnails(info_dict
)
2361 if self
.params
.get('listformats'):
2362 if not info_dict
.get('formats') and not info_dict
.get('url'):
2363 self
.to_screen('%s has no formats' % info_dict
['id'])
2365 self
.list_formats(info_dict
)
2366 if self
.params
.get('listsubtitles'):
2367 if 'automatic_captions' in info_dict
:
2368 self
.list_subtitles(
2369 info_dict
['id'], automatic_captions
, 'automatic captions')
2370 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
2371 list_only
= self
.params
.get('simulate') is None and (
2372 self
.params
.get('list_thumbnails') or self
.params
.get('listformats') or self
.params
.get('listsubtitles'))
2374 # Without this printing, -F --print-json will not work
2375 self
.__forced
_printings
(info_dict
, self
.prepare_filename(info_dict
), incomplete
=True)
2378 format_selector
= self
.format_selector
2379 if format_selector
is None:
2380 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
2381 self
.write_debug('Default format spec: %s' % req_format
)
2382 format_selector
= self
.build_format_selector(req_format
)
2384 # While in format selection we may need to have an access to the original
2385 # format set in order to calculate some metrics or do some processing.
2386 # For now we need to be able to guess whether original formats provided
2387 # by extractor are incomplete or not (i.e. whether extractor provides only
2388 # video-only or audio-only formats) for proper formats selection for
2389 # extractors with such incomplete formats (see
2390 # https://github.com/ytdl-org/youtube-dl/pull/5556).
2391 # Since formats may be filtered during format selection and may not match
2392 # the original formats the results may be incorrect. Thus original formats
2393 # or pre-calculated metrics should be passed to format selection routines
2395 # We will pass a context object containing all necessary additional data
2396 # instead of just formats.
2397 # This fixes incorrect format selection issue (see
2398 # https://github.com/ytdl-org/youtube-dl/issues/10083).
2399 incomplete_formats
= (
2400 # All formats are video-only or
2401 all(f
.get('vcodec') != 'none' and f
.get('acodec') == 'none' for f
in formats
)
2402 # all formats are audio-only
2403 or all(f
.get('vcodec') == 'none' and f
.get('acodec') != 'none' for f
in formats
))
2407 'incomplete_formats': incomplete_formats
,
2410 formats_to_download
= list(format_selector(ctx
))
2411 if not formats_to_download
:
2412 if not self
.params
.get('ignore_no_formats_error'):
2413 raise ExtractorError('Requested format is not available', expected
=True,
2414 video_id
=info_dict
['id'], ie
=info_dict
['extractor'])
2416 self
.report_warning('Requested format is not available')
2417 # Process what we can, even without any available formats.
2418 self
.process_info(dict(info_dict
))
2421 '[info] %s: Downloading %d format(s): %s' % (
2422 info_dict
['id'], len(formats_to_download
),
2423 ", ".join([f
['format_id'] for f
in formats_to_download
])))
2424 for fmt
in formats_to_download
:
2425 new_info
= dict(info_dict
)
2426 # Save a reference to the original info_dict so that it can be modified in process_info if needed
2427 new_info
['__original_infodict'] = info_dict
2428 new_info
.update(fmt
)
2429 self
.process_info(new_info
)
2430 # We update the info dict with the selected best quality format (backwards compatibility)
2431 if formats_to_download
:
2432 info_dict
.update(formats_to_download
[-1])
2435 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
2436 """Select the requested subtitles and their format"""
2438 if normal_subtitles
and self
.params
.get('writesubtitles'):
2439 available_subs
.update(normal_subtitles
)
2440 if automatic_captions
and self
.params
.get('writeautomaticsub'):
2441 for lang
, cap_info
in automatic_captions
.items():
2442 if lang
not in available_subs
:
2443 available_subs
[lang
] = cap_info
2445 if (not self
.params
.get('writesubtitles') and not
2446 self
.params
.get('writeautomaticsub') or not
2450 all_sub_langs
= available_subs
.keys()
2451 if self
.params
.get('allsubtitles', False):
2452 requested_langs
= all_sub_langs
2453 elif self
.params
.get('subtitleslangs', False):
2454 # A list is used so that the order of languages will be the same as
2455 # given in subtitleslangs. See https://github.com/yt-dlp/yt-dlp/issues/1041
2456 requested_langs
= []
2457 for lang_re
in self
.params
.get('subtitleslangs'):
2458 if lang_re
== 'all':
2459 requested_langs
.extend(all_sub_langs
)
2461 discard
= lang_re
[0] == '-'
2463 lang_re
= lang_re
[1:]
2464 current_langs
= filter(re
.compile(lang_re
+ '$').match
, all_sub_langs
)
2466 for lang
in current_langs
:
2467 while lang
in requested_langs
:
2468 requested_langs
.remove(lang
)
2470 requested_langs
.extend(current_langs
)
2471 requested_langs
= orderedSet(requested_langs
)
2472 elif 'en' in available_subs
:
2473 requested_langs
= ['en']
2475 requested_langs
= [list(all_sub_langs
)[0]]
2477 self
.write_debug('Downloading subtitles: %s' % ', '.join(requested_langs
))
2479 formats_query
= self
.params
.get('subtitlesformat', 'best')
2480 formats_preference
= formats_query
.split('/') if formats_query
else []
2482 for lang
in requested_langs
:
2483 formats
= available_subs
.get(lang
)
2485 self
.report_warning('%s subtitles not available for %s' % (lang
, video_id
))
2487 for ext
in formats_preference
:
2491 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
2497 self
.report_warning(
2498 'No subtitle format found matching "%s" for language %s, '
2499 'using %s' % (formats_query
, lang
, f
['ext']))
2503 def __forced_printings(self
, info_dict
, filename
, incomplete
):
2504 def print_mandatory(field
, actual_field
=None):
2505 if actual_field
is None:
2506 actual_field
= field
2507 if (self
.params
.get('force%s' % field
, False)
2508 and (not incomplete
or info_dict
.get(actual_field
) is not None)):
2509 self
.to_stdout(info_dict
[actual_field
])
2511 def print_optional(field
):
2512 if (self
.params
.get('force%s' % field
, False)
2513 and info_dict
.get(field
) is not None):
2514 self
.to_stdout(info_dict
[field
])
2516 info_dict
= info_dict
.copy()
2517 if filename
is not None:
2518 info_dict
['filename'] = filename
2519 if info_dict
.get('requested_formats') is not None:
2520 # For RTMP URLs, also include the playpath
2521 info_dict
['urls'] = '\n'.join(f
['url'] + f
.get('play_path', '') for f
in info_dict
['requested_formats'])
2522 elif 'url' in info_dict
:
2523 info_dict
['urls'] = info_dict
['url'] + info_dict
.get('play_path', '')
2525 if self
.params
.get('forceprint') or self
.params
.get('forcejson'):
2526 self
.post_extract(info_dict
)
2527 for tmpl
in self
.params
.get('forceprint', []):
2528 mobj
= re
.match(r
'\w+(=?)$', tmpl
)
2529 if mobj
and mobj
.group(1):
2530 tmpl
= f
'{tmpl[:-1]} = %({tmpl[:-1]})s'
2532 tmpl
= '%({})s'.format(tmpl
)
2533 self
.to_stdout(self
.evaluate_outtmpl(tmpl
, info_dict
))
2535 print_mandatory('title')
2536 print_mandatory('id')
2537 print_mandatory('url', 'urls')
2538 print_optional('thumbnail')
2539 print_optional('description')
2540 print_optional('filename')
2541 if self
.params
.get('forceduration') and info_dict
.get('duration') is not None:
2542 self
.to_stdout(formatSeconds(info_dict
['duration']))
2543 print_mandatory('format')
2545 if self
.params
.get('forcejson'):
2546 self
.to_stdout(json
.dumps(self
.sanitize_info(info_dict
)))
2548 def dl(self
, name
, info
, subtitle
=False, test
=False):
2549 if not info
.get('url'):
2550 self
.raise_no_formats(info
, True)
2553 verbose
= self
.params
.get('verbose')
2556 'quiet': self
.params
.get('quiet') or not verbose
,
2558 'noprogress': not verbose
,
2560 'skip_unavailable_fragments': False,
2561 'keep_fragments': False,
2563 '_no_ytdl_file': True,
2566 params
= self
.params
2567 fd
= get_suitable_downloader(info
, params
, to_stdout
=(name
== '-'))(self
, params
)
2569 for ph
in self
._progress
_hooks
:
2570 fd
.add_progress_hook(ph
)
2571 urls
= '", "'.join([f
['url'] for f
in info
.get('requested_formats', [])] or [info
['url']])
2572 self
.write_debug('Invoking downloader on "%s"' % urls
)
2574 new_info
= copy
.deepcopy(self
._copy
_infodict
(info
))
2575 if new_info
.get('http_headers') is None:
2576 new_info
['http_headers'] = self
._calc
_headers
(new_info
)
2577 return fd
.download(name
, new_info
, subtitle
)
2579 def process_info(self
, info_dict
):
2580 """Process a single resolved IE result."""
2582 assert info_dict
.get('_type', 'video') == 'video'
2584 max_downloads
= self
.params
.get('max_downloads')
2585 if max_downloads
is not None:
2586 if self
._num
_downloads
>= int(max_downloads
):
2587 raise MaxDownloadsReached()
2589 # TODO: backward compatibility, to be removed
2590 info_dict
['fulltitle'] = info_dict
['title']
2592 if 'format' not in info_dict
and 'ext' in info_dict
:
2593 info_dict
['format'] = info_dict
['ext']
2595 if self
._match
_entry
(info_dict
) is not None:
2598 self
.post_extract(info_dict
)
2599 self
._num
_downloads
+= 1
2601 # info_dict['_filename'] needs to be set for backward compatibility
2602 info_dict
['_filename'] = full_filename
= self
.prepare_filename(info_dict
, warn
=True)
2603 temp_filename
= self
.prepare_filename(info_dict
, 'temp')
2607 self
.__forced
_printings
(info_dict
, full_filename
, incomplete
=('format' not in info_dict
))
2609 if self
.params
.get('simulate'):
2610 if self
.params
.get('force_write_download_archive', False):
2611 self
.record_download_archive(info_dict
)
2612 # Do nothing else if in simulate mode
2615 if full_filename
is None:
2617 if not self
._ensure
_dir
_exists
(encodeFilename(full_filename
)):
2619 if not self
._ensure
_dir
_exists
(encodeFilename(temp_filename
)):
2622 if self
._write
_description
('video', info_dict
,
2623 self
.prepare_filename(info_dict
, 'description')) is None:
2626 sub_files
= self
._write
_subtitles
(info_dict
, temp_filename
)
2627 if sub_files
is None:
2629 files_to_move
.update(dict(sub_files
))
2631 thumb_files
= self
._write
_thumbnails
(
2632 'video', info_dict
, temp_filename
, self
.prepare_filename(info_dict
, 'thumbnail'))
2633 if thumb_files
is None:
2635 files_to_move
.update(dict(thumb_files
))
2637 infofn
= self
.prepare_filename(info_dict
, 'infojson')
2638 _infojson_written
= self
._write
_info
_json
('video', info_dict
, infofn
)
2639 if _infojson_written
:
2640 info_dict
['__infojson_filename'] = infofn
2641 elif _infojson_written
is None:
2644 # Note: Annotations are deprecated
2646 if self
.params
.get('writeannotations', False):
2647 annofn
= self
.prepare_filename(info_dict
, 'annotation')
2649 if not self
._ensure
_dir
_exists
(encodeFilename(annofn
)):
2651 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(annofn
)):
2652 self
.to_screen('[info] Video annotations are already present')
2653 elif not info_dict
.get('annotations'):
2654 self
.report_warning('There are no annotations to write.')
2657 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
2658 with io
.open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
2659 annofile
.write(info_dict
['annotations'])
2660 except (KeyError, TypeError):
2661 self
.report_warning('There are no annotations to write.')
2662 except (OSError, IOError):
2663 self
.report_error('Cannot write annotations file: ' + annofn
)
2666 # Write internet shortcut files
2667 def _write_link_file(link_type
):
2668 if 'webpage_url' not in info_dict
:
2669 self
.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information')
2671 linkfn
= replace_extension(self
.prepare_filename(info_dict
, 'link'), link_type
, info_dict
.get('ext'))
2672 if self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(linkfn
)):
2673 self
.to_screen(f
'[info] Internet shortcut (.{link_type}) is already present')
2676 self
.to_screen(f
'[info] Writing internet shortcut (.{link_type}) to: {linkfn}')
2677 with io
.open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8',
2678 newline
='\r\n' if link_type
== 'url' else '\n') as linkfile
:
2679 template_vars
= {'url': iri_to_uri(info_dict['webpage_url'])}
2680 if link_type
== 'desktop':
2681 template_vars
['filename'] = linkfn
[:-(len(link_type
) + 1)]
2682 linkfile
.write(LINK_TEMPLATES
[link_type
] % template_vars
)
2683 except (OSError, IOError):
2684 self
.report_error(f
'Cannot write internet shortcut {linkfn}')
2689 'url': self
.params
.get('writeurllink'),
2690 'webloc': self
.params
.get('writewebloclink'),
2691 'desktop': self
.params
.get('writedesktoplink'),
2693 if self
.params
.get('writelink'):
2694 link_type
= ('webloc' if sys
.platform
== 'darwin'
2695 else 'desktop' if sys
.platform
.startswith('linux')
2697 write_links
[link_type
] = True
2699 if any(should_write
and not _write_link_file(link_type
)
2700 for link_type
, should_write
in write_links
.items()):
2704 info_dict
, files_to_move
= self
.pre_process(info_dict
, 'before_dl', files_to_move
)
2705 except PostProcessingError
as err
:
2706 self
.report_error('Preprocessing: %s' % str(err
))
2709 must_record_download_archive
= False
2710 if self
.params
.get('skip_download', False):
2711 info_dict
['filepath'] = temp_filename
2712 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2713 info_dict
['__files_to_move'] = files_to_move
2714 info_dict
= self
.run_pp(MoveFilesAfterDownloadPP(self
, False), info_dict
)
2717 info_dict
.setdefault('__postprocessors', [])
2720 def existing_file(*filepaths
):
2721 ext
= info_dict
.get('ext')
2722 final_ext
= self
.params
.get('final_ext', ext
)
2724 for file in orderedSet(filepaths
):
2725 if final_ext
!= ext
:
2726 converted
= replace_extension(file, final_ext
, ext
)
2727 if os
.path
.exists(encodeFilename(converted
)):
2728 existing_files
.append(converted
)
2729 if os
.path
.exists(encodeFilename(file)):
2730 existing_files
.append(file)
2732 if not existing_files
or self
.params
.get('overwrites', False):
2733 for file in orderedSet(existing_files
):
2734 self
.report_file_delete(file)
2735 os
.remove(encodeFilename(file))
2738 info_dict
['ext'] = os
.path
.splitext(existing_files
[0])[1][1:]
2739 return existing_files
[0]
2742 if info_dict
.get('requested_formats') is not None:
2744 def compatible_formats(formats
):
2745 # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
2746 video_formats
= [format
for format
in formats
if format
.get('vcodec') != 'none']
2747 audio_formats
= [format
for format
in formats
if format
.get('acodec') != 'none']
2748 if len(video_formats
) > 2 or len(audio_formats
) > 2:
2752 exts
= set(format
.get('ext') for format
in formats
)
2754 set(('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma')),
2757 for ext_sets
in COMPATIBLE_EXTS
:
2758 if ext_sets
.issuperset(exts
):
2760 # TODO: Check acodec/vcodec
2763 requested_formats
= info_dict
['requested_formats']
2764 old_ext
= info_dict
['ext']
2765 if self
.params
.get('merge_output_format') is None:
2766 if not compatible_formats(requested_formats
):
2767 info_dict
['ext'] = 'mkv'
2768 self
.report_warning(
2769 'Requested formats are incompatible for merge and will be merged into mkv')
2770 if (info_dict
['ext'] == 'webm'
2771 and info_dict
.get('thumbnails')
2772 # check with type instead of pp_key, __name__, or isinstance
2773 # since we dont want any custom PPs to trigger this
2774 and any(type(pp
) == EmbedThumbnailPP
for pp
in self
._pps
['post_process'])):
2775 info_dict
['ext'] = 'mkv'
2776 self
.report_warning(
2777 'webm doesn\'t support embedding a thumbnail, mkv will be used')
2778 new_ext
= info_dict
['ext']
2780 def correct_ext(filename
, ext
=new_ext
):
2783 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
2785 os
.path
.splitext(filename
)[0]
2786 if filename_real_ext
in (old_ext
, new_ext
)
2788 return '%s.%s' % (filename_wo_ext
, ext
)
2790 # Ensure filename always has a correct extension for successful merge
2791 full_filename
= correct_ext(full_filename
)
2792 temp_filename
= correct_ext(temp_filename
)
2793 dl_filename
= existing_file(full_filename
, temp_filename
)
2794 info_dict
['__real_download'] = False
2796 if dl_filename
is not None:
2797 self
.report_file_already_downloaded(dl_filename
)
2798 elif get_suitable_downloader(info_dict
, self
.params
, to_stdout
=temp_filename
== '-'):
2799 info_dict
['url'] = '\n'.join(f
['url'] for f
in requested_formats
)
2800 success
, real_download
= self
.dl(temp_filename
, info_dict
)
2801 info_dict
['__real_download'] = real_download
2804 merger
= FFmpegMergerPP(self
)
2805 if self
.params
.get('allow_unplayable_formats'):
2806 self
.report_warning(
2807 'You have requested merging of multiple formats '
2808 'while also allowing unplayable formats to be downloaded. '
2809 'The formats won\'t be merged to prevent data corruption.')
2810 elif not merger
.available
:
2811 self
.report_warning(
2812 'You have requested merging of multiple formats but ffmpeg is not installed. '
2813 'The formats won\'t be merged.')
2815 if temp_filename
== '-':
2816 reason
= ('using a downloader other than ffmpeg' if FFmpegFD
.can_merge_formats(info_dict
)
2817 else 'but the formats are incompatible for simultaneous download' if merger
.available
2818 else 'but ffmpeg is not installed')
2819 self
.report_warning(
2820 f
'You have requested downloading multiple formats to stdout {reason}. '
2821 'The formats will be streamed one after the other')
2822 fname
= temp_filename
2823 for f
in requested_formats
:
2824 new_info
= dict(info_dict
)
2825 del new_info
['requested_formats']
2827 if temp_filename
!= '-':
2828 fname
= prepend_extension(
2829 correct_ext(temp_filename
, new_info
['ext']),
2830 'f%s' % f
['format_id'], new_info
['ext'])
2831 if not self
._ensure
_dir
_exists
(fname
):
2833 f
['filepath'] = fname
2834 downloaded
.append(fname
)
2835 partial_success
, real_download
= self
.dl(fname
, new_info
)
2836 info_dict
['__real_download'] = info_dict
['__real_download'] or real_download
2837 success
= success
and partial_success
2838 if merger
.available
and not self
.params
.get('allow_unplayable_formats'):
2839 info_dict
['__postprocessors'].append(merger
)
2840 info_dict
['__files_to_merge'] = downloaded
2841 # Even if there were no downloads, it is being merged only now
2842 info_dict
['__real_download'] = True
2844 for file in downloaded
:
2845 files_to_move
[file] = None
2847 # Just a single file
2848 dl_filename
= existing_file(full_filename
, temp_filename
)
2849 if dl_filename
is None or dl_filename
== temp_filename
:
2850 # dl_filename == temp_filename could mean that the file was partially downloaded with --no-part.
2851 # So we should try to resume the download
2852 success
, real_download
= self
.dl(temp_filename
, info_dict
)
2853 info_dict
['__real_download'] = real_download
2855 self
.report_file_already_downloaded(dl_filename
)
2857 dl_filename
= dl_filename
or temp_filename
2858 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2860 except network_exceptions
as err
:
2861 self
.report_error('unable to download video data: %s' % error_to_compat_str(err
))
2863 except (OSError, IOError) as err
:
2864 raise UnavailableVideoError(err
)
2865 except (ContentTooShortError
, ) as err
:
2866 self
.report_error('content too short (expected %s bytes and served %s)' % (err
.expected
, err
.downloaded
))
2869 if success
and full_filename
!= '-':
2873 fixup_policy
= self
.params
.get('fixup')
2874 vid
= info_dict
['id']
2876 if fixup_policy
in ('ignore', 'never'):
2878 elif fixup_policy
== 'warn':
2880 elif fixup_policy
!= 'force':
2881 assert fixup_policy
in ('detect_or_warn', None)
2882 if not info_dict
.get('__real_download'):
2885 def ffmpeg_fixup(cndn
, msg
, cls
):
2889 self
.report_warning(f
'{vid}: {msg}')
2893 info_dict
['__postprocessors'].append(pp
)
2895 self
.report_warning(f
'{vid}: {msg}. Install ffmpeg to fix this automatically')
2897 stretched_ratio
= info_dict
.get('stretched_ratio')
2899 stretched_ratio
not in (1, None),
2900 f
'Non-uniform pixel ratio {stretched_ratio}',
2901 FFmpegFixupStretchedPP
)
2904 (info_dict
.get('requested_formats') is None
2905 and info_dict
.get('container') == 'm4a_dash'
2906 and info_dict
.get('ext') == 'm4a'),
2907 'writing DASH m4a. Only some players support this container',
2910 downloader
= get_suitable_downloader(info_dict
, self
.params
) if 'protocol' in info_dict
else None
2911 downloader
= downloader
.__name
__ if downloader
else None
2912 ffmpeg_fixup(info_dict
.get('requested_formats') is None and downloader
== 'HlsFD',
2913 'malformed AAC bitstream detected', FFmpegFixupM3u8PP
)
2914 ffmpeg_fixup(downloader
== 'WebSocketFragmentFD', 'malformed timestamps detected', FFmpegFixupTimestampPP
)
2915 ffmpeg_fixup(downloader
== 'WebSocketFragmentFD', 'malformed duration detected', FFmpegFixupDurationPP
)
2919 info_dict
= self
.post_process(dl_filename
, info_dict
, files_to_move
)
2920 except PostProcessingError
as err
:
2921 self
.report_error('Postprocessing: %s' % str(err
))
2924 for ph
in self
._post
_hooks
:
2925 ph(info_dict
['filepath'])
2926 except Exception as err
:
2927 self
.report_error('post hooks: %s' % str(err
))
2929 must_record_download_archive
= True
2931 if must_record_download_archive
or self
.params
.get('force_write_download_archive', False):
2932 self
.record_download_archive(info_dict
)
2933 max_downloads
= self
.params
.get('max_downloads')
2934 if max_downloads
is not None and self
._num
_downloads
>= int(max_downloads
):
2935 raise MaxDownloadsReached()
2937 def download(self
, url_list
):
2938 """Download a given list of URLs."""
2939 outtmpl
= self
.outtmpl_dict
['default']
2940 if (len(url_list
) > 1
2942 and '%' not in outtmpl
2943 and self
.params
.get('max_downloads') != 1):
2944 raise SameFileError(outtmpl
)
2946 for url
in url_list
:
2948 # It also downloads the videos
2949 res
= self
.extract_info(
2950 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
2951 except UnavailableVideoError
:
2952 self
.report_error('unable to download video')
2953 except DownloadCancelled
as e
:
2954 self
.to_screen(f
'[info] {e.msg}')
2957 if self
.params
.get('dump_single_json', False):
2958 self
.post_extract(res
)
2959 self
.to_stdout(json
.dumps(self
.sanitize_info(res
)))
2961 return self
._download
_retcode
2963 def download_with_info_file(self
, info_filename
):
2964 with contextlib
.closing(fileinput
.FileInput(
2965 [info_filename
], mode
='r',
2966 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
2967 # FileInput doesn't have a read method, we can't call json.load
2968 info
= self
.sanitize_info(json
.loads('\n'.join(f
)), self
.params
.get('clean_infojson', True))
2970 self
.process_ie_result(info
, download
=True)
2971 except (DownloadError
, EntryNotInPlaylist
, ThrottledDownload
):
2972 webpage_url
= info
.get('webpage_url')
2973 if webpage_url
is not None:
2974 self
.report_warning('The info failed to download, trying with "%s"' % webpage_url
)
2975 return self
.download([webpage_url
])
2978 return self
._download
_retcode
2981 def sanitize_info(info_dict
, remove_private_keys
=False):
2982 ''' Sanitize the infodict for converting to json '''
2983 if info_dict
is None:
2985 info_dict
.setdefault('epoch', int(time
.time()))
2986 remove_keys
= {'__original_infodict'}
# Always remove this since this may contain a copy of the entire dict
2987 keep_keys
= ['_type'], # Always keep this to facilitate load-info-json
2988 if remove_private_keys
:
2990 'requested_formats', 'requested_subtitles', 'requested_entries',
2991 'filepath', 'entries', 'original_url', 'playlist_autonumber',
2993 empty_values
= (None, {}, [], set(), tuple())
2994 reject
= lambda k
, v
: k
not in keep_keys
and (
2995 k
.startswith('_') or k
in remove_keys
or v
in empty_values
)
2997 reject
= lambda k
, v
: k
in remove_keys
2998 filter_fn
= lambda obj
: (
2999 list(map(filter_fn
, obj
)) if isinstance(obj
, (LazyList
, list, tuple, set))
3000 else obj
if not isinstance(obj
, dict)
3001 else dict((k
, filter_fn(v
)) for k
, v
in obj
.items() if not reject(k
, v
)))
3002 return filter_fn(info_dict
)
3005 def filter_requested_info(info_dict
, actually_filter
=True):
3006 ''' Alias of sanitize_info for backward compatibility '''
3007 return YoutubeDL
.sanitize_info(info_dict
, actually_filter
)
3009 def run_pp(self
, pp
, infodict
):
3010 files_to_delete
= []
3011 if '__files_to_move' not in infodict
:
3012 infodict
['__files_to_move'] = {}
3014 files_to_delete
, infodict
= pp
.run(infodict
)
3015 except PostProcessingError
as e
:
3016 # Must be True and not 'only_download'
3017 if self
.params
.get('ignoreerrors') is True:
3018 self
.report_error(e
)
3022 if not files_to_delete
:
3024 if self
.params
.get('keepvideo', False):
3025 for f
in files_to_delete
:
3026 infodict
['__files_to_move'].setdefault(f
, '')
3028 for old_filename
in set(files_to_delete
):
3029 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
3031 os
.remove(encodeFilename(old_filename
))
3032 except (IOError, OSError):
3033 self
.report_warning('Unable to remove downloaded original file')
3034 if old_filename
in infodict
['__files_to_move']:
3035 del infodict
['__files_to_move'][old_filename
]
3039 def post_extract(info_dict
):
3040 def actual_post_extract(info_dict
):
3041 if info_dict
.get('_type') in ('playlist', 'multi_video'):
3042 for video_dict
in info_dict
.get('entries', {}):
3043 actual_post_extract(video_dict
or {})
3046 post_extractor
= info_dict
.get('__post_extractor') or (lambda: {})
3047 extra
= post_extractor().items()
3048 info_dict
.update(extra
)
3049 info_dict
.pop('__post_extractor', None)
3051 original_infodict
= info_dict
.get('__original_infodict') or {}
3052 original_infodict
.update(extra
)
3053 original_infodict
.pop('__post_extractor', None)
3055 actual_post_extract(info_dict
or {})
3057 def pre_process(self
, ie_info
, key
='pre_process', files_to_move
=None):
3058 info
= dict(ie_info
)
3059 info
['__files_to_move'] = files_to_move
or {}
3060 for pp
in self
._pps
[key
]:
3061 info
= self
.run_pp(pp
, info
)
3062 return info
, info
.pop('__files_to_move', None)
3064 def post_process(self
, filename
, ie_info
, files_to_move
=None):
3065 """Run all the postprocessors on the given file."""
3066 info
= dict(ie_info
)
3067 info
['filepath'] = filename
3068 info
['__files_to_move'] = files_to_move
or {}
3070 for pp
in ie_info
.get('__postprocessors', []) + self
._pps
['post_process']:
3071 info
= self
.run_pp(pp
, info
)
3072 info
= self
.run_pp(MoveFilesAfterDownloadPP(self
), info
)
3073 del info
['__files_to_move']
3074 for pp
in self
._pps
['after_move']:
3075 info
= self
.run_pp(pp
, info
)
3078 def _make_archive_id(self
, info_dict
):
3079 video_id
= info_dict
.get('id')
3082 # Future-proof against any change in case
3083 # and backwards compatibility with prior versions
3084 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
3085 if extractor
is None:
3086 url
= str_or_none(info_dict
.get('url'))
3089 # Try to find matching extractor for the URL and take its ie_key
3090 for ie_key
, ie
in self
._ies
.items():
3091 if ie
.suitable(url
):
3096 return '%s %s' % (extractor
.lower(), video_id
)
3098 def in_download_archive(self
, info_dict
):
3099 fn
= self
.params
.get('download_archive')
3103 vid_id
= self
._make
_archive
_id
(info_dict
)
3105 return False # Incomplete video information
3107 return vid_id
in self
.archive
3109 def record_download_archive(self
, info_dict
):
3110 fn
= self
.params
.get('download_archive')
3113 vid_id
= self
._make
_archive
_id
(info_dict
)
3115 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
3116 archive_file
.write(vid_id
+ '\n')
3117 self
.archive
.add(vid_id
)
3120 def format_resolution(format
, default
='unknown'):
3121 is_images
= format
.get('vcodec') == 'none' and format
.get('acodec') == 'none'
3122 if format
.get('vcodec') == 'none' and format
.get('acodec') != 'none':
3124 if format
.get('resolution') is not None:
3125 return format
['resolution']
3126 if format
.get('width') and format
.get('height'):
3127 res
= '%dx%d' % (format
['width'], format
['height'])
3128 elif format
.get('height'):
3129 res
= '%sp' % format
['height']
3130 elif format
.get('width'):
3131 res
= '%dx?' % format
['width']
3136 return f
'{res} images' if is_images
else res
3138 def _format_note(self
, fdict
):
3140 if fdict
.get('ext') in ['f4f', 'f4m']:
3141 res
+= '(unsupported) '
3142 if fdict
.get('language'):
3145 res
+= '[%s] ' % fdict
['language']
3146 if fdict
.get('format_note') is not None:
3147 res
+= fdict
['format_note'] + ' '
3148 if fdict
.get('tbr') is not None:
3149 res
+= '%4dk ' % fdict
['tbr']
3150 if fdict
.get('container') is not None:
3153 res
+= '%s container' % fdict
['container']
3154 if (fdict
.get('vcodec') is not None
3155 and fdict
.get('vcodec') != 'none'):
3158 res
+= fdict
['vcodec']
3159 if fdict
.get('vbr') is not None:
3161 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
3163 if fdict
.get('vbr') is not None:
3164 res
+= '%4dk' % fdict
['vbr']
3165 if fdict
.get('fps') is not None:
3168 res
+= '%sfps' % fdict
['fps']
3169 if fdict
.get('acodec') is not None:
3172 if fdict
['acodec'] == 'none':
3175 res
+= '%-5s' % fdict
['acodec']
3176 elif fdict
.get('abr') is not None:
3180 if fdict
.get('abr') is not None:
3181 res
+= '@%3dk' % fdict
['abr']
3182 if fdict
.get('asr') is not None:
3183 res
+= ' (%5dHz)' % fdict
['asr']
3184 if fdict
.get('filesize') is not None:
3187 res
+= format_bytes(fdict
['filesize'])
3188 elif fdict
.get('filesize_approx') is not None:
3191 res
+= '~' + format_bytes(fdict
['filesize_approx'])
3194 def _list_format_headers(self
, *headers
):
3195 if self
.params
.get('listformats_table', True) is not False:
3196 return [self
._format
_screen
(header
, self
.Styles
.HEADERS
) for header
in headers
]
3199 def list_formats(self
, info_dict
):
3200 formats
= info_dict
.get('formats', [info_dict
])
3201 new_format
= self
.params
.get('listformats_table', True) is not False
3203 tbr_digits
= number_of_digits(max(f
.get('tbr') or 0 for f
in formats
))
3204 vbr_digits
= number_of_digits(max(f
.get('vbr') or 0 for f
in formats
))
3205 abr_digits
= number_of_digits(max(f
.get('abr') or 0 for f
in formats
))
3206 delim
= self
._format
_screen
('\u2502', self
.Styles
.DELIM
, '|', test_encoding
=True)
3209 self
._format
_screen
(format_field(f
, 'format_id'), self
.Styles
.ID
),
3210 format_field(f
, 'ext'),
3211 self
.format_resolution(f
),
3212 format_field(f
, 'fps', '%3d'),
3213 format_field(f
, 'dynamic_range', '%s', ignore
=(None, 'SDR')).replace('HDR', ''),
3215 format_field(f
, 'filesize', ' %s', func
=format_bytes
) + format_field(f
, 'filesize_approx', '~%s', func
=format_bytes
),
3216 format_field(f
, 'tbr', f
'%{tbr_digits}dk'),
3217 shorten_protocol_name(f
.get('protocol', '').replace("native", "n")),
3219 format_field(f
, 'vcodec', default
='unknown').replace('none', ''),
3220 format_field(f
, 'vbr', f
'%{vbr_digits}dk'),
3221 format_field(f
, 'acodec', default
='unknown').replace('none', ''),
3222 format_field(f
, 'abr', f
'%{abr_digits}dk'),
3223 format_field(f
, 'asr', '%5dHz'),
3224 ', '.join(filter(None, (
3225 self
._format
_screen
('UNSUPPORTED', 'light red') if f
.get('ext') in ('f4f', 'f4m') else '',
3226 format_field(f
, 'language', '[%s]'),
3227 format_field(f
, 'format_note'),
3228 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
3230 ] for f
in formats
if f
.get('preference') is None or f
['preference'] >= -1000]
3231 header_line
= self
._list
_format
_headers
(
3232 'ID', 'EXT', 'RESOLUTION', 'FPS', 'HDR', delim
, ' FILESIZE', ' TBR', 'PROTO',
3233 delim
, 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'MORE INFO')
3237 format_field(f
, 'format_id'),
3238 format_field(f
, 'ext'),
3239 self
.format_resolution(f
),
3240 self
._format
_note
(f
)]
3242 if f
.get('preference') is None or f
['preference'] >= -1000]
3243 header_line
= ['format code', 'extension', 'resolution', 'note']
3246 '[info] Available formats for %s:' % info_dict
['id'])
3247 self
.to_stdout(render_table(
3249 extraGap
=(0 if new_format
else 1),
3250 hideEmpty
=new_format
,
3251 delim
=new_format
and self
._format
_screen
('\u2500', self
.Styles
.DELIM
, '-', test_encoding
=True)))
3253 def list_thumbnails(self
, info_dict
):
3254 thumbnails
= list(info_dict
.get('thumbnails'))
3256 self
.to_screen('[info] No thumbnails present for %s' % info_dict
['id'])
3260 '[info] Thumbnails for %s:' % info_dict
['id'])
3261 self
.to_stdout(render_table(
3262 self
._list
_format
_headers
('ID', 'Width', 'Height', 'URL'),
3263 [[t
['id'], t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
]))
3265 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
3267 self
.to_screen('%s has no %s' % (video_id
, name
))
3270 'Available %s for %s:' % (name
, video_id
))
3272 def _row(lang
, formats
):
3273 exts
, names
= zip(*((f
['ext'], f
.get('name') or 'unknown') for f
in reversed(formats
)))
3274 if len(set(names
)) == 1:
3275 names
= [] if names
[0] == 'unknown' else names
[:1]
3276 return [lang
, ', '.join(names
), ', '.join(exts
)]
3278 self
.to_stdout(render_table(
3279 self
._list
_format
_headers
('Language', 'Name', 'Formats'),
3280 [_row(lang
, formats
) for lang
, formats
in subtitles
.items()],
3283 def urlopen(self
, req
):
3284 """ Start an HTTP download """
3285 if isinstance(req
, compat_basestring
):
3286 req
= sanitized_Request(req
)
3287 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
3289 def print_debug_header(self
):
3290 if not self
.params
.get('verbose'):
3293 def get_encoding(stream
):
3294 ret
= getattr(stream
, 'encoding', 'missing (%s)' % type(stream
).__name
__)
3295 if not supports_terminal_sequences(stream
):
3299 encoding_str
= 'Encodings: locale %s, fs %s, out %s, err %s, pref %s' % (
3300 locale
.getpreferredencoding(),
3301 sys
.getfilesystemencoding(),
3302 get_encoding(self
._screen
_file
), get_encoding(self
._err
_file
),
3303 self
.get_encoding())
3305 logger
= self
.params
.get('logger')
3307 write_debug
= lambda msg
: logger
.debug(f
'[debug] {msg}')
3308 write_debug(encoding_str
)
3310 write_string(f
'[debug] {encoding_str}\n', encoding
=None)
3311 write_debug
= lambda msg
: self
._write
_string
(f
'[debug] {msg}\n')
3313 source
= detect_variant()
3314 write_debug('yt-dlp version %s%s' % (__version__
, '' if source
== 'unknown' else f
' ({source})'))
3315 if not _LAZY_LOADER
:
3316 if os
.environ
.get('YTDLP_NO_LAZY_EXTRACTORS'):
3317 write_debug('Lazy loading extractors is forcibly disabled')
3319 write_debug('Lazy loading extractors is disabled')
3320 if plugin_extractors
or plugin_postprocessors
:
3321 write_debug('Plugins: %s' % [
3322 '%s%s' % (klass
.__name
__, '' if klass
.__name
__ == name
else f
' as {name}')
3323 for name
, klass
in itertools
.chain(plugin_extractors
.items(), plugin_postprocessors
.items())])
3324 if self
.params
.get('compat_opts'):
3325 write_debug('Compatibility options: %s' % ', '.join(self
.params
.get('compat_opts')))
3328 ['git', 'rev-parse', '--short', 'HEAD'],
3329 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
3330 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
3331 out
, err
= sp
.communicate_or_kill()
3332 out
= out
.decode().strip()
3333 if re
.match('[0-9a-f]+', out
):
3334 write_debug('Git HEAD: %s' % out
)
3341 def python_implementation():
3342 impl_name
= platform
.python_implementation()
3343 if impl_name
== 'PyPy' and hasattr(sys
, 'pypy_version_info'):
3344 return impl_name
+ ' version %d.%d.%d' % sys
.pypy_version_info
[:3]
3347 write_debug('Python version %s (%s %s) - %s' % (
3348 platform
.python_version(),
3349 python_implementation(),
3350 platform
.architecture()[0],
3353 exe_versions
, ffmpeg_features
= FFmpegPostProcessor
.get_versions_and_features(self
)
3354 ffmpeg_features
= {key for key, val in ffmpeg_features.items() if val}
3356 exe_versions
['ffmpeg'] += ' (%s)' % ','.join(ffmpeg_features
)
3358 exe_versions
['rtmpdump'] = rtmpdump_version()
3359 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
3360 exe_str
= ', '.join(
3361 f
'{exe} {v}' for exe
, v
in sorted(exe_versions
.items()) if v
3363 write_debug('exe versions: %s' % exe_str
)
3365 from .downloader
.websocket
import has_websockets
3366 from .postprocessor
.embedthumbnail
import has_mutagen
3367 from .cookies
import SQLITE_AVAILABLE
, KEYRING_AVAILABLE
3369 lib_str
= ', '.join(sorted(filter(None, (
3370 compat_pycrypto_AES
and compat_pycrypto_AES
.__name
__.split('.')[0],
3371 has_websockets
and 'websockets',
3372 has_mutagen
and 'mutagen',
3373 SQLITE_AVAILABLE
and 'sqlite',
3374 KEYRING_AVAILABLE
and 'keyring',
3376 write_debug('Optional libraries: %s' % lib_str
)
3379 for handler
in self
._opener
.handlers
:
3380 if hasattr(handler
, 'proxies'):
3381 proxy_map
.update(handler
.proxies
)
3382 write_debug(f
'Proxy map: {proxy_map}')
3385 if False and self
.params
.get('call_home'):
3386 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
3387 write_debug('Public IP address: %s' % ipaddr
)
3388 latest_version
= self
.urlopen(
3389 'https://yt-dl.org/latest/version').read().decode('utf-8')
3390 if version_tuple(latest_version
) > version_tuple(__version__
):
3391 self
.report_warning(
3392 'You are using an outdated version (newest version: %s)! '
3393 'See https://yt-dl.org/update if you need help updating.' %
3396 def _setup_opener(self
):
3397 timeout_val
= self
.params
.get('socket_timeout')
3398 self
._socket
_timeout
= 20 if timeout_val
is None else float(timeout_val
)
3400 opts_cookiesfrombrowser
= self
.params
.get('cookiesfrombrowser')
3401 opts_cookiefile
= self
.params
.get('cookiefile')
3402 opts_proxy
= self
.params
.get('proxy')
3404 self
.cookiejar
= load_cookies(opts_cookiefile
, opts_cookiesfrombrowser
, self
)
3406 cookie_processor
= YoutubeDLCookieProcessor(self
.cookiejar
)
3407 if opts_proxy
is not None:
3408 if opts_proxy
== '':
3411 proxies
= {'http': opts_proxy, 'https': opts_proxy}
3413 proxies
= compat_urllib_request
.getproxies()
3414 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
3415 if 'http' in proxies
and 'https' not in proxies
:
3416 proxies
['https'] = proxies
['http']
3417 proxy_handler
= PerRequestProxyHandler(proxies
)
3419 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
3420 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
3421 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
3422 redirect_handler
= YoutubeDLRedirectHandler()
3423 data_handler
= compat_urllib_request_DataHandler()
3425 # When passing our own FileHandler instance, build_opener won't add the
3426 # default FileHandler and allows us to disable the file protocol, which
3427 # can be used for malicious purposes (see
3428 # https://github.com/ytdl-org/youtube-dl/issues/8227)
3429 file_handler
= compat_urllib_request
.FileHandler()
3431 def file_open(*args
, **kwargs
):
3432 raise compat_urllib_error
.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
3433 file_handler
.file_open
= file_open
3435 opener
= compat_urllib_request
.build_opener(
3436 proxy_handler
, https_handler
, cookie_processor
, ydlh
, redirect_handler
, data_handler
, file_handler
)
3438 # Delete the default user-agent header, which would otherwise apply in
3439 # cases where our custom HTTP handler doesn't come into play
3440 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
3441 opener
.addheaders
= []
3442 self
._opener
= opener
3444 def encode(self
, s
):
3445 if isinstance(s
, bytes):
3446 return s
# Already encoded
3449 return s
.encode(self
.get_encoding())
3450 except UnicodeEncodeError as err
:
3451 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
3454 def get_encoding(self
):
3455 encoding
= self
.params
.get('encoding')
3456 if encoding
is None:
3457 encoding
= preferredencoding()
3460 def _write_info_json(self
, label
, ie_result
, infofn
):
3461 ''' Write infojson and returns True = written, False = skip, None = error '''
3462 if not self
.params
.get('writeinfojson'):
3465 self
.write_debug(f
'Skipping writing {label} infojson')
3467 elif not self
._ensure
_dir
_exists
(infofn
):
3469 elif not self
.params
.get('overwrites', True) and os
.path
.exists(infofn
):
3470 self
.to_screen(f
'[info] {label.title()} metadata is already present')
3472 self
.to_screen(f
'[info] Writing {label} metadata as JSON to: {infofn}')
3474 write_json_file(self
.sanitize_info(ie_result
, self
.params
.get('clean_infojson', True)), infofn
)
3475 except (OSError, IOError):
3476 self
.report_error(f
'Cannot write {label} metadata to JSON file {infofn}')
3480 def _write_description(self
, label
, ie_result
, descfn
):
3481 ''' Write description and returns True = written, False = skip, None = error '''
3482 if not self
.params
.get('writedescription'):
3485 self
.write_debug(f
'Skipping writing {label} description')
3487 elif not self
._ensure
_dir
_exists
(descfn
):
3489 elif not self
.params
.get('overwrites', True) and os
.path
.exists(descfn
):
3490 self
.to_screen(f
'[info] {label.title()} description is already present')
3491 elif ie_result
.get('description') is None:
3492 self
.report_warning(f
'There\'s no {label} description to write')
3496 self
.to_screen(f
'[info] Writing {label} description to: {descfn}')
3497 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
3498 descfile
.write(ie_result
['description'])
3499 except (OSError, IOError):
3500 self
.report_error(f
'Cannot write {label} description file {descfn}')
3504 def _write_subtitles(self
, info_dict
, filename
):
3505 ''' Write subtitles to file and return list of (sub_filename, final_sub_filename); or None if error'''
3507 subtitles
= info_dict
.get('requested_subtitles')
3508 if not subtitles
or not (self
.params
.get('writesubtitles') or self
.params
.get('writeautomaticsub')):
3509 # subtitles download errors are already managed as troubles in relevant IE
3510 # that way it will silently go on when used with unsupporting IE
3513 sub_filename_base
= self
.prepare_filename(info_dict
, 'subtitle')
3514 if not sub_filename_base
:
3515 self
.to_screen('[info] Skipping writing video subtitles')
3517 for sub_lang
, sub_info
in subtitles
.items():
3518 sub_format
= sub_info
['ext']
3519 sub_filename
= subtitles_filename(filename
, sub_lang
, sub_format
, info_dict
.get('ext'))
3520 sub_filename_final
= subtitles_filename(sub_filename_base
, sub_lang
, sub_format
, info_dict
.get('ext'))
3521 if not self
.params
.get('overwrites', True) and os
.path
.exists(sub_filename
):
3522 self
.to_screen(f
'[info] Video subtitle {sub_lang}.{sub_format} is already present')
3523 sub_info
['filepath'] = sub_filename
3524 ret
.append((sub_filename
, sub_filename_final
))
3527 self
.to_screen(f
'[info] Writing video subtitles to: {sub_filename}')
3528 if sub_info
.get('data') is not None:
3530 # Use newline='' to prevent conversion of newline characters
3531 # See https://github.com/ytdl-org/youtube-dl/issues/10268
3532 with io
.open(sub_filename
, 'w', encoding
='utf-8', newline
='') as subfile
:
3533 subfile
.write(sub_info
['data'])
3534 sub_info
['filepath'] = sub_filename
3535 ret
.append((sub_filename
, sub_filename_final
))
3537 except (OSError, IOError):
3538 self
.report_error(f
'Cannot write video subtitles file {sub_filename}')
3542 sub_copy
= sub_info
.copy()
3543 sub_copy
.setdefault('http_headers', info_dict
.get('http_headers'))
3544 self
.dl(sub_filename
, sub_copy
, subtitle
=True)
3545 sub_info
['filepath'] = sub_filename
3546 ret
.append((sub_filename
, sub_filename_final
))
3547 except (ExtractorError
, IOError, OSError, ValueError) + network_exceptions
as err
:
3548 self
.report_warning(f
'Unable to download video subtitles for {sub_lang!r}: {err}')
3552 def _write_thumbnails(self
, label
, info_dict
, filename
, thumb_filename_base
=None):
3553 ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename) '''
3554 write_all
= self
.params
.get('write_all_thumbnails', False)
3555 thumbnails
, ret
= [], []
3556 if write_all
or self
.params
.get('writethumbnail', False):
3557 thumbnails
= info_dict
.get('thumbnails') or []
3558 multiple
= write_all
and len(thumbnails
) > 1
3560 if thumb_filename_base
is None:
3561 thumb_filename_base
= filename
3562 if thumbnails
and not thumb_filename_base
:
3563 self
.write_debug(f
'Skipping writing {label} thumbnail')
3566 for t
in thumbnails
[::-1]:
3567 thumb_ext
= (f
'{t["id"]}.' if multiple
else '') + determine_ext(t
['url'], 'jpg')
3568 thumb_display_id
= f
'{label} thumbnail' + (f
' {t["id"]}' if multiple
else '')
3569 thumb_filename
= replace_extension(filename
, thumb_ext
, info_dict
.get('ext'))
3570 thumb_filename_final
= replace_extension(thumb_filename_base
, thumb_ext
, info_dict
.get('ext'))
3572 if not self
.params
.get('overwrites', True) and os
.path
.exists(thumb_filename
):
3573 ret
.append((thumb_filename
, thumb_filename_final
))
3574 t
['filepath'] = thumb_filename
3575 self
.to_screen(f
'[info] {thumb_display_id.title()} is already present')
3577 self
.to_screen(f
'[info] Downloading {thumb_display_id} ...')
3579 uf
= self
.urlopen(t
['url'])
3580 self
.to_screen(f
'[info] Writing {thumb_display_id} to: {thumb_filename}')
3581 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
3582 shutil
.copyfileobj(uf
, thumbf
)
3583 ret
.append((thumb_filename
, thumb_filename_final
))
3584 t
['filepath'] = thumb_filename
3585 except network_exceptions
as err
:
3586 self
.report_warning(f
'Unable to download {thumb_display_id}: {err}')
3587 if ret
and not write_all
: