27 from .cache
import Cache
28 from .compat
import urllib
# isort: split
29 from .compat
import compat_os_name
, urllib_req_to_req
30 from .cookies
import LenientSimpleCookie
, load_cookies
31 from .downloader
import FFmpegFD
, get_suitable_downloader
, shorten_protocol_name
32 from .downloader
.rtmp
import rtmpdump_version
33 from .extractor
import gen_extractor_classes
, get_info_extractor
34 from .extractor
.common
import UnsupportedURLIE
35 from .extractor
.openload
import PhantomJSwrapper
36 from .minicurses
import format_text
37 from .networking
import HEADRequest
, Request
, RequestDirector
38 from .networking
.common
import _REQUEST_HANDLERS
, _RH_PREFERENCES
39 from .networking
.exceptions
import (
46 from .networking
.impersonate
import ImpersonateRequestHandler
47 from .plugins
import directories
as plugin_directories
48 from .postprocessor
import _PLUGIN_CLASSES
as plugin_pps
49 from .postprocessor
import (
51 FFmpegFixupDuplicateMoovPP
,
52 FFmpegFixupDurationPP
,
55 FFmpegFixupStretchedPP
,
56 FFmpegFixupTimestampPP
,
59 FFmpegVideoConvertorPP
,
60 MoveFilesAfterDownloadPP
,
63 from .postprocessor
.ffmpeg
import resolve_mapping
as resolve_recode_mapping
66 _get_system_deprecation
,
100 RejectedVideoReached
,
102 UnavailableVideoError
,
119 format_decimal_suffix
,
133 orderedSet_from_options
,
137 remove_terminal_sequences
,
147 supports_terminal_sequences
,
158 windows_enable_vt_mode
,
162 from .utils
._utils
import _UnsafeExtensionError
, _YDLLogger
163 from .utils
.networking
import (
169 from .version
import CHANNEL
, ORIGIN
, RELEASE_GIT_HEAD
, VARIANT
, __version__
171 if compat_os_name
== 'nt':
175 def _catch_unsafe_extension_error(func
):
176 @functools.wraps(func
)
177 def wrapper(self
, *args
, **kwargs
):
179 return func(self
, *args
, **kwargs
)
180 except _UnsafeExtensionError
as error
:
182 f
'The extracted extension ({error.extension!r}) is unusual '
183 'and will be skipped for safety reasons. '
184 f
'If you believe this is an error{bug_reports_message(",")}')
192 YoutubeDL objects are the ones responsible of downloading the
193 actual video file and writing it to disk if the user has requested
194 it, among some other tasks. In most cases there should be one per
195 program. As, given a video URL, the downloader doesn't know how to
196 extract all the needed information, task that InfoExtractors do, it
197 has to pass the URL to one of them.
199 For this, YoutubeDL objects have a method that allows
200 InfoExtractors to be registered in a given order. When it is passed
201 a URL, the YoutubeDL object handles it to the first InfoExtractor it
202 finds that reports being able to handle it. The InfoExtractor extracts
203 all the information about the video or videos the URL refers to, and
204 YoutubeDL process the extracted information, possibly using a File
205 Downloader to download the video.
207 YoutubeDL objects accept a lot of parameters. In order not to saturate
208 the object constructor with arguments, it receives a dictionary of
209 options instead. These options are available through the params
210 attribute for the InfoExtractors to use. The YoutubeDL also
211 registers itself as the downloader in charge for the InfoExtractors
212 that are added to it, so this is a "mutual registration".
216 username: Username for authentication purposes.
217 password: Password for authentication purposes.
218 videopassword: Password for accessing a video.
219 ap_mso: Adobe Pass multiple-system operator identifier.
220 ap_username: Multiple-system operator account username.
221 ap_password: Multiple-system operator account password.
222 usenetrc: Use netrc for authentication instead.
223 netrc_location: Location of the netrc file. Defaults to ~/.netrc.
224 netrc_cmd: Use a shell command to get credentials
225 verbose: Print additional info to stdout.
226 quiet: Do not print messages to stdout.
227 no_warnings: Do not print out anything for warnings.
228 forceprint: A dict with keys WHEN mapped to a list of templates to
229 print to stdout. The allowed keys are video or any of the
230 items in utils.POSTPROCESS_WHEN.
231 For compatibility, a single list is also accepted
232 print_to_file: A dict with keys WHEN (same as forceprint) mapped to
233 a list of tuples with (template, filename)
234 forcejson: Force printing info_dict as JSON.
235 dump_single_json: Force printing the info_dict of the whole playlist
236 (or video) as a single JSON line.
237 force_write_download_archive: Force writing download archive regardless
238 of 'skip_download' or 'simulate'.
239 simulate: Do not download the video files. If unset (or None),
240 simulate only if listsubtitles, listformats or list_thumbnails is used
241 format: Video format code. see "FORMAT SELECTION" for more details.
242 You can also pass a function. The function takes 'ctx' as
243 argument and returns the formats to download.
244 See "build_format_selector" for an implementation
245 allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
246 ignore_no_formats_error: Ignore "No video formats" error. Usefull for
247 extracting metadata even if the video is not actually
248 available for download (experimental)
249 format_sort: A list of fields by which to sort the video formats.
250 See "Sorting Formats" for more details.
251 format_sort_force: Force the given format_sort. see "Sorting Formats"
253 prefer_free_formats: Whether to prefer video formats with free containers
254 over non-free ones of same quality.
255 allow_multiple_video_streams: Allow multiple video streams to be merged
257 allow_multiple_audio_streams: Allow multiple audio streams to be merged
259 check_formats Whether to test if the formats are downloadable.
260 Can be True (check all), False (check none),
261 'selected' (check selected formats),
262 or None (check only if requested by extractor)
263 paths: Dictionary of output paths. The allowed keys are 'home'
264 'temp' and the keys of OUTTMPL_TYPES (in utils/_utils.py)
265 outtmpl: Dictionary of templates for output names. Allowed keys
266 are 'default' and the keys of OUTTMPL_TYPES (in utils/_utils.py).
267 For compatibility with youtube-dl, a single string can also be used
268 outtmpl_na_placeholder: Placeholder for unavailable meta fields.
269 restrictfilenames: Do not allow "&" and spaces in file names
270 trim_file_name: Limit length of filename (extension excluded)
271 windowsfilenames: Force the filenames to be windows compatible
272 ignoreerrors: Do not stop on download/postprocessing errors.
273 Can be 'only_download' to ignore only download errors.
274 Default is 'only_download' for CLI, but False for API
275 skip_playlist_after_errors: Number of allowed failures until the rest of
276 the playlist is skipped
277 allowed_extractors: List of regexes to match against extractor names that are allowed
278 overwrites: Overwrite all video and metadata files if True,
279 overwrite only non-video files if None
280 and don't overwrite any file if False
281 playlist_items: Specific indices of playlist to download.
282 playlistrandom: Download playlist items in random order.
283 lazy_playlist: Process playlist entries as they are received.
284 matchtitle: Download only matching titles.
285 rejecttitle: Reject downloads for matching titles.
286 logger: Log messages to a logging.Logger instance.
287 logtostderr: Print everything to stderr instead of stdout.
288 consoletitle: Display progress in console window's titlebar.
289 writedescription: Write the video description to a .description file
290 writeinfojson: Write the video description to a .info.json file
291 clean_infojson: Remove internal metadata from the infojson
292 getcomments: Extract video comments. This will not be written to disk
293 unless writeinfojson is also given
294 writeannotations: Write the video annotations to a .annotations.xml file
295 writethumbnail: Write the thumbnail image to a file
296 allow_playlist_files: Whether to write playlists' description, infojson etc
297 also to disk when using the 'write*' options
298 write_all_thumbnails: Write all thumbnail formats to files
299 writelink: Write an internet shortcut file, depending on the
300 current platform (.url/.webloc/.desktop)
301 writeurllink: Write a Windows internet shortcut file (.url)
302 writewebloclink: Write a macOS internet shortcut file (.webloc)
303 writedesktoplink: Write a Linux internet shortcut file (.desktop)
304 writesubtitles: Write the video subtitles to a file
305 writeautomaticsub: Write the automatically generated subtitles to a file
306 listsubtitles: Lists all available subtitles for the video
307 subtitlesformat: The format code for subtitles
308 subtitleslangs: List of languages of the subtitles to download (can be regex).
309 The list may contain "all" to refer to all the available
310 subtitles. The language can be prefixed with a "-" to
311 exclude it from the requested languages, e.g. ['all', '-live_chat']
312 keepvideo: Keep the video file after post-processing
313 daterange: A utils.DateRange object, download only if the upload_date is in the range.
314 skip_download: Skip the actual download of the video file
315 cachedir: Location of the cache files in the filesystem.
316 False to disable filesystem cache.
317 noplaylist: Download single video instead of a playlist if in doubt.
318 age_limit: An integer representing the user's age in years.
319 Unsuitable videos for the given age are skipped.
320 min_views: An integer representing the minimum view count the video
321 must have in order to not be skipped.
322 Videos without view count information are always
323 downloaded. None for no limit.
324 max_views: An integer representing the maximum view count.
325 Videos that are more popular than that are not
327 Videos without view count information are always
328 downloaded. None for no limit.
329 download_archive: A set, or the name of a file where all downloads are recorded.
330 Videos already present in the file are not downloaded again.
331 break_on_existing: Stop the download process after attempting to download a
332 file that is in the archive.
333 break_per_url: Whether break_on_reject and break_on_existing
334 should act on each input URL as opposed to for the entire queue
335 cookiefile: File name or text stream from where cookies should be read and dumped to
336 cookiesfrombrowser: A tuple containing the name of the browser, the profile
337 name/path from where cookies are loaded, the name of the keyring,
338 and the container name, e.g. ('chrome', ) or
339 ('vivaldi', 'default', 'BASICTEXT') or ('firefox', 'default', None, 'Meta')
340 legacyserverconnect: Explicitly allow HTTPS connection to servers that do not
341 support RFC 5746 secure renegotiation
342 nocheckcertificate: Do not verify SSL certificates
343 client_certificate: Path to client certificate file in PEM format. May include the private key
344 client_certificate_key: Path to private key file for client certificate
345 client_certificate_password: Password for client certificate private key, if encrypted.
346 If not provided and the key is encrypted, yt-dlp will ask interactively
347 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
348 (Only supported by some extractors)
349 enable_file_urls: Enable file:// URLs. This is disabled by default for security reasons.
350 http_headers: A dictionary of custom headers to be used for all requests
351 proxy: URL of the proxy server to use
352 geo_verification_proxy: URL of the proxy to use for IP address verification
353 on geo-restricted sites.
354 socket_timeout: Time to wait for unresponsive hosts, in seconds
355 bidi_workaround: Work around buggy terminals without bidirectional text
356 support, using fridibi
357 debug_printtraffic:Print out sent and received HTTP traffic
358 default_search: Prepend this string if an input url is not valid.
359 'auto' for elaborate guessing
360 encoding: Use this encoding instead of the system-specified.
361 extract_flat: Whether to resolve and process url_results further
362 * False: Always process. Default for API
363 * True: Never process
364 * 'in_playlist': Do not process inside playlist/multi_video
365 * 'discard': Always process, but don't return the result
366 from inside playlist/multi_video
367 * 'discard_in_playlist': Same as "discard", but only for
368 playlists (not multi_video). Default for CLI
369 wait_for_video: If given, wait for scheduled streams to become available.
370 The value should be a tuple containing the range
371 (min_secs, max_secs) to wait between retries
372 postprocessors: A list of dictionaries, each with an entry
373 * key: The name of the postprocessor. See
374 yt_dlp/postprocessor/__init__.py for a list.
375 * when: When to run the postprocessor. Allowed values are
376 the entries of utils.POSTPROCESS_WHEN
377 Assumed to be 'post_process' if not given
378 progress_hooks: A list of functions that get called on download
379 progress, with a dictionary with the entries
380 * status: One of "downloading", "error", or "finished".
381 Check this first and ignore unknown values.
382 * info_dict: The extracted info_dict
384 If status is one of "downloading", or "finished", the
385 following properties may also be present:
386 * filename: The final filename (always present)
387 * tmpfilename: The filename we're currently writing to
388 * downloaded_bytes: Bytes on disk
389 * total_bytes: Size of the whole file, None if unknown
390 * total_bytes_estimate: Guess of the eventual file size,
392 * elapsed: The number of seconds since download started.
393 * eta: The estimated time in seconds, None if unknown
394 * speed: The download speed in bytes/second, None if
396 * fragment_index: The counter of the currently
397 downloaded video fragment.
398 * fragment_count: The number of fragments (= individual
399 files that will be merged)
401 Progress hooks are guaranteed to be called at least once
402 (with status "finished") if the download is successful.
403 postprocessor_hooks: A list of functions that get called on postprocessing
404 progress, with a dictionary with the entries
405 * status: One of "started", "processing", or "finished".
406 Check this first and ignore unknown values.
407 * postprocessor: Name of the postprocessor
408 * info_dict: The extracted info_dict
410 Progress hooks are guaranteed to be called at least twice
411 (with status "started" and "finished") if the processing is successful.
412 merge_output_format: "/" separated list of extensions to use when merging formats.
413 final_ext: Expected final extension; used to detect when the file was
414 already downloaded and converted
415 fixup: Automatically correct known faults of the file.
417 - "never": do nothing
418 - "warn": only emit a warning
419 - "detect_or_warn": check whether we can do anything
420 about it, warn otherwise (default)
421 source_address: Client-side IP address to bind to.
422 impersonate: Client to impersonate for requests.
423 An ImpersonateTarget (from yt_dlp.networking.impersonate)
424 sleep_interval_requests: Number of seconds to sleep between requests
426 sleep_interval: Number of seconds to sleep before each download when
427 used alone or a lower bound of a range for randomized
428 sleep before each download (minimum possible number
429 of seconds to sleep) when used along with
431 max_sleep_interval:Upper bound of a range for randomized sleep before each
432 download (maximum possible number of seconds to sleep).
433 Must only be used along with sleep_interval.
434 Actual sleep time will be a random float from range
435 [sleep_interval; max_sleep_interval].
436 sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
437 listformats: Print an overview of available video formats and exit.
438 list_thumbnails: Print a table of all thumbnails and exit.
439 match_filter: A function that gets called for every video with the signature
440 (info_dict, *, incomplete: bool) -> Optional[str]
441 For backward compatibility with youtube-dl, the signature
442 (info_dict) -> Optional[str] is also allowed.
443 - If it returns a message, the video is ignored.
444 - If it returns None, the video is downloaded.
445 - If it returns utils.NO_DEFAULT, the user is interactively
446 asked whether to download the video.
447 - Raise utils.DownloadCancelled(msg) to abort remaining
448 downloads when a video is rejected.
449 match_filter_func in utils/_utils.py is one example for this.
450 color: A Dictionary with output stream names as keys
451 and their respective color policy as values.
452 Can also just be a single color policy,
453 in which case it applies to all outputs.
454 Valid stream names are 'stdout' and 'stderr'.
455 Valid color policies are one of 'always', 'auto', 'no_color' or 'never'.
456 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
459 Two-letter ISO 3166-2 country code that will be used for
460 explicit geographic restriction bypassing via faking
461 X-Forwarded-For HTTP header
463 IP range in CIDR notation that will be used similarly to
465 external_downloader: A dictionary of protocol keys and the executable of the
466 external downloader to use for it. The allowed protocols
467 are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
468 Set the value to 'native' to use the native downloader
469 compat_opts: Compatibility options. See "Differences in default behavior".
470 The following options do not work when used through the API:
471 filename, abort-on-error, multistreams, no-live-chat,
472 format-sort, no-clean-infojson, no-playlist-metafiles,
473 no-keep-subs, no-attach-info-json, allow-unsafe-ext.
474 Refer __init__.py for their implementation
475 progress_template: Dictionary of templates for progress outputs.
476 Allowed keys are 'download', 'postprocess',
477 'download-title' (console title) and 'postprocess-title'.
478 The template is mapped on a dictionary with keys 'progress' and 'info'
479 retry_sleep_functions: Dictionary of functions that takes the number of attempts
480 as argument and returns the time to sleep in seconds.
481 Allowed keys are 'http', 'fragment', 'file_access'
482 download_ranges: A callback function that gets called for every video with
483 the signature (info_dict, ydl) -> Iterable[Section].
484 Only the returned sections will be downloaded.
485 Each Section is a dict with the following keys:
486 * start_time: Start time of the section in seconds
487 * end_time: End time of the section in seconds
488 * title: Section title (Optional)
489 * index: Section number (Optional)
490 force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts
491 noprogress: Do not print the progress bar
492 live_from_start: Whether to download livestreams videos from the start
494 The following parameters are not used by YoutubeDL itself, they are used by
495 the downloader (see yt_dlp/downloader/common.py):
496 nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
497 max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries,
498 continuedl, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
499 external_downloader_args, concurrent_fragment_downloads, progress_delta.
501 The following options are used by the post processors:
502 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
503 to the binary or its containing directory.
504 postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
505 and a list of additional command-line arguments for the
506 postprocessor/executable. The dict can also have "PP+EXE" keys
507 which are used when the given exe is used by the given PP.
508 Use 'default' as the name for arguments to passed to all PP
509 For compatibility with youtube-dl, a single list of args
512 The following options are used by the extractors:
513 extractor_retries: Number of times to retry for known errors (default: 3)
514 dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
515 hls_split_discontinuity: Split HLS playlists to different formats at
516 discontinuities such as ad breaks (default: False)
517 extractor_args: A dictionary of arguments to be passed to the extractors.
518 See "EXTRACTOR ARGUMENTS" for details.
519 E.g. {'youtube': {'skip': ['dash', 'hls']}}
520 mark_watched: Mark videos watched (even with --simulate). Only for YouTube
522 The following options are deprecated and may be removed in the future:
524 break_on_reject: Stop the download process when encountering a video that
525 has been filtered out.
526 - `raise DownloadCancelled(msg)` in match_filter instead
527 force_generic_extractor: Force downloader to use the generic extractor
528 - Use allowed_extractors = ['generic', 'default']
529 playliststart: - Use playlist_items
530 Playlist item to start at.
531 playlistend: - Use playlist_items
532 Playlist item to end at.
533 playlistreverse: - Use playlist_items
534 Download playlist items in reverse order.
535 forceurl: - Use forceprint
536 Force printing final URL.
537 forcetitle: - Use forceprint
538 Force printing title.
539 forceid: - Use forceprint
541 forcethumbnail: - Use forceprint
542 Force printing thumbnail URL.
543 forcedescription: - Use forceprint
544 Force printing description.
545 forcefilename: - Use forceprint
546 Force printing final filename.
547 forceduration: - Use forceprint
548 Force printing duration.
549 allsubtitles: - Use subtitleslangs = ['all']
550 Downloads all the subtitles of the video
551 (requires writesubtitles or writeautomaticsub)
552 include_ads: - Doesn't work
554 call_home: - Not implemented
555 Boolean, true iff we are allowed to contact the
556 yt-dlp servers for debugging.
557 post_hooks: - Register a custom postprocessor
558 A list of functions that get called as the final step
559 for each video file, after all postprocessors have been
560 called. The filename will be passed as the only argument.
561 hls_prefer_native: - Use external_downloader = {'m3u8': 'native'} or {'m3u8': 'ffmpeg'}.
562 Use the native HLS downloader instead of ffmpeg/avconv
563 if True, otherwise use ffmpeg/avconv if False, otherwise
564 use downloader suggested by extractor if None.
565 prefer_ffmpeg: - avconv support is deprecated
566 If False, use avconv instead of ffmpeg if both are available,
567 otherwise prefer ffmpeg.
568 youtube_include_dash_manifest: - Use extractor_args
569 If True (default), DASH manifests and related
570 data will be downloaded and processed by extractor.
571 You can reduce network I/O by disabling it if you don't
572 care about DASH. (only for youtube)
573 youtube_include_hls_manifest: - Use extractor_args
574 If True (default), HLS manifests and related
575 data will be downloaded and processed by extractor.
576 You can reduce network I/O by disabling it if you don't
577 care about HLS. (only for youtube)
578 no_color: Same as `color='no_color'`
579 no_overwrites: Same as `overwrites=False`
583 'width', 'height', 'asr', 'audio_channels', 'fps',
584 'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx',
585 'timestamp', 'release_timestamp',
586 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
587 'average_rating', 'comment_count', 'age_limit',
588 'start_time', 'end_time',
589 'chapter_number', 'season_number', 'episode_number',
590 'track_number', 'disc_number', 'release_year',
594 # NB: Keep in sync with the docstring of extractor/common.py
595 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note',
596 'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
597 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns',
598 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data',
599 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies',
600 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'extra_param_to_key_url',
601 'hls_aes', 'downloader_options', 'page_url', 'app', 'play_path', 'tc_url', 'flash_version',
602 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time',
604 _deprecated_multivalue_fields
= {
605 'album_artist': 'album_artists',
607 'composer': 'composers',
608 'creator': 'creators',
611 _format_selection_exts
= {
612 'audio': set(MEDIA_EXTENSIONS
.common_audio
),
613 'video': {*MEDIA_EXTENSIONS.common_video, '3gp'}
,
614 'storyboards': set(MEDIA_EXTENSIONS
.storyboards
),
617 def __init__(self
, params
=None, auto_init
=True):
618 """Create a FileDownloader object with the given options.
619 @param auto_init Whether to load the default extractors and print header (if verbose).
620 Set to 'no_verbose_header' to not print the header
626 self
._ies
_instances
= {}
627 self
._pps
= {k: [] for k in POSTPROCESS_WHEN}
628 self
._printed
_messages
= set()
629 self
._first
_webpage
_request
= True
630 self
._post
_hooks
= []
631 self
._progress
_hooks
= []
632 self
._postprocessor
_hooks
= []
633 self
._download
_retcode
= 0
634 self
._num
_downloads
= 0
636 self
._playlist
_level
= 0
637 self
._playlist
_urls
= set()
638 self
.cache
= Cache(self
)
639 self
.__header
_cookies
= []
641 stdout
= sys
.stderr
if self
.params
.get('logtostderr') else sys
.stdout
642 self
._out
_files
= Namespace(
645 screen
=sys
.stderr
if self
.params
.get('quiet') else stdout
,
646 console
=None if compat_os_name
== 'nt' else next(
647 filter(supports_terminal_sequences
, (sys
.stderr
, sys
.stdout
)), None),
651 windows_enable_vt_mode()
652 except Exception as e
:
653 self
.write_debug(f
'Failed to enable VT mode: {e}')
655 if self
.params
.get('no_color'):
656 if self
.params
.get('color') is not None:
657 self
.params
.setdefault('_warnings', []).append(
658 'Overwriting params from "color" with "no_color"')
659 self
.params
['color'] = 'no_color'
661 term_allow_color
= os
.getenv('TERM', '').lower() != 'dumb'
662 no_color
= bool(os
.getenv('NO_COLOR'))
664 def process_color_policy(stream
):
665 stream_name
= {sys.stdout: 'stdout', sys.stderr: 'stderr'}
[stream
]
666 policy
= traverse_obj(self
.params
, ('color', (stream_name
, None), {str}
), get_all
=False)
667 if policy
in ('auto', None):
668 if term_allow_color
and supports_terminal_sequences(stream
):
669 return 'no_color' if no_color
else True
671 assert policy
in ('always', 'never', 'no_color'), policy
672 return {'always': True, 'never': False}
.get(policy
, policy
)
674 self
._allow
_colors
= Namespace(**{
675 name
: process_color_policy(stream
)
676 for name
, stream
in self
._out
_files
.items_
if name
!= 'console'
679 system_deprecation
= _get_system_deprecation()
680 if system_deprecation
:
681 self
.deprecated_feature(system_deprecation
.replace('\n', '\n '))
683 if self
.params
.get('allow_unplayable_formats'):
685 f
'You have asked for {self._format_err("UNPLAYABLE", self.Styles.EMPHASIS)} formats to be listed/downloaded. '
686 'This is a developer option intended for debugging. \n'
687 ' If you experience any issues while using this option, '
688 f
'{self._format_err("DO NOT", self.Styles.ERROR)} open a bug report')
690 if self
.params
.get('bidi_workaround', False):
693 master
, slave
= pty
.openpty()
694 width
= shutil
.get_terminal_size().columns
695 width_args
= [] if width
is None else ['-w', str(width
)]
696 sp_kwargs
= {'stdin': subprocess.PIPE, 'stdout': slave, 'stderr': self._out_files.error}
698 self
._output
_process
= Popen(['bidiv', *width_args
], **sp_kwargs
)
700 self
._output
_process
= Popen(['fribidi', '-c', 'UTF-8', *width_args
], **sp_kwargs
)
701 self
._output
_channel
= os
.fdopen(master
, 'rb')
702 except OSError as ose
:
703 if ose
.errno
== errno
.ENOENT
:
705 'Could not find fribidi executable, ignoring --bidi-workaround. '
706 'Make sure that fribidi is an executable file in one of the directories in your $PATH.')
710 self
.params
['compat_opts'] = set(self
.params
.get('compat_opts', ()))
711 self
.params
['http_headers'] = HTTPHeaderDict(std_headers
, self
.params
.get('http_headers'))
712 self
._load
_cookies
(self
.params
['http_headers'].get('Cookie')) # compat
713 self
.params
['http_headers'].pop('Cookie', None)
715 if auto_init
and auto_init
!= 'no_verbose_header':
716 self
.print_debug_header()
718 def check_deprecated(param
, option
, suggestion
):
719 if self
.params
.get(param
) is not None:
720 self
.report_warning(f
'{option} is deprecated. Use {suggestion} instead')
724 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
725 if self
.params
.get('geo_verification_proxy') is None:
726 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
728 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
729 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
730 check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
732 for msg
in self
.params
.get('_warnings', []):
733 self
.report_warning(msg
)
734 for msg
in self
.params
.get('_deprecation_warnings', []):
735 self
.deprecated_feature(msg
)
737 if impersonate_target
:= self
.params
.get('impersonate'):
738 if not self
._impersonate
_target
_available
(impersonate_target
):
739 raise YoutubeDLError(
740 f
'Impersonate target "{impersonate_target}" is not available. '
741 f
'Use --list-impersonate-targets to see available targets. '
742 f
'You may be missing dependencies required to support this target.')
744 if 'list-formats' in self
.params
['compat_opts']:
745 self
.params
['listformats_table'] = False
747 if 'overwrites' not in self
.params
and self
.params
.get('nooverwrites') is not None:
748 # nooverwrites was unnecessarily changed to overwrites
749 # in 0c3d0f51778b153f65c21906031c2e091fcfb641
750 # This ensures compatibility with both keys
751 self
.params
['overwrites'] = not self
.params
['nooverwrites']
752 elif self
.params
.get('overwrites') is None:
753 self
.params
.pop('overwrites', None)
755 self
.params
['nooverwrites'] = not self
.params
['overwrites']
757 if self
.params
.get('simulate') is None and any((
758 self
.params
.get('list_thumbnails'),
759 self
.params
.get('listformats'),
760 self
.params
.get('listsubtitles'),
762 self
.params
['simulate'] = 'list_only'
764 self
.params
.setdefault('forceprint', {})
765 self
.params
.setdefault('print_to_file', {})
767 # Compatibility with older syntax
768 if not isinstance(params
['forceprint'], dict):
769 self
.params
['forceprint'] = {'video': params['forceprint']}
772 self
.add_default_info_extractors()
774 if (sys
.platform
!= 'win32'
775 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
776 and not self
.params
.get('restrictfilenames', False)):
777 # Unicode filesystem API will throw errors (#1474, #13027)
779 'Assuming --restrict-filenames since file system encoding '
780 'cannot encode all characters. '
781 'Set the LC_ALL environment variable to fix this.')
782 self
.params
['restrictfilenames'] = True
784 self
._parse
_outtmpl
()
786 # Creating format selector here allows us to catch syntax errors before the extraction
787 self
.format_selector
= (
788 self
.params
.get('format') if self
.params
.get('format') in (None, '-')
789 else self
.params
['format'] if callable(self
.params
['format'])
790 else self
.build_format_selector(self
.params
['format']))
793 'post_hooks': self
.add_post_hook
,
794 'progress_hooks': self
.add_progress_hook
,
795 'postprocessor_hooks': self
.add_postprocessor_hook
,
797 for opt
, fn
in hooks
.items():
798 for ph
in self
.params
.get(opt
, []):
801 for pp_def_raw
in self
.params
.get('postprocessors', []):
802 pp_def
= dict(pp_def_raw
)
803 when
= pp_def
.pop('when', 'post_process')
804 self
.add_post_processor(
805 get_postprocessor(pp_def
.pop('key'))(self
, **pp_def
),
808 def preload_download_archive(fn
):
809 """Preload the archive, if any is specified"""
813 elif not is_path_like(fn
):
816 self
.write_debug(f
'Loading archive file {fn!r}')
818 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
819 for line
in archive_file
:
820 archive
.add(line
.strip())
821 except OSError as ioe
:
822 if ioe
.errno
!= errno
.ENOENT
:
826 self
.archive
= preload_download_archive(self
.params
.get('download_archive'))
828 def warn_if_short_id(self
, argv
):
829 # short YouTube ID starting with dash?
831 i
for i
, a
in enumerate(argv
)
832 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
836 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
837 + ['--'] + [argv
[i
] for i
in idxs
]
840 'Long argument string detected. '
841 f
'Use -- to separate parameters and URLs, like this:\n{shell_quote(correct_argv)}')
843 def add_info_extractor(self
, ie
):
844 """Add an InfoExtractor object to the end of the list."""
846 self
._ies
[ie_key
] = ie
847 if not isinstance(ie
, type):
848 self
._ies
_instances
[ie_key
] = ie
849 ie
.set_downloader(self
)
851 def get_info_extractor(self
, ie_key
):
853 Get an instance of an IE with name ie_key, it will try to get one from
854 the _ies list, if there's no instance it will create a new one and add
855 it to the extractor list.
857 ie
= self
._ies
_instances
.get(ie_key
)
859 ie
= get_info_extractor(ie_key
)()
860 self
.add_info_extractor(ie
)
863 def add_default_info_extractors(self
):
865 Add the InfoExtractors returned by gen_extractors to the end of the list
867 all_ies
= {ie.IE_NAME.lower(): ie for ie in gen_extractor_classes()}
868 all_ies
['end'] = UnsupportedURLIE()
870 ie_names
= orderedSet_from_options(
871 self
.params
.get('allowed_extractors', ['default']), {
872 'all': list(all_ies
),
873 'default': [name
for name
, ie
in all_ies
.items() if ie
._ENABLED
],
875 except re
.error
as e
:
876 raise ValueError(f
'Wrong regex for allowed_extractors: {e.pattern}')
877 for name
in ie_names
:
878 self
.add_info_extractor(all_ies
[name
])
879 self
.write_debug(f
'Loaded {len(ie_names)} extractors')
881 def add_post_processor(self
, pp
, when
='post_process'):
882 """Add a PostProcessor object to the end of the chain."""
883 assert when
in POSTPROCESS_WHEN
, f
'Invalid when={when}'
884 self
._pps
[when
].append(pp
)
885 pp
.set_downloader(self
)
887 def add_post_hook(self
, ph
):
888 """Add the post hook"""
889 self
._post
_hooks
.append(ph
)
891 def add_progress_hook(self
, ph
):
892 """Add the download progress hook"""
893 self
._progress
_hooks
.append(ph
)
895 def add_postprocessor_hook(self
, ph
):
896 """Add the postprocessing progress hook"""
897 self
._postprocessor
_hooks
.append(ph
)
898 for pps
in self
._pps
.values():
900 pp
.add_progress_hook(ph
)
902 def _bidi_workaround(self
, message
):
903 if not hasattr(self
, '_output_channel'):
906 assert hasattr(self
, '_output_process')
907 assert isinstance(message
, str)
908 line_count
= message
.count('\n') + 1
909 self
._output
_process
.stdin
.write((message
+ '\n').encode())
910 self
._output
_process
.stdin
.flush()
911 res
= ''.join(self
._output
_channel
.readline().decode()
912 for _
in range(line_count
))
913 return res
[:-len('\n')]
915 def _write_string(self
, message
, out
=None, only_once
=False):
917 if message
in self
._printed
_messages
:
919 self
._printed
_messages
.add(message
)
920 write_string(message
, out
=out
, encoding
=self
.params
.get('encoding'))
922 def to_stdout(self
, message
, skip_eol
=False, quiet
=None):
923 """Print message to stdout"""
924 if quiet
is not None:
925 self
.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. '
926 'Use "YoutubeDL.to_screen" instead')
927 if skip_eol
is not False:
928 self
.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. '
929 'Use "YoutubeDL.to_screen" instead')
930 self
._write
_string
(f
'{self._bidi_workaround(message)}\n', self
._out
_files
.out
)
932 def to_screen(self
, message
, skip_eol
=False, quiet
=None, only_once
=False):
933 """Print message to screen if not in quiet mode"""
934 if self
.params
.get('logger'):
935 self
.params
['logger'].debug(message
)
937 if (self
.params
.get('quiet') if quiet
is None else quiet
) and not self
.params
.get('verbose'):
940 '{}{}'.format(self
._bidi
_workaround
(message
), ('' if skip_eol
else '\n')),
941 self
._out
_files
.screen
, only_once
=only_once
)
943 def to_stderr(self
, message
, only_once
=False):
944 """Print message to stderr"""
945 assert isinstance(message
, str)
946 if self
.params
.get('logger'):
947 self
.params
['logger'].error(message
)
949 self
._write
_string
(f
'{self._bidi_workaround(message)}\n', self
._out
_files
.error
, only_once
=only_once
)
951 def _send_console_code(self
, code
):
952 if compat_os_name
== 'nt' or not self
._out
_files
.console
:
954 self
._write
_string
(code
, self
._out
_files
.console
)
956 def to_console_title(self
, message
):
957 if not self
.params
.get('consoletitle', False):
959 message
= remove_terminal_sequences(message
)
960 if compat_os_name
== 'nt':
961 if ctypes
.windll
.kernel32
.GetConsoleWindow():
962 # c_wchar_p() might not be necessary if `message` is
963 # already of type unicode()
964 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
966 self
._send
_console
_code
(f
'\033]0;{message}\007')
968 def save_console_title(self
):
969 if not self
.params
.get('consoletitle') or self
.params
.get('simulate'):
971 self
._send
_console
_code
('\033[22;0t') # Save the title on stack
973 def restore_console_title(self
):
974 if not self
.params
.get('consoletitle') or self
.params
.get('simulate'):
976 self
._send
_console
_code
('\033[23;0t') # Restore the title from stack
979 self
.save_console_title()
982 def save_cookies(self
):
983 if self
.params
.get('cookiefile') is not None:
984 self
.cookiejar
.save()
986 def __exit__(self
, *args
):
987 self
.restore_console_title()
992 if '_request_director' in self
.__dict
__:
993 self
._request
_director
.close()
994 del self
._request
_director
996 def trouble(self
, message
=None, tb
=None, is_error
=True):
997 """Determine action to take when a download problem appears.
999 Depending on if the downloader has been configured to ignore
1000 download errors or not, this method may throw an exception or
1001 not when errors are found, after printing the message.
1003 @param tb If given, is additional traceback information
1004 @param is_error Whether to raise error according to ignorerrors
1006 if message
is not None:
1007 self
.to_stderr(message
)
1008 if self
.params
.get('verbose'):
1010 if sys
.exc_info()[0]: # if .trouble has been called from an except block
1012 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
1013 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
1014 tb
+= encode_compat_str(traceback
.format_exc())
1016 tb_data
= traceback
.format_list(traceback
.extract_stack())
1017 tb
= ''.join(tb_data
)
1022 if not self
.params
.get('ignoreerrors'):
1023 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
1024 exc_info
= sys
.exc_info()[1].exc_info
1026 exc_info
= sys
.exc_info()
1027 raise DownloadError(message
, exc_info
)
1028 self
._download
_retcode
= 1
1032 EMPHASIS
='light blue',
1037 BAD_FORMAT
='light red',
1039 SUPPRESS
='light black',
1042 def _format_text(self
, handle
, allow_colors
, text
, f
, fallback
=None, *, test_encoding
=False):
1045 original_text
= text
1046 # handle.encoding can be None. See https://github.com/yt-dlp/yt-dlp/issues/2711
1047 encoding
= self
.params
.get('encoding') or getattr(handle
, 'encoding', None) or 'ascii'
1048 text
= text
.encode(encoding
, 'ignore').decode(encoding
)
1049 if fallback
is not None and text
!= original_text
:
1051 return format_text(text
, f
) if allow_colors
is True else text
if fallback
is None else fallback
1053 def _format_out(self
, *args
, **kwargs
):
1054 return self
._format
_text
(self
._out
_files
.out
, self
._allow
_colors
.out
, *args
, **kwargs
)
1056 def _format_screen(self
, *args
, **kwargs
):
1057 return self
._format
_text
(self
._out
_files
.screen
, self
._allow
_colors
.screen
, *args
, **kwargs
)
1059 def _format_err(self
, *args
, **kwargs
):
1060 return self
._format
_text
(self
._out
_files
.error
, self
._allow
_colors
.error
, *args
, **kwargs
)
1062 def report_warning(self
, message
, only_once
=False):
1064 Print the message to stderr, it will be prefixed with 'WARNING:'
1065 If stderr is a tty file the 'WARNING:' will be colored
1067 if self
.params
.get('logger') is not None:
1068 self
.params
['logger'].warning(message
)
1070 if self
.params
.get('no_warnings'):
1072 self
.to_stderr(f
'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once
)
1074 def deprecation_warning(self
, message
, *, stacklevel
=0):
1075 deprecation_warning(
1076 message
, stacklevel
=stacklevel
+ 1, printer
=self
.report_error
, is_error
=False)
1078 def deprecated_feature(self
, message
):
1079 if self
.params
.get('logger') is not None:
1080 self
.params
['logger'].warning(f
'Deprecated Feature: {message}')
1081 self
.to_stderr(f
'{self._format_err("Deprecated Feature:", self.Styles.ERROR)} {message}', True)
1083 def report_error(self
, message
, *args
, **kwargs
):
1085 Do the same as trouble, but prefixes the message with 'ERROR:', colored
1086 in red if stderr is a tty file.
1088 self
.trouble(f
'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args
, **kwargs
)
1090 def write_debug(self
, message
, only_once
=False):
1091 """Log debug message or Print message to stderr"""
1092 if not self
.params
.get('verbose', False):
1094 message
= f
'[debug] {message}'
1095 if self
.params
.get('logger'):
1096 self
.params
['logger'].debug(message
)
1098 self
.to_stderr(message
, only_once
)
1100 def report_file_already_downloaded(self
, file_name
):
1101 """Report file has already been fully downloaded."""
1103 self
.to_screen(f
'[download] {file_name} has already been downloaded')
1104 except UnicodeEncodeError:
1105 self
.to_screen('[download] The file has already been downloaded')
1107 def report_file_delete(self
, file_name
):
1108 """Report that existing file will be deleted."""
1110 self
.to_screen(f
'Deleting existing file {file_name}')
1111 except UnicodeEncodeError:
1112 self
.to_screen('Deleting existing file')
1114 def raise_no_formats(self
, info
, forced
=False, *, msg
=None):
1115 has_drm
= info
.get('_has_drm')
1116 ignored
, expected
= self
.params
.get('ignore_no_formats_error'), bool(msg
)
1117 msg
= msg
or has_drm
and 'This video is DRM protected' or 'No video formats found!'
1118 if forced
or not ignored
:
1119 raise ExtractorError(msg
, video_id
=info
['id'], ie
=info
['extractor'],
1120 expected
=has_drm
or ignored
or expected
)
1122 self
.report_warning(msg
)
1124 def parse_outtmpl(self
):
1125 self
.deprecation_warning('"YoutubeDL.parse_outtmpl" is deprecated and may be removed in a future version')
1126 self
._parse
_outtmpl
()
1127 return self
.params
['outtmpl']
1129 def _parse_outtmpl(self
):
1131 if self
.params
.get('restrictfilenames'): # Remove spaces in the default template
1132 sanitize
= lambda x
: x
.replace(' - ', ' ').replace(' ', '-')
1134 outtmpl
= self
.params
.setdefault('outtmpl', {})
1135 if not isinstance(outtmpl
, dict):
1136 self
.params
['outtmpl'] = outtmpl
= {'default': outtmpl}
1137 outtmpl
.update({k: sanitize(v) for k, v in DEFAULT_OUTTMPL.items() if outtmpl.get(k) is None}
)
1139 def get_output_path(self
, dir_type
='', filename
=None):
1140 paths
= self
.params
.get('paths', {})
1141 assert isinstance(paths
, dict), '"paths" parameter must be a dictionary'
1142 path
= os
.path
.join(
1143 expand_path(paths
.get('home', '').strip()),
1144 expand_path(paths
.get(dir_type
, '').strip()) if dir_type
else '',
1146 return sanitize_path(path
, force
=self
.params
.get('windowsfilenames'))
1149 def _outtmpl_expandpath(outtmpl
):
1150 # expand_path translates '%%' into '%' and '$$' into '$'
1151 # correspondingly that is not what we want since we need to keep
1152 # '%%' intact for template dict substitution step. Working around
1153 # with boundary-alike separator hack.
1154 sep
= ''.join(random
.choices(string
.ascii_letters
, k
=32))
1155 outtmpl
= outtmpl
.replace('%%', f
'%{sep}%').replace('$$', f
'${sep}$')
1157 # outtmpl should be expand_path'ed before template dict substitution
1158 # because meta fields may contain env variables we don't want to
1159 # be expanded. E.g. for outtmpl "%(title)s.%(ext)s" and
1160 # title "Hello $PATH", we don't want `$PATH` to be expanded.
1161 return expand_path(outtmpl
).replace(sep
, '')
1164 def escape_outtmpl(outtmpl
):
1165 """ Escape any remaining strings like %s, %abc% etc. """
1167 STR_FORMAT_RE_TMPL
.format('', '(?![%(\0])'),
1168 lambda mobj
: ('' if mobj
.group('has_key') else '%') + mobj
.group(0),
1172 def validate_outtmpl(cls
, outtmpl
):
1173 """ @return None or Exception object """
1175 STR_FORMAT_RE_TMPL
.format('[^)]*', '[ljhqBUDS]'),
1176 lambda mobj
: f
'{mobj.group(0)[:-1]}s',
1177 cls
._outtmpl
_expandpath
(outtmpl
))
1179 cls
.escape_outtmpl(outtmpl
) % collections
.defaultdict(int)
1181 except ValueError as err
:
1185 def _copy_infodict(info_dict
):
1186 info_dict
= dict(info_dict
)
1187 info_dict
.pop('__postprocessors', None)
1188 info_dict
.pop('__pending_error', None)
1191 def prepare_outtmpl(self
, outtmpl
, info_dict
, sanitize
=False):
1192 """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict
1193 @param sanitize Whether to sanitize the output as a filename.
1194 For backward compatibility, a function can also be passed
1197 info_dict
.setdefault('epoch', int(time
.time())) # keep epoch consistent once set
1199 info_dict
= self
._copy
_infodict
(info_dict
)
1200 info_dict
['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
1201 formatSeconds(info_dict
['duration'], '-' if sanitize
else ':')
1202 if info_dict
.get('duration', None) is not None
1204 info_dict
['autonumber'] = int(self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
)
1205 info_dict
['video_autonumber'] = self
._num
_videos
1206 if info_dict
.get('resolution') is None:
1207 info_dict
['resolution'] = self
.format_resolution(info_dict
, default
=None)
1209 # For fields playlist_index, playlist_autonumber and autonumber convert all occurrences
1210 # of %(field)s to %(field)0Nd for backward compatibility
1211 field_size_compat_map
= {
1212 'playlist_index': number_of_digits(info_dict
.get('__last_playlist_index') or 0),
1213 'playlist_autonumber': number_of_digits(info_dict
.get('n_entries') or 0),
1214 'autonumber': self
.params
.get('autonumber_size') or 5,
1218 EXTERNAL_FORMAT_RE
= re
.compile(STR_FORMAT_RE_TMPL
.format('[^)]*', f
'[{STR_FORMAT_TYPES}ljhqBUDS]'))
1224 # Field is of the form key1.key2...
1225 # where keys (except first) can be string, int, slice or "{field, ...}"
1226 FIELD_INNER_RE
= r
'(?:\w+|%(num)s|%(num)s?(?::%(num)s?){1,2})' % {'num': r'(?:-?\d+)'}
# noqa: UP031
1227 FIELD_RE
= r
'\w*(?:\.(?:%(inner)s|{%(field)s(?:,%(field)s)*}))*' % { # noqa: UP031
1228 'inner': FIELD_INNER_RE
,
1229 'field': rf
'\w*(?:\.{FIELD_INNER_RE})*',
1231 MATH_FIELD_RE
= rf
'(?:{FIELD_RE}|-?{NUMBER_RE})'
1232 MATH_OPERATORS_RE
= r
'(?:{})'.format('|'.join(map(re
.escape
, MATH_FUNCTIONS
.keys())))
1233 INTERNAL_FORMAT_RE
= re
.compile(rf
'''(?xs)
1235 (?P<fields>{FIELD_RE})
1236 (?P<maths>(?:{MATH_OPERATORS_RE}{MATH_FIELD_RE})*)
1237 (?:>(?P<strf_format>.+?))?
1239 (?P<alternate>(?<!\\),[^|&)]+)?
1240 (?:&(?P<replacement>.*?))?
1241 (?:\|(?P<default>.*?))?
1244 def _from_user_input(field
):
1248 return slice(*map(int_or_none
, field
.split(':')))
1249 elif int_or_none(field
) is not None:
1253 def _traverse_infodict(fields
):
1254 fields
= [f
for x
in re
.split(r
'\.({.+?})\.?', fields
)
1255 for f
in ([x
] if x
.startswith('{') else x
.split('.'))]
1257 if fields
and not fields
[i
]:
1260 for i
, f
in enumerate(fields
):
1261 if not f
.startswith('{'):
1262 fields
[i
] = _from_user_input(f
)
1264 assert f
.endswith('}'), f
'No closing brace for {f} in {fields}'
1265 fields
[i
] = {k: list(map(_from_user_input, k.split('.'))) for k in f[1:-1].split(',')}
1267 return traverse_obj(info_dict
, fields
, traverse_string
=True)
1269 def get_value(mdict
):
1271 value
= _traverse_infodict(mdict
['fields'])
1274 value
= float_or_none(value
)
1275 if value
is not None:
1278 offset_key
= mdict
['maths']
1280 value
= float_or_none(value
)
1284 MATH_FIELD_RE
if operator
else MATH_OPERATORS_RE
,
1285 offset_key
).group(0)
1286 offset_key
= offset_key
[len(item
):]
1287 if operator
is None:
1288 operator
= MATH_FUNCTIONS
[item
]
1290 item
, multiplier
= (item
[1:], -1) if item
[0] == '-' else (item
, 1)
1291 offset
= float_or_none(item
)
1293 offset
= float_or_none(_traverse_infodict(item
))
1295 value
= operator(value
, multiplier
* offset
)
1296 except (TypeError, ZeroDivisionError):
1299 # Datetime formatting
1300 if mdict
['strf_format']:
1301 value
= strftime_or_none(value
, mdict
['strf_format'].replace('\\,', ','))
1303 # XXX: Workaround for https://github.com/yt-dlp/yt-dlp/issues/4485
1304 if sanitize
and value
== '':
1308 na
= self
.params
.get('outtmpl_na_placeholder', 'NA')
1310 def filename_sanitizer(key
, value
, restricted
=self
.params
.get('restrictfilenames')):
1311 return sanitize_filename(str(value
), restricted
=restricted
, is_id
=(
1312 bool(re
.search(r
'(^|[_.])id(\.|$)', key
))
1313 if 'filename-sanitization' in self
.params
['compat_opts']
1316 sanitizer
= sanitize
if callable(sanitize
) else filename_sanitizer
1317 sanitize
= bool(sanitize
)
1319 def _dumpjson_default(obj
):
1320 if isinstance(obj
, (set, LazyList
)):
1324 class _ReplacementFormatter(string
.Formatter
):
1325 def get_field(self
, field_name
, args
, kwargs
):
1326 if field_name
.isdigit():
1328 raise ValueError('Unsupported field')
1330 replacement_formatter
= _ReplacementFormatter()
1332 def create_key(outer_mobj
):
1333 if not outer_mobj
.group('has_key'):
1334 return outer_mobj
.group(0)
1335 key
= outer_mobj
.group('key')
1336 mobj
= re
.match(INTERNAL_FORMAT_RE
, key
)
1337 value
, replacement
, default
, last_field
= None, None, na
, ''
1339 mobj
= mobj
.groupdict()
1340 default
= mobj
['default'] if mobj
['default'] is not None else default
1341 value
= get_value(mobj
)
1342 last_field
, replacement
= mobj
['fields'], mobj
['replacement']
1343 if value
is None and mobj
['alternate']:
1344 mobj
= re
.match(INTERNAL_FORMAT_RE
, mobj
['remaining'][1:])
1348 if None not in (value
, replacement
):
1350 value
= replacement_formatter
.format(replacement
, value
)
1352 value
, default
= None, na
1354 fmt
= outer_mobj
.group('format')
1355 if fmt
== 's' and last_field
in field_size_compat_map
and isinstance(value
, int):
1356 fmt
= f
'0{field_size_compat_map[last_field]:d}d'
1358 flags
= outer_mobj
.group('conversion') or ''
1359 str_fmt
= f
'{fmt[:-1]}s'
1361 value
, fmt
= default
, 's'
1362 elif fmt
[-1] == 'l': # list
1363 delim
= '\n' if '#' in flags
else ', '
1364 value
, fmt
= delim
.join(map(str, variadic(value
, allowed_types
=(str, bytes)))), str_fmt
1365 elif fmt
[-1] == 'j': # json
1366 value
, fmt
= json
.dumps(
1367 value
, default
=_dumpjson_default
,
1368 indent
=4 if '#' in flags
else None, ensure_ascii
='+' not in flags
), str_fmt
1369 elif fmt
[-1] == 'h': # html
1370 value
, fmt
= escapeHTML(str(value
)), str_fmt
1371 elif fmt
[-1] == 'q': # quoted
1372 value
= map(str, variadic(value
) if '#' in flags
else [value
])
1373 value
, fmt
= shell_quote(value
, shell
=True), str_fmt
1374 elif fmt
[-1] == 'B': # bytes
1375 value
= f
'%{str_fmt}'.encode() % str(value
).encode()
1376 value
, fmt
= value
.decode('utf-8', 'ignore'), 's'
1377 elif fmt
[-1] == 'U': # unicode normalized
1378 value
, fmt
= unicodedata
.normalize(
1379 # "+" = compatibility equivalence, "#" = NFD
1380 'NF{}{}'.format('K' if '+' in flags
else '', 'D' if '#' in flags
else 'C'),
1382 elif fmt
[-1] == 'D': # decimal suffix
1383 num_fmt
, fmt
= fmt
[:-1].replace('#', ''), 's'
1384 value
= format_decimal_suffix(value
, f
'%{num_fmt}f%s' if num_fmt
else '%d%s',
1385 factor
=1024 if '#' in flags
else 1000)
1386 elif fmt
[-1] == 'S': # filename sanitization
1387 value
, fmt
= filename_sanitizer(last_field
, value
, restricted
='#' in flags
), str_fmt
1388 elif fmt
[-1] == 'c':
1390 value
= str(value
)[0]
1393 elif fmt
[-1] not in 'rsa': # numeric
1394 value
= float_or_none(value
)
1396 value
, fmt
= default
, 's'
1399 # If value is an object, sanitize might convert it to a string
1400 # So we convert it to repr first
1402 value
, fmt
= repr(value
), str_fmt
1403 elif fmt
[-1] == 'a':
1404 value
, fmt
= ascii(value
), str_fmt
1405 if fmt
[-1] in 'csra':
1406 value
= sanitizer(last_field
, value
)
1408 key
= '{}\0{}'.format(key
.replace('%', '%\0'), outer_mobj
.group('format'))
1409 TMPL_DICT
[key
] = value
1410 return '{prefix}%({key}){fmt}'.format(key
=key
, fmt
=fmt
, prefix
=outer_mobj
.group('prefix'))
1412 return EXTERNAL_FORMAT_RE
.sub(create_key
, outtmpl
), TMPL_DICT
1414 def evaluate_outtmpl(self
, outtmpl
, info_dict
, *args
, **kwargs
):
1415 outtmpl
, info_dict
= self
.prepare_outtmpl(outtmpl
, info_dict
, *args
, **kwargs
)
1416 return self
.escape_outtmpl(outtmpl
) % info_dict
1418 @_catch_unsafe_extension_error
1419 def _prepare_filename(self
, info_dict
, *, outtmpl
=None, tmpl_type
=None):
1420 assert None in (outtmpl
, tmpl_type
), 'outtmpl and tmpl_type are mutually exclusive'
1422 outtmpl
= self
.params
['outtmpl'].get(tmpl_type
or 'default', self
.params
['outtmpl']['default'])
1424 outtmpl
= self
._outtmpl
_expandpath
(outtmpl
)
1425 filename
= self
.evaluate_outtmpl(outtmpl
, info_dict
, True)
1429 if tmpl_type
in ('', 'temp'):
1430 final_ext
, ext
= self
.params
.get('final_ext'), info_dict
.get('ext')
1431 if final_ext
and ext
and final_ext
!= ext
and filename
.endswith(f
'.{final_ext}'):
1432 filename
= replace_extension(filename
, ext
, final_ext
)
1434 force_ext
= OUTTMPL_TYPES
[tmpl_type
]
1436 filename
= replace_extension(filename
, force_ext
, info_dict
.get('ext'))
1438 # https://github.com/blackjack4494/youtube-dlc/issues/85
1439 trim_file_name
= self
.params
.get('trim_file_name', False)
1441 no_ext
, *ext
= filename
.rsplit('.', 2)
1442 filename
= join_nonempty(no_ext
[:trim_file_name
], *ext
, delim
='.')
1445 except ValueError as err
:
1446 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
1449 def prepare_filename(self
, info_dict
, dir_type
='', *, outtmpl
=None, warn
=False):
1450 """Generate the output filename"""
1452 assert not dir_type
, 'outtmpl and dir_type are mutually exclusive'
1454 filename
= self
._prepare
_filename
(info_dict
, tmpl_type
=dir_type
, outtmpl
=outtmpl
)
1455 if not filename
and dir_type
not in ('', 'temp'):
1459 if not self
.params
.get('paths'):
1461 elif filename
== '-':
1462 self
.report_warning('--paths is ignored when an outputting to stdout', only_once
=True)
1463 elif os
.path
.isabs(filename
):
1464 self
.report_warning('--paths is ignored since an absolute path is given in output template', only_once
=True)
1465 if filename
== '-' or not filename
:
1468 return self
.get_output_path(dir_type
, filename
)
1470 def _match_entry(self
, info_dict
, incomplete
=False, silent
=False):
1471 """Returns None if the file should be downloaded"""
1472 _type
= 'video' if 'playlist-match-filter' in self
.params
['compat_opts'] else info_dict
.get('_type', 'video')
1473 assert incomplete
or _type
== 'video', 'Only video result can be considered complete'
1475 video_title
= info_dict
.get('title', info_dict
.get('id', 'entry'))
1478 if _type
in ('playlist', 'multi_video'):
1480 elif _type
in ('url', 'url_transparent') and not try_call(
1481 lambda: self
.get_info_extractor(info_dict
['ie_key']).is_single_video(info_dict
['url'])):
1484 if 'title' in info_dict
:
1485 # This can happen when we're just evaluating the playlist
1486 title
= info_dict
['title']
1487 matchtitle
= self
.params
.get('matchtitle', False)
1489 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
1490 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
1491 rejecttitle
= self
.params
.get('rejecttitle', False)
1493 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
1494 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
1496 date
= info_dict
.get('upload_date')
1497 if date
is not None:
1498 date_range
= self
.params
.get('daterange', DateRange())
1499 if date
not in date_range
:
1500 return f
'{date_from_str(date).isoformat()} upload date is not in range {date_range}'
1501 view_count
= info_dict
.get('view_count')
1502 if view_count
is not None:
1503 min_views
= self
.params
.get('min_views')
1504 if min_views
is not None and view_count
< min_views
:
1505 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
1506 max_views
= self
.params
.get('max_views')
1507 if max_views
is not None and view_count
> max_views
:
1508 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
1509 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
1510 return f
'Skipping "{video_title}" because it is age restricted'
1512 match_filter
= self
.params
.get('match_filter')
1513 if match_filter
is None:
1519 ret
= match_filter(info_dict
, incomplete
=incomplete
)
1521 # For backward compatibility
1522 ret
= None if incomplete
else match_filter(info_dict
)
1523 except DownloadCancelled
as err
:
1524 if err
.msg
is not NO_DEFAULT
:
1526 ret
, cancelled
= err
.msg
, err
1528 if ret
is NO_DEFAULT
:
1530 filename
= self
._format
_screen
(self
.prepare_filename(info_dict
), self
.Styles
.FILENAME
)
1531 reply
= input(self
._format
_screen
(
1532 f
'Download "{filename}"? (Y/n): ', self
.Styles
.EMPHASIS
)).lower().strip()
1533 if reply
in {'y', ''}
:
1537 raise type(cancelled
)(f
'Skipping {video_title}')
1538 return f
'Skipping {video_title}'
1541 if self
.in_download_archive(info_dict
):
1543 format_field(info_dict
, 'id', f
'{self._format_screen("%s", self.Styles.ID)}: '),
1544 format_field(info_dict
, 'title', f
'{self._format_screen("%s", self.Styles.EMPHASIS)} '),
1545 'has already been recorded in the archive'))
1546 break_opt
, break_err
= 'break_on_existing', ExistingVideoReached
1549 reason
= check_filter()
1550 except DownloadCancelled
as e
:
1551 reason
, break_opt
, break_err
= e
.msg
, 'match_filter', type(e
)
1553 break_opt
, break_err
= 'break_on_reject', RejectedVideoReached
1554 if reason
is not None:
1556 self
.to_screen('[download] ' + reason
)
1557 if self
.params
.get(break_opt
, False):
1562 def add_extra_info(info_dict
, extra_info
):
1563 """Set the keys from extra_info in info dict if they are missing"""
1564 for key
, value
in extra_info
.items():
1565 info_dict
.setdefault(key
, value
)
1567 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
=None,
1568 process
=True, force_generic_extractor
=False):
1570 Extract and return the information dictionary of the URL
1573 @param url URL to extract
1576 @param download Whether to download videos
1577 @param process Whether to resolve all unresolved references (URLs, playlist items).
1578 Must be True for download to work
1579 @param ie_key Use only the extractor with this key
1581 @param extra_info Dictionary containing the extra values to add to the info (For internal use only)
1582 @force_generic_extractor Force using the generic extractor (Deprecated; use ie_key='Generic')
1585 if extra_info
is None:
1588 if not ie_key
and force_generic_extractor
:
1592 ies
= {ie_key: self._ies[ie_key]}
if ie_key
in self
._ies
else {}
1596 for key
, ie
in ies
.items():
1597 if not ie
.suitable(url
):
1600 if not ie
.working():
1601 self
.report_warning('The program functionality for this site has been marked as broken, '
1602 'and will probably not work.')
1604 temp_id
= ie
.get_temp_id(url
)
1605 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': key}
):
1606 self
.to_screen(f
'[download] {self._format_screen(temp_id, self.Styles.ID)}: '
1607 'has already been recorded in the archive')
1608 if self
.params
.get('break_on_existing', False):
1609 raise ExistingVideoReached
1611 return self
.__extract
_info
(url
, self
.get_info_extractor(key
), download
, extra_info
, process
)
1613 extractors_restricted
= self
.params
.get('allowed_extractors') not in (None, ['default'])
1614 self
.report_error(f
'No suitable extractor{format_field(ie_key, None, " (%s)")} found for URL {url}',
1615 tb
=False if extractors_restricted
else None)
1617 def _handle_extraction_exceptions(func
):
1618 @functools.wraps(func
)
1619 def wrapper(self
, *args
, **kwargs
):
1622 return func(self
, *args
, **kwargs
)
1623 except (DownloadCancelled
, LazyList
.IndexError, PagedList
.IndexError):
1625 except ReExtractInfo
as e
:
1627 self
.to_screen(f
'{e}; Re-extracting data')
1629 self
.to_stderr('\r')
1630 self
.report_warning(f
'{e}; Re-extracting data')
1632 except GeoRestrictedError
as e
:
1635 msg
+= '\nThis video is available in {}.'.format(', '.join(
1636 map(ISO3166Utils
.short2full
, e
.countries
)))
1637 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
1638 self
.report_error(msg
)
1639 except ExtractorError
as e
: # An error we somewhat expected
1640 self
.report_error(str(e
), e
.format_traceback())
1641 except Exception as e
:
1642 if self
.params
.get('ignoreerrors'):
1643 self
.report_error(str(e
), tb
=encode_compat_str(traceback
.format_exc()))
1649 def _wait_for_video(self
, ie_result
={}):
1650 if (not self
.params
.get('wait_for_video')
1651 or ie_result
.get('_type', 'video') != 'video'
1652 or ie_result
.get('formats') or ie_result
.get('url')):
1655 format_dur
= lambda dur
: '%02d:%02d:%02d' % timetuple_from_msec(dur
* 1000)[:-1]
1660 full_msg
= f
'{msg}\n'
1661 if not self
.params
.get('noprogress'):
1662 full_msg
= msg
+ ' ' * (len(last_msg
) - len(msg
)) + '\r'
1665 self
.to_screen(full_msg
, skip_eol
=True)
1668 min_wait
, max_wait
= self
.params
.get('wait_for_video')
1669 diff
= try_get(ie_result
, lambda x
: x
['release_timestamp'] - time
.time())
1670 if diff
is None and ie_result
.get('live_status') == 'is_upcoming':
1671 diff
= round(random
.uniform(min_wait
, max_wait
) if (max_wait
and min_wait
) else (max_wait
or min_wait
), 0)
1672 self
.report_warning('Release time of video is not known')
1673 elif ie_result
and (diff
or 0) <= 0:
1674 self
.report_warning('Video should already be available according to extracted info')
1675 diff
= min(max(diff
or 0, min_wait
or 0), max_wait
or float('inf'))
1676 self
.to_screen(f
'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now')
1678 wait_till
= time
.time() + diff
1681 diff
= wait_till
- time
.time()
1684 raise ReExtractInfo('[wait] Wait period ended', expected
=True)
1685 progress(f
'[wait] Remaining time until next attempt: {self._format_screen(format_dur(diff), self.Styles.EMPHASIS)}')
1687 except KeyboardInterrupt:
1689 raise ReExtractInfo('[wait] Interrupted by user', expected
=True)
1690 except BaseException
as e
:
1691 if not isinstance(e
, ReExtractInfo
):
1695 def _load_cookies(self
, data
, *, autoscope
=True):
1696 """Loads cookies from a `Cookie` header
1698 This tries to work around the security vulnerability of passing cookies to every domain.
1699 See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
1701 @param data The Cookie header as string to load the cookies from
1702 @param autoscope If `False`, scope cookies using Set-Cookie syntax and error for cookie without domains
1703 If `True`, save cookies for later to be stored in the jar with a limited scope
1704 If a URL, save cookies in the jar with the domain of the URL
1706 for cookie
in LenientSimpleCookie(data
).values():
1707 if autoscope
and any(cookie
.values()):
1708 raise ValueError('Invalid syntax in Cookie Header')
1710 domain
= cookie
.get('domain') or ''
1711 expiry
= cookie
.get('expires')
1712 if expiry
== '': # 0 is valid
1714 prepared_cookie
= http
.cookiejar
.Cookie(
1715 cookie
.get('version') or 0, cookie
.key
, cookie
.value
, None, False,
1716 domain
, True, True, cookie
.get('path') or '', bool(cookie
.get('path')),
1717 cookie
.get('secure') or False, expiry
, False, None, None, {})
1720 self
.cookiejar
.set_cookie(prepared_cookie
)
1721 elif autoscope
is True:
1722 self
.deprecated_feature(
1723 'Passing cookies as a header is a potential security risk; '
1724 'they will be scoped to the domain of the downloaded urls. '
1725 'Please consider loading cookies from a file or browser instead.')
1726 self
.__header
_cookies
.append(prepared_cookie
)
1728 self
.report_warning(
1729 'The extractor result contains an unscoped cookie as an HTTP header. '
1730 f
'If you are using yt-dlp with an input URL{bug_reports_message(before=",")}',
1732 self
._apply
_header
_cookies
(autoscope
, [prepared_cookie
])
1734 self
.report_error('Unscoped cookies are not allowed; please specify some sort of scoping',
1735 tb
=False, is_error
=False)
1737 def _apply_header_cookies(self
, url
, cookies
=None):
1738 """Applies stray header cookies to the provided url
1740 This loads header cookies and scopes them to the domain provided in `url`.
1741 While this is not ideal, it helps reduce the risk of them being sent
1742 to an unintended destination while mostly maintaining compatibility.
1744 parsed
= urllib
.parse
.urlparse(url
)
1745 if not parsed
.hostname
:
1748 for cookie
in map(copy
.copy
, cookies
or self
.__header
_cookies
):
1749 cookie
.domain
= f
'.{parsed.hostname}'
1750 self
.cookiejar
.set_cookie(cookie
)
1752 @_handle_extraction_exceptions
1753 def __extract_info(self
, url
, ie
, download
, extra_info
, process
):
1754 self
._apply
_header
_cookies
(url
)
1757 ie_result
= ie
.extract(url
)
1758 except UserNotLive
as e
:
1760 if self
.params
.get('wait_for_video'):
1761 self
.report_warning(e
)
1762 self
._wait
_for
_video
()
1764 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
1765 self
.report_warning(f
'Extractor {ie.IE_NAME} returned nothing{bug_reports_message()}')
1767 if isinstance(ie_result
, list):
1768 # Backwards compatibility: old IE result format
1770 '_type': 'compat_list',
1771 'entries': ie_result
,
1773 if extra_info
.get('original_url'):
1774 ie_result
.setdefault('original_url', extra_info
['original_url'])
1775 self
.add_default_extra_info(ie_result
, ie
, url
)
1777 self
._wait
_for
_video
(ie_result
)
1778 return self
.process_ie_result(ie_result
, download
, extra_info
)
1782 def add_default_extra_info(self
, ie_result
, ie
, url
):
1784 self
.add_extra_info(ie_result
, {
1786 'original_url': url
,
1788 webpage_url
= ie_result
.get('webpage_url')
1790 self
.add_extra_info(ie_result
, {
1791 'webpage_url_basename': url_basename(webpage_url
),
1792 'webpage_url_domain': get_domain(webpage_url
),
1795 self
.add_extra_info(ie_result
, {
1796 'extractor': ie
.IE_NAME
,
1797 'extractor_key': ie
.ie_key(),
1800 def process_ie_result(self
, ie_result
, download
=True, extra_info
=None):
1802 Take the result of the ie(may be modified) and resolve all unresolved
1803 references (URLs, playlist items).
1805 It will also download the videos if 'download'.
1806 Returns the resolved ie_result.
1808 if extra_info
is None:
1810 result_type
= ie_result
.get('_type', 'video')
1812 if result_type
in ('url', 'url_transparent'):
1813 ie_result
['url'] = sanitize_url(
1814 ie_result
['url'], scheme
='http' if self
.params
.get('prefer_insecure') else 'https')
1815 if ie_result
.get('original_url') and not extra_info
.get('original_url'):
1816 extra_info
= {'original_url': ie_result['original_url'], **extra_info}
1818 extract_flat
= self
.params
.get('extract_flat', False)
1819 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
1820 or extract_flat
is True):
1821 info_copy
= ie_result
.copy()
1822 ie
= try_get(ie_result
.get('ie_key'), self
.get_info_extractor
)
1823 if ie
and not ie_result
.get('id'):
1824 info_copy
['id'] = ie
.get_temp_id(ie_result
['url'])
1825 self
.add_default_extra_info(info_copy
, ie
, ie_result
['url'])
1826 self
.add_extra_info(info_copy
, extra_info
)
1827 info_copy
, _
= self
.pre_process(info_copy
)
1828 self
._fill
_common
_fields
(info_copy
, False)
1829 self
.__forced
_printings
(info_copy
)
1830 self
._raise
_pending
_errors
(info_copy
)
1831 if self
.params
.get('force_write_download_archive', False):
1832 self
.record_download_archive(info_copy
)
1835 if result_type
== 'video':
1836 self
.add_extra_info(ie_result
, extra_info
)
1837 ie_result
= self
.process_video_result(ie_result
, download
=download
)
1838 self
._raise
_pending
_errors
(ie_result
)
1839 additional_urls
= (ie_result
or {}).get('additional_urls')
1841 # TODO: Improve MetadataParserPP to allow setting a list
1842 if isinstance(additional_urls
, str):
1843 additional_urls
= [additional_urls
]
1845 '[info] {}: {} additional URL(s) requested'.format(ie_result
['id'], len(additional_urls
)))
1846 self
.write_debug('Additional URLs: "{}"'.format('", "'.join(additional_urls
)))
1847 ie_result
['additional_entries'] = [
1849 url
, download
, extra_info
=extra_info
,
1850 force_generic_extractor
=self
.params
.get('force_generic_extractor'))
1851 for url
in additional_urls
1854 elif result_type
== 'url':
1855 # We have to add extra_info to the results because it may be
1856 # contained in a playlist
1857 return self
.extract_info(
1858 ie_result
['url'], download
,
1859 ie_key
=ie_result
.get('ie_key'),
1860 extra_info
=extra_info
)
1861 elif result_type
== 'url_transparent':
1862 # Use the information from the embedding page
1863 info
= self
.extract_info(
1864 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
1865 extra_info
=extra_info
, download
=False, process
=False)
1867 # extract_info may return None when ignoreerrors is enabled and
1868 # extraction failed with an error, don't crash and return early
1873 exempted_fields
= {'_type', 'url', 'ie_key'}
1874 if not ie_result
.get('section_end') and ie_result
.get('section_start') is None:
1875 # For video clips, the id etc of the clip extractor should be used
1876 exempted_fields |
= {'id', 'extractor', 'extractor_key'}
1878 new_result
= info
.copy()
1879 new_result
.update(filter_dict(ie_result
, lambda k
, v
: v
is not None and k
not in exempted_fields
))
1881 # Extracted info may not be a video result (i.e.
1882 # info.get('_type', 'video') != video) but rather an url or
1883 # url_transparent. In such cases outer metadata (from ie_result)
1884 # should be propagated to inner one (info). For this to happen
1885 # _type of info should be overridden with url_transparent. This
1886 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
1887 if new_result
.get('_type') == 'url':
1888 new_result
['_type'] = 'url_transparent'
1890 return self
.process_ie_result(
1891 new_result
, download
=download
, extra_info
=extra_info
)
1892 elif result_type
in ('playlist', 'multi_video'):
1893 # Protect from infinite recursion due to recursively nested playlists
1894 # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
1895 webpage_url
= ie_result
.get('webpage_url') # Playlists maynot have webpage_url
1896 if webpage_url
and webpage_url
in self
._playlist
_urls
:
1898 '[download] Skipping already downloaded playlist: {}'.format(
1899 ie_result
.get('title')) or ie_result
.get('id'))
1902 self
._playlist
_level
+= 1
1903 self
._playlist
_urls
.add(webpage_url
)
1904 self
._fill
_common
_fields
(ie_result
, False)
1905 self
._sanitize
_thumbnails
(ie_result
)
1907 return self
.__process
_playlist
(ie_result
, download
)
1909 self
._playlist
_level
-= 1
1910 if not self
._playlist
_level
:
1911 self
._playlist
_urls
.clear()
1912 elif result_type
== 'compat_list':
1913 self
.report_warning(
1914 'Extractor {} returned a compat_list result. '
1915 'It needs to be updated.'.format(ie_result
.get('extractor')))
1918 self
.add_extra_info(r
, {
1919 'extractor': ie_result
['extractor'],
1920 'webpage_url': ie_result
['webpage_url'],
1921 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1922 'webpage_url_domain': get_domain(ie_result
['webpage_url']),
1923 'extractor_key': ie_result
['extractor_key'],
1926 ie_result
['entries'] = [
1927 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1928 for r
in ie_result
['entries']
1932 raise Exception(f
'Invalid result type: {result_type}')
1934 def _ensure_dir_exists(self
, path
):
1935 return make_dir(path
, self
.report_error
)
1938 def _playlist_infodict(ie_result
, strict
=False, **kwargs
):
1940 'playlist_count': ie_result
.get('playlist_count'),
1941 'playlist': ie_result
.get('title') or ie_result
.get('id'),
1942 'playlist_id': ie_result
.get('id'),
1943 'playlist_title': ie_result
.get('title'),
1944 'playlist_uploader': ie_result
.get('uploader'),
1945 'playlist_uploader_id': ie_result
.get('uploader_id'),
1946 'playlist_channel': ie_result
.get('channel'),
1947 'playlist_channel_id': ie_result
.get('channel_id'),
1952 if ie_result
.get('webpage_url'):
1954 'webpage_url': ie_result
['webpage_url'],
1955 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1956 'webpage_url_domain': get_domain(ie_result
['webpage_url']),
1960 'playlist_index': 0,
1961 '__last_playlist_index': max(ie_result
.get('requested_entries') or (0, 0)),
1962 'extractor': ie_result
['extractor'],
1963 'extractor_key': ie_result
['extractor_key'],
1966 def __process_playlist(self
, ie_result
, download
):
1967 """Process each entry in the playlist"""
1968 assert ie_result
['_type'] in ('playlist', 'multi_video')
1970 common_info
= self
._playlist
_infodict
(ie_result
, strict
=True)
1971 title
= common_info
.get('playlist') or '<Untitled>'
1972 if self
._match
_entry
(common_info
, incomplete
=True) is not None:
1974 self
.to_screen(f
'[download] Downloading {ie_result["_type"]}: {title}')
1976 all_entries
= PlaylistEntries(self
, ie_result
)
1977 entries
= orderedSet(all_entries
.get_requested_items(), lazy
=True)
1979 lazy
= self
.params
.get('lazy_playlist')
1981 resolved_entries
, n_entries
= [], 'N/A'
1982 ie_result
['requested_entries'], ie_result
['entries'] = None, None
1984 entries
= resolved_entries
= list(entries
)
1985 n_entries
= len(resolved_entries
)
1986 ie_result
['requested_entries'], ie_result
['entries'] = tuple(zip(*resolved_entries
)) or ([], [])
1987 if not ie_result
.get('playlist_count'):
1988 # Better to do this after potentially exhausting entries
1989 ie_result
['playlist_count'] = all_entries
.get_full_count()
1991 extra
= self
._playlist
_infodict
(ie_result
, n_entries
=int_or_none(n_entries
))
1992 ie_copy
= collections
.ChainMap(ie_result
, extra
)
1994 _infojson_written
= False
1995 write_playlist_files
= self
.params
.get('allow_playlist_files', True)
1996 if write_playlist_files
and self
.params
.get('list_thumbnails'):
1997 self
.list_thumbnails(ie_result
)
1998 if write_playlist_files
and not self
.params
.get('simulate'):
1999 _infojson_written
= self
._write
_info
_json
(
2000 'playlist', ie_result
, self
.prepare_filename(ie_copy
, 'pl_infojson'))
2001 if _infojson_written
is None:
2003 if self
._write
_description
('playlist', ie_result
,
2004 self
.prepare_filename(ie_copy
, 'pl_description')) is None:
2006 # TODO: This should be passed to ThumbnailsConvertor if necessary
2007 self
._write
_thumbnails
('playlist', ie_result
, self
.prepare_filename(ie_copy
, 'pl_thumbnail'))
2010 if self
.params
.get('playlistreverse') or self
.params
.get('playlistrandom'):
2011 self
.report_warning('playlistreverse and playlistrandom are not supported with lazy_playlist', only_once
=True)
2012 elif self
.params
.get('playlistreverse'):
2014 elif self
.params
.get('playlistrandom'):
2015 random
.shuffle(entries
)
2017 self
.to_screen(f
'[{ie_result["extractor"]}] Playlist {title}: Downloading {n_entries} items'
2018 f
'{format_field(ie_result, "playlist_count", " of %s")}')
2020 keep_resolved_entries
= self
.params
.get('extract_flat') != 'discard'
2021 if self
.params
.get('extract_flat') == 'discard_in_playlist':
2022 keep_resolved_entries
= ie_result
['_type'] != 'playlist'
2023 if keep_resolved_entries
:
2024 self
.write_debug('The information of all playlist entries will be held in memory')
2027 max_failures
= self
.params
.get('skip_playlist_after_errors') or float('inf')
2028 for i
, (playlist_index
, entry
) in enumerate(entries
):
2030 resolved_entries
.append((playlist_index
, entry
))
2034 entry
['__x_forwarded_for_ip'] = ie_result
.get('__x_forwarded_for_ip')
2035 if not lazy
and 'playlist-index' in self
.params
['compat_opts']:
2036 playlist_index
= ie_result
['requested_entries'][i
]
2038 entry_copy
= collections
.ChainMap(entry
, {
2040 'n_entries': int_or_none(n_entries
),
2041 'playlist_index': playlist_index
,
2042 'playlist_autonumber': i
+ 1,
2045 if self
._match
_entry
(entry_copy
, incomplete
=True) is not None:
2046 # For compatabilty with youtube-dl. See https://github.com/yt-dlp/yt-dlp/issues/4369
2047 resolved_entries
[i
] = (playlist_index
, NO_DEFAULT
)
2051 f
'[download] Downloading item {self._format_screen(i + 1, self.Styles.ID)} '
2052 f
'of {self._format_screen(n_entries, self.Styles.EMPHASIS)}')
2054 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, collections
.ChainMap({
2055 'playlist_index': playlist_index
,
2056 'playlist_autonumber': i
+ 1,
2058 if not entry_result
:
2060 if failures
>= max_failures
:
2062 f
'Skipping the remaining entries in playlist "{title}" since {failures} items failed extraction')
2064 if keep_resolved_entries
:
2065 resolved_entries
[i
] = (playlist_index
, entry_result
)
2067 # Update with processed data
2068 ie_result
['entries'] = [e
for _
, e
in resolved_entries
if e
is not NO_DEFAULT
]
2069 ie_result
['requested_entries'] = [i
for i
, e
in resolved_entries
if e
is not NO_DEFAULT
]
2070 if ie_result
['requested_entries'] == try_call(lambda: list(range(1, ie_result
['playlist_count'] + 1))):
2071 # Do not set for full playlist
2072 ie_result
.pop('requested_entries')
2074 # Write the updated info to json
2075 if _infojson_written
is True and self
._write
_info
_json
(
2076 'updated playlist', ie_result
,
2077 self
.prepare_filename(ie_copy
, 'pl_infojson'), overwrite
=True) is None:
2080 ie_result
= self
.run_all_pps('playlist', ie_result
)
2081 self
.to_screen(f
'[download] Finished downloading playlist: {title}')
2084 @_handle_extraction_exceptions
2085 def __process_iterable_entry(self
, entry
, download
, extra_info
):
2086 return self
.process_ie_result(
2087 entry
, download
=download
, extra_info
=extra_info
)
2089 def _build_format_filter(self
, filter_spec
):
2090 " Returns a function to filter the formats according to the filter_spec "
2100 operator_rex
= re
.compile(r
'''(?x)\s*
2102 (?P<op>{})(?P<none_inclusive>\s*\?)?\s*
2103 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s*
2104 '''.format('|'.join(map(re
.escape
, OPERATORS
.keys()))))
2105 m
= operator_rex
.fullmatch(filter_spec
)
2108 comparison_value
= int(m
.group('value'))
2110 comparison_value
= parse_filesize(m
.group('value'))
2111 if comparison_value
is None:
2112 comparison_value
= parse_filesize(m
.group('value') + 'B')
2113 if comparison_value
is None:
2115 'Invalid value {!r} in format specification {!r}'.format(
2116 m
.group('value'), filter_spec
))
2117 op
= OPERATORS
[m
.group('op')]
2122 '^=': lambda attr
, value
: attr
.startswith(value
),
2123 '$=': lambda attr
, value
: attr
.endswith(value
),
2124 '*=': lambda attr
, value
: value
in attr
,
2125 '~=': lambda attr
, value
: value
.search(attr
) is not None,
2127 str_operator_rex
= re
.compile(r
'''(?x)\s*
2128 (?P<key>[a-zA-Z0-9._-]+)\s*
2129 (?P<negation>!\s*)?(?P<op>{})\s*(?P<none_inclusive>\?\s*)?
2131 (?P<value>(?(quote)(?:(?!(?P=quote))[^\\]|\\.)+|[\w.-]+))
2132 (?(quote)(?P=quote))\s*
2133 '''.format('|'.join(map(re
.escape
, STR_OPERATORS
.keys()))))
2134 m
= str_operator_rex
.fullmatch(filter_spec
)
2136 if m
.group('op') == '~=':
2137 comparison_value
= re
.compile(m
.group('value'))
2139 comparison_value
= re
.sub(r
'''\\([\\"'])''', r
'\1', m
.group('value'))
2140 str_op
= STR_OPERATORS
[m
.group('op')]
2141 if m
.group('negation'):
2142 op
= lambda attr
, value
: not str_op(attr
, value
)
2147 raise SyntaxError(f
'Invalid filter specification {filter_spec!r}')
2150 actual_value
= f
.get(m
.group('key'))
2151 if actual_value
is None:
2152 return m
.group('none_inclusive')
2153 return op(actual_value
, comparison_value
)
2156 def _check_formats(self
, formats
):
2158 working
= f
.get('__working')
2159 if working
is not None:
2163 self
.to_screen('[info] Testing format {}'.format(f
['format_id']))
2164 path
= self
.get_output_path('temp')
2165 if not self
._ensure
_dir
_exists
(f
'{path}/'):
2167 temp_file
= tempfile
.NamedTemporaryFile(suffix
='.tmp', delete
=False, dir=path
or None)
2170 success
, _
= self
.dl(temp_file
.name
, f
, test
=True)
2171 except (DownloadError
, OSError, ValueError, *network_exceptions
):
2174 if os
.path
.exists(temp_file
.name
):
2176 os
.remove(temp_file
.name
)
2178 self
.report_warning(f
'Unable to delete temporary file "{temp_file.name}"')
2179 f
['__working'] = success
2183 self
.to_screen('[info] Unable to download format {}. Skipping...'.format(f
['format_id']))
2185 def _select_formats(self
, formats
, selector
):
2186 return list(selector({
2188 'has_merged_format': any('none' not in (f
.get('acodec'), f
.get('vcodec')) for f
in formats
),
2189 'incomplete_formats': (all(f
.get('vcodec') == 'none' for f
in formats
) # No formats with video
2190 or all(f
.get('acodec') == 'none' for f
in formats
)), # OR, No formats with audio
2193 def _default_format_spec(self
, info_dict
, download
=True):
2194 download
= download
and not self
.params
.get('simulate')
2195 prefer_best
= download
and (
2196 self
.params
['outtmpl']['default'] == '-'
2197 or info_dict
.get('is_live') and not self
.params
.get('live_from_start'))
2200 merger
= FFmpegMergerPP(self
)
2201 return merger
.available
and merger
.can_merge()
2203 if not prefer_best
and download
and not can_merge():
2205 formats
= self
._get
_formats
(info_dict
)
2206 evaluate_formats
= lambda spec
: self
._select
_formats
(formats
, self
.build_format_selector(spec
))
2207 if evaluate_formats('b/bv+ba') != evaluate_formats('bv*+ba/b'):
2208 self
.report_warning('ffmpeg not found. The downloaded format may not be the best available. '
2209 'Installing ffmpeg is strongly recommended: https://github.com/yt-dlp/yt-dlp#dependencies')
2211 compat
= (self
.params
.get('allow_multiple_audio_streams')
2212 or 'format-spec' in self
.params
['compat_opts'])
2214 return ('best/bestvideo+bestaudio' if prefer_best
2215 else 'bestvideo+bestaudio/best' if compat
2216 else 'bestvideo*+bestaudio/best')
2218 def build_format_selector(self
, format_spec
):
2219 def syntax_error(note
, start
):
2221 'Invalid format specification: '
2222 '{}\n\t{}\n\t{}^'.format(note
, format_spec
, ' ' * start
[1]))
2223 return SyntaxError(message
)
2225 PICKFIRST
= 'PICKFIRST'
2229 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
2231 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
2232 'video': self
.params
.get('allow_multiple_video_streams', False)}
2234 def _parse_filter(tokens
):
2236 for type_
, string_
, _start
, _
, _
in tokens
:
2237 if type_
== tokenize
.OP
and string_
== ']':
2238 return ''.join(filter_parts
)
2240 filter_parts
.append(string_
)
2242 def _remove_unused_ops(tokens
):
2243 # Remove operators that we don't use and join them with the surrounding strings.
2244 # E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
2245 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
2246 last_string
, last_start
, last_end
, last_line
= None, None, None, None
2247 for type_
, string_
, start
, end
, line
in tokens
:
2248 if type_
== tokenize
.OP
and string_
== '[':
2250 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
2252 yield type_
, string_
, start
, end
, line
2253 # everything inside brackets will be handled by _parse_filter
2254 for type_
, string_
, start
, end
, line
in tokens
:
2255 yield type_
, string_
, start
, end
, line
2256 if type_
== tokenize
.OP
and string_
== ']':
2258 elif type_
== tokenize
.OP
and string_
in ALLOWED_OPS
:
2260 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
2262 yield type_
, string_
, start
, end
, line
2263 elif type_
in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
2265 last_string
= string_
2269 last_string
+= string_
2271 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
2273 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
2275 current_selector
= None
2276 for type_
, string_
, start
, _
, _
in tokens
:
2277 # ENCODING is only defined in Python 3.x
2278 if type_
== getattr(tokenize
, 'ENCODING', None):
2280 elif type_
in [tokenize
.NAME
, tokenize
.NUMBER
]:
2281 current_selector
= FormatSelector(SINGLE
, string_
, [])
2282 elif type_
== tokenize
.OP
:
2284 if not inside_group
:
2285 # ')' will be handled by the parentheses group
2286 tokens
.restore_last_token()
2288 elif inside_merge
and string_
in ['/', ',']:
2289 tokens
.restore_last_token()
2291 elif inside_choice
and string_
== ',':
2292 tokens
.restore_last_token()
2294 elif string_
== ',':
2295 if not current_selector
:
2296 raise syntax_error('"," must follow a format selector', start
)
2297 selectors
.append(current_selector
)
2298 current_selector
= None
2299 elif string_
== '/':
2300 if not current_selector
:
2301 raise syntax_error('"/" must follow a format selector', start
)
2302 first_choice
= current_selector
2303 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
2304 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
2305 elif string_
== '[':
2306 if not current_selector
:
2307 current_selector
= FormatSelector(SINGLE
, 'best', [])
2308 format_filter
= _parse_filter(tokens
)
2309 current_selector
.filters
.append(format_filter
)
2310 elif string_
== '(':
2311 if current_selector
:
2312 raise syntax_error('Unexpected "("', start
)
2313 group
= _parse_format_selection(tokens
, inside_group
=True)
2314 current_selector
= FormatSelector(GROUP
, group
, [])
2315 elif string_
== '+':
2316 if not current_selector
:
2317 raise syntax_error('Unexpected "+"', start
)
2318 selector_1
= current_selector
2319 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
2321 raise syntax_error('Expected a selector', start
)
2322 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
2324 raise syntax_error(f
'Operator not recognized: "{string_}"', start
)
2325 elif type_
== tokenize
.ENDMARKER
:
2327 if current_selector
:
2328 selectors
.append(current_selector
)
2331 def _merge(formats_pair
):
2332 format_1
, format_2
= formats_pair
2335 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
2336 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
2338 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
2339 get_no_more
= {'video': False, 'audio': False}
2340 for (i
, fmt_info
) in enumerate(formats_info
):
2341 if fmt_info
.get('acodec') == fmt_info
.get('vcodec') == 'none':
2344 for aud_vid
in ['audio', 'video']:
2345 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
2346 if get_no_more
[aud_vid
]:
2349 get_no_more
[aud_vid
] = True
2351 if len(formats_info
) == 1:
2352 return formats_info
[0]
2354 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
2355 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
2357 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
2358 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
2360 output_ext
= get_compatible_ext(
2361 vcodecs
=[f
.get('vcodec') for f
in video_fmts
],
2362 acodecs
=[f
.get('acodec') for f
in audio_fmts
],
2363 vexts
=[f
['ext'] for f
in video_fmts
],
2364 aexts
=[f
['ext'] for f
in audio_fmts
],
2365 preferences
=(try_call(lambda: self
.params
['merge_output_format'].split('/'))
2366 or self
.params
.get('prefer_free_formats') and ('webm', 'mkv')))
2368 filtered
= lambda *keys
: filter(None, (traverse_obj(fmt
, *keys
) for fmt
in formats_info
))
2371 'requested_formats': formats_info
,
2372 'format': '+'.join(filtered('format')),
2373 'format_id': '+'.join(filtered('format_id')),
2375 'protocol': '+'.join(map(determine_protocol
, formats_info
)),
2376 'language': '+'.join(orderedSet(filtered('language'))) or None,
2377 'format_note': '+'.join(orderedSet(filtered('format_note'))) or None,
2378 'filesize_approx': sum(filtered('filesize', 'filesize_approx')) or None,
2379 'tbr': sum(filtered('tbr', 'vbr', 'abr')),
2384 'width': the_only_video
.get('width'),
2385 'height': the_only_video
.get('height'),
2386 'resolution': the_only_video
.get('resolution') or self
.format_resolution(the_only_video
),
2387 'fps': the_only_video
.get('fps'),
2388 'dynamic_range': the_only_video
.get('dynamic_range'),
2389 'vcodec': the_only_video
.get('vcodec'),
2390 'vbr': the_only_video
.get('vbr'),
2391 'stretched_ratio': the_only_video
.get('stretched_ratio'),
2392 'aspect_ratio': the_only_video
.get('aspect_ratio'),
2397 'acodec': the_only_audio
.get('acodec'),
2398 'abr': the_only_audio
.get('abr'),
2399 'asr': the_only_audio
.get('asr'),
2400 'audio_channels': the_only_audio
.get('audio_channels'),
2405 def _check_formats(formats
):
2406 if self
.params
.get('check_formats') == 'selected':
2407 yield from self
._check
_formats
(formats
)
2409 elif (self
.params
.get('check_formats') is not None
2410 or self
.params
.get('allow_unplayable_formats')):
2415 if f
.get('has_drm') or f
.get('__needs_testing'):
2416 yield from self
._check
_formats
([f
])
2420 def _build_selector_function(selector
):
2421 if isinstance(selector
, list): # ,
2422 fs
= [_build_selector_function(s
) for s
in selector
]
2424 def selector_function(ctx
):
2427 return selector_function
2429 elif selector
.type == GROUP
: # ()
2430 selector_function
= _build_selector_function(selector
.selector
)
2432 elif selector
.type == PICKFIRST
: # /
2433 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
2435 def selector_function(ctx
):
2437 picked_formats
= list(f(ctx
))
2439 return picked_formats
2442 elif selector
.type == MERGE
: # +
2443 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
2445 def selector_function(ctx
):
2446 for pair
in itertools
.product(selector_1(ctx
), selector_2(ctx
)):
2449 elif selector
.type == SINGLE
: # atom
2450 format_spec
= selector
.selector
or 'best'
2452 # TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
2453 if format_spec
== 'all':
2454 def selector_function(ctx
):
2455 yield from _check_formats(ctx
['formats'][::-1])
2456 elif format_spec
== 'mergeall':
2457 def selector_function(ctx
):
2458 formats
= list(_check_formats(
2459 f
for f
in ctx
['formats'] if f
.get('vcodec') != 'none' or f
.get('acodec') != 'none'))
2462 merged_format
= formats
[-1]
2463 for f
in formats
[-2::-1]:
2464 merged_format
= _merge((merged_format
, f
))
2468 format_fallback
, seperate_fallback
, format_reverse
, format_idx
= False, None, True, 1
2470 r
'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
2472 if mobj
is not None:
2473 format_idx
= int_or_none(mobj
.group('n'), default
=1)
2474 format_reverse
= mobj
.group('bw')[0] == 'b'
2475 format_type
= (mobj
.group('type') or [None])[0]
2476 not_format_type
= {'v': 'a', 'a': 'v'}
.get(format_type
)
2477 format_modified
= mobj
.group('mod') is not None
2479 format_fallback
= not format_type
and not format_modified
# for b, w
2481 (lambda f
: f
.get(f
'{format_type}codec') != 'none')
2482 if format_type
and format_modified
# bv*, ba*, wv*, wa*
2483 else (lambda f
: f
.get(f
'{not_format_type}codec') == 'none')
2484 if format_type
# bv, ba, wv, wa
2485 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
2486 if not format_modified
# b, w
2487 else lambda f
: True) # b*, w*
2488 filter_f
= lambda f
: _filter_f(f
) and (
2489 f
.get('vcodec') != 'none' or f
.get('acodec') != 'none')
2491 if format_spec
in self
._format
_selection
_exts
['audio']:
2492 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none'
2493 elif format_spec
in self
._format
_selection
_exts
['video']:
2494 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none' and f
.get('vcodec') != 'none'
2495 seperate_fallback
= lambda f
: f
.get('ext') == format_spec
and f
.get('vcodec') != 'none'
2496 elif format_spec
in self
._format
_selection
_exts
['storyboards']:
2497 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') == 'none' and f
.get('vcodec') == 'none'
2499 filter_f
= lambda f
: f
.get('format_id') == format_spec
# id
2501 def selector_function(ctx
):
2502 formats
= list(ctx
['formats'])
2503 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
2505 if format_fallback
and ctx
['incomplete_formats']:
2506 # for extractors with incomplete formats (audio only (soundcloud)
2507 # or video only (imgur)) best/worst will fallback to
2508 # best/worst {video,audio}-only format
2509 matches
= list(filter(lambda f
: f
.get('vcodec') != 'none' or f
.get('acodec') != 'none', formats
))
2510 elif seperate_fallback
and not ctx
['has_merged_format']:
2511 # for compatibility with youtube-dl when there is no pre-merged format
2512 matches
= list(filter(seperate_fallback
, formats
))
2513 matches
= LazyList(_check_formats(matches
[::-1 if format_reverse
else 1]))
2515 yield matches
[format_idx
- 1]
2516 except LazyList
.IndexError:
2519 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
2521 def final_selector(ctx
):
2522 ctx_copy
= dict(ctx
)
2523 for _filter
in filters
:
2524 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
2525 return selector_function(ctx_copy
)
2526 return final_selector
2528 # HACK: Python 3.12 changed the underlying parser, rendering '7_a' invalid
2529 # Prefix numbers with random letters to avoid it being classified as a number
2530 # See: https://github.com/yt-dlp/yt-dlp/pulls/8797
2531 # TODO: Implement parser not reliant on tokenize.tokenize
2532 prefix
= ''.join(random
.choices(string
.ascii_letters
, k
=32))
2533 stream
= io
.BytesIO(re
.sub(r
'\d[_\d]*', rf
'{prefix}\g<0>', format_spec
).encode())
2535 tokens
= list(_remove_unused_ops(
2536 token
._replace
(string
=token
.string
.replace(prefix
, ''))
2537 for token
in tokenize
.tokenize(stream
.readline
)))
2538 except tokenize
.TokenError
:
2539 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
2541 class TokenIterator
:
2542 def __init__(self
, tokens
):
2543 self
.tokens
= tokens
2550 if self
.counter
>= len(self
.tokens
):
2552 value
= self
.tokens
[self
.counter
]
2558 def restore_last_token(self
):
2561 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
2562 return _build_selector_function(parsed_selector
)
2564 def _calc_headers(self
, info_dict
, load_cookies
=False):
2565 res
= HTTPHeaderDict(self
.params
['http_headers'], info_dict
.get('http_headers'))
2568 if load_cookies
: # For --load-info-json
2569 self
._load
_cookies
(res
.get('Cookie'), autoscope
=info_dict
['url']) # compat
2570 self
._load
_cookies
(info_dict
.get('cookies'), autoscope
=False)
2571 # The `Cookie` header is removed to prevent leaks and unscoped cookies.
2572 # See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
2573 res
.pop('Cookie', None)
2574 cookies
= self
.cookiejar
.get_cookies_for_url(info_dict
['url'])
2576 encoder
= LenientSimpleCookie()
2578 for cookie
in cookies
:
2579 _
, value
= encoder
.value_encode(cookie
.value
)
2580 values
.append(f
'{cookie.name}={value}')
2582 values
.append(f
'Domain={cookie.domain}')
2584 values
.append(f
'Path={cookie.path}')
2586 values
.append('Secure')
2588 values
.append(f
'Expires={cookie.expires}')
2590 values
.append(f
'Version={cookie.version}')
2591 info_dict
['cookies'] = '; '.join(values
)
2593 if 'X-Forwarded-For' not in res
:
2594 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
2595 if x_forwarded_for_ip
:
2596 res
['X-Forwarded-For'] = x_forwarded_for_ip
2600 def _calc_cookies(self
, url
):
2601 self
.deprecation_warning('"YoutubeDL._calc_cookies" is deprecated and may be removed in a future version')
2602 return self
.cookiejar
.get_cookie_header(url
)
2604 def _sort_thumbnails(self
, thumbnails
):
2605 thumbnails
.sort(key
=lambda t
: (
2606 t
.get('preference') if t
.get('preference') is not None else -1,
2607 t
.get('width') if t
.get('width') is not None else -1,
2608 t
.get('height') if t
.get('height') is not None else -1,
2609 t
.get('id') if t
.get('id') is not None else '',
2612 def _sanitize_thumbnails(self
, info_dict
):
2613 thumbnails
= info_dict
.get('thumbnails')
2614 if thumbnails
is None:
2615 thumbnail
= info_dict
.get('thumbnail')
2617 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
2621 def check_thumbnails(thumbnails
):
2622 for t
in thumbnails
:
2623 self
.to_screen(f
'[info] Testing thumbnail {t["id"]}')
2625 self
.urlopen(HEADRequest(t
['url']))
2626 except network_exceptions
as err
:
2627 self
.to_screen(f
'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...')
2631 self
._sort
_thumbnails
(thumbnails
)
2632 for i
, t
in enumerate(thumbnails
):
2633 if t
.get('id') is None:
2635 if t
.get('width') and t
.get('height'):
2636 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
2637 t
['url'] = sanitize_url(t
['url'])
2639 if self
.params
.get('check_formats') is True:
2640 info_dict
['thumbnails'] = LazyList(check_thumbnails(thumbnails
[::-1]), reverse
=True)
2642 info_dict
['thumbnails'] = thumbnails
2644 def _fill_common_fields(self
, info_dict
, final
=True):
2645 # TODO: move sanitization here
2647 title
= info_dict
['fulltitle'] = info_dict
.get('title')
2650 self
.write_debug('Extractor gave empty title. Creating a generic title')
2652 self
.report_warning('Extractor failed to obtain "title". Creating a generic title instead')
2653 info_dict
['title'] = f
'{info_dict["extractor"].replace(":", "-")} video #{info_dict["id"]}'
2655 if info_dict
.get('duration') is not None:
2656 info_dict
['duration_string'] = formatSeconds(info_dict
['duration'])
2658 for ts_key
, date_key
in (
2659 ('timestamp', 'upload_date'),
2660 ('release_timestamp', 'release_date'),
2661 ('modified_timestamp', 'modified_date'),
2663 if info_dict
.get(date_key
) is None and info_dict
.get(ts_key
) is not None:
2664 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
2665 # see http://bugs.python.org/issue1646728)
2666 with contextlib
.suppress(ValueError, OverflowError, OSError):
2667 upload_date
= dt
.datetime
.fromtimestamp(info_dict
[ts_key
], dt
.timezone
.utc
)
2668 info_dict
[date_key
] = upload_date
.strftime('%Y%m%d')
2670 if not info_dict
.get('release_year'):
2671 info_dict
['release_year'] = traverse_obj(info_dict
, ('release_date', {lambda x: int(x[:4])}
))
2673 live_keys
= ('is_live', 'was_live')
2674 live_status
= info_dict
.get('live_status')
2675 if live_status
is None:
2676 for key
in live_keys
:
2677 if info_dict
.get(key
) is False:
2679 if info_dict
.get(key
):
2682 if all(info_dict
.get(key
) is False for key
in live_keys
):
2683 live_status
= 'not_live'
2685 info_dict
['live_status'] = live_status
2686 for key
in live_keys
:
2687 if info_dict
.get(key
) is None:
2688 info_dict
[key
] = (live_status
== key
)
2689 if live_status
== 'post_live':
2690 info_dict
['was_live'] = True
2692 # Auto generate title fields corresponding to the *_number fields when missing
2693 # in order to always have clean titles. This is very common for TV series.
2694 for field
in ('chapter', 'season', 'episode'):
2695 if final
and info_dict
.get(f
'{field}_number') is not None and not info_dict
.get(field
):
2696 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
[f
'{field}_number'])
2698 for old_key
, new_key
in self
._deprecated
_multivalue
_fields
.items():
2699 if new_key
in info_dict
and old_key
in info_dict
:
2700 if '_version' not in info_dict
: # HACK: Do not warn when using --load-info-json
2701 self
.deprecation_warning(f
'Do not return {old_key!r} when {new_key!r} is present')
2702 elif old_value
:= info_dict
.get(old_key
):
2703 info_dict
[new_key
] = old_value
.split(', ')
2704 elif new_value
:= info_dict
.get(new_key
):
2705 info_dict
[old_key
] = ', '.join(v
.replace(',', '\N{FULLWIDTH COMMA}') for v
in new_value
)
2707 def _raise_pending_errors(self
, info
):
2708 err
= info
.pop('__pending_error', None)
2710 self
.report_error(err
, tb
=False)
2712 def sort_formats(self
, info_dict
):
2713 formats
= self
._get
_formats
(info_dict
)
2714 formats
.sort(key
=FormatSorter(
2715 self
, info_dict
.get('_format_sort_fields') or []).calculate_preference
)
2717 def process_video_result(self
, info_dict
, download
=True):
2718 assert info_dict
.get('_type', 'video') == 'video'
2719 self
._num
_videos
+= 1
2721 if 'id' not in info_dict
:
2722 raise ExtractorError('Missing "id" field in extractor result', ie
=info_dict
['extractor'])
2723 elif not info_dict
.get('id'):
2724 raise ExtractorError('Extractor failed to obtain "id"', ie
=info_dict
['extractor'])
2726 def report_force_conversion(field
, field_not
, conversion
):
2727 self
.report_warning(
2728 f
'"{field}" field is not {field_not} - forcing {conversion} conversion, '
2729 'there is an error in extractor')
2731 def sanitize_string_field(info
, string_field
):
2732 field
= info
.get(string_field
)
2733 if field
is None or isinstance(field
, str):
2735 report_force_conversion(string_field
, 'a string', 'string')
2736 info
[string_field
] = str(field
)
2738 def sanitize_numeric_fields(info
):
2739 for numeric_field
in self
._NUMERIC
_FIELDS
:
2740 field
= info
.get(numeric_field
)
2741 if field
is None or isinstance(field
, (int, float)):
2743 report_force_conversion(numeric_field
, 'numeric', 'int')
2744 info
[numeric_field
] = int_or_none(field
)
2746 sanitize_string_field(info_dict
, 'id')
2747 sanitize_numeric_fields(info_dict
)
2748 if info_dict
.get('section_end') and info_dict
.get('section_start') is not None:
2749 info_dict
['duration'] = round(info_dict
['section_end'] - info_dict
['section_start'], 3)
2750 if (info_dict
.get('duration') or 0) <= 0 and info_dict
.pop('duration', None):
2751 self
.report_warning('"duration" field is negative, there is an error in extractor')
2753 chapters
= info_dict
.get('chapters') or []
2754 if chapters
and chapters
[0].get('start_time'):
2755 chapters
.insert(0, {'start_time': 0}
)
2757 dummy_chapter
= {'end_time': 0, 'start_time': info_dict.get('duration')}
2758 for idx
, (prev
, current
, next_
) in enumerate(zip(
2759 (dummy_chapter
, *chapters
), chapters
, (*chapters
[1:], dummy_chapter
)), 1):
2760 if current
.get('start_time') is None:
2761 current
['start_time'] = prev
.get('end_time')
2762 if not current
.get('end_time'):
2763 current
['end_time'] = next_
.get('start_time')
2764 if not current
.get('title'):
2765 current
['title'] = f
'<Untitled Chapter {idx}>'
2767 if 'playlist' not in info_dict
:
2768 # It isn't part of a playlist
2769 info_dict
['playlist'] = None
2770 info_dict
['playlist_index'] = None
2772 self
._sanitize
_thumbnails
(info_dict
)
2774 thumbnail
= info_dict
.get('thumbnail')
2775 thumbnails
= info_dict
.get('thumbnails')
2777 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
2779 info_dict
['thumbnail'] = thumbnails
[-1]['url']
2781 if info_dict
.get('display_id') is None and 'id' in info_dict
:
2782 info_dict
['display_id'] = info_dict
['id']
2784 self
._fill
_common
_fields
(info_dict
)
2786 for cc_kind
in ('subtitles', 'automatic_captions'):
2787 cc
= info_dict
.get(cc_kind
)
2789 for _
, subtitle
in cc
.items():
2790 for subtitle_format
in subtitle
:
2791 if subtitle_format
.get('url'):
2792 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
2793 if subtitle_format
.get('ext') is None:
2794 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
2796 automatic_captions
= info_dict
.get('automatic_captions')
2797 subtitles
= info_dict
.get('subtitles')
2799 info_dict
['requested_subtitles'] = self
.process_subtitles(
2800 info_dict
['id'], subtitles
, automatic_captions
)
2802 formats
= self
._get
_formats
(info_dict
)
2804 # Backward compatibility with InfoExtractor._sort_formats
2805 field_preference
= (formats
or [{}])[0].pop('__sort_fields', None)
2806 if field_preference
:
2807 info_dict
['_format_sort_fields'] = field_preference
2809 info_dict
['_has_drm'] = any( # or None ensures --clean-infojson removes it
2810 f
.get('has_drm') and f
['has_drm'] != 'maybe' for f
in formats
) or None
2811 if not self
.params
.get('allow_unplayable_formats'):
2812 formats
= [f
for f
in formats
if not f
.get('has_drm') or f
['has_drm'] == 'maybe']
2814 if formats
and all(f
.get('acodec') == f
.get('vcodec') == 'none' for f
in formats
):
2815 self
.report_warning(
2816 f
'{"This video is DRM protected and " if info_dict["_has_drm"] else ""}'
2817 'only images are available for download. Use --list-formats to see them'.capitalize())
2819 get_from_start
= not info_dict
.get('is_live') or bool(self
.params
.get('live_from_start'))
2820 if not get_from_start
:
2821 info_dict
['title'] += ' ' + dt
.datetime
.now().strftime('%Y-%m-%d %H:%M')
2822 if info_dict
.get('is_live') and formats
:
2823 formats
= [f
for f
in formats
if bool(f
.get('is_from_start')) == get_from_start
]
2824 if get_from_start
and not formats
:
2825 self
.raise_no_formats(info_dict
, msg
=(
2826 '--live-from-start is passed, but there are no formats that can be downloaded from the start. '
2827 'If you want to download from the current time, use --no-live-from-start'))
2829 def is_wellformed(f
):
2832 self
.report_warning(
2833 '"url" field is missing or empty - skipping format, '
2834 'there is an error in extractor')
2836 if isinstance(url
, bytes):
2837 sanitize_string_field(f
, 'url')
2840 # Filter out malformed formats for better extraction robustness
2841 formats
= list(filter(is_wellformed
, formats
or []))
2844 self
.raise_no_formats(info_dict
)
2847 sanitize_string_field(fmt
, 'format_id')
2848 sanitize_numeric_fields(fmt
)
2849 fmt
['url'] = sanitize_url(fmt
['url'])
2850 if fmt
.get('ext') is None:
2851 fmt
['ext'] = determine_ext(fmt
['url']).lower()
2852 if fmt
['ext'] in ('aac', 'opus', 'mp3', 'flac', 'vorbis'):
2853 if fmt
.get('acodec') is None:
2854 fmt
['acodec'] = fmt
['ext']
2855 if fmt
.get('protocol') is None:
2856 fmt
['protocol'] = determine_protocol(fmt
)
2857 if fmt
.get('resolution') is None:
2858 fmt
['resolution'] = self
.format_resolution(fmt
, default
=None)
2859 if fmt
.get('dynamic_range') is None and fmt
.get('vcodec') != 'none':
2860 fmt
['dynamic_range'] = 'SDR'
2861 if fmt
.get('aspect_ratio') is None:
2862 fmt
['aspect_ratio'] = try_call(lambda: round(fmt
['width'] / fmt
['height'], 2))
2863 # For fragmented formats, "tbr" is often max bitrate and not average
2864 if (('manifest-filesize-approx' in self
.params
['compat_opts'] or not fmt
.get('manifest_url'))
2865 and not fmt
.get('filesize') and not fmt
.get('filesize_approx')):
2866 fmt
['filesize_approx'] = filesize_from_tbr(fmt
.get('tbr'), info_dict
.get('duration'))
2867 fmt
['http_headers'] = self
._calc
_headers
(collections
.ChainMap(fmt
, info_dict
), load_cookies
=True)
2869 # Safeguard against old/insecure infojson when using --load-info-json
2870 if info_dict
.get('http_headers'):
2871 info_dict
['http_headers'] = HTTPHeaderDict(info_dict
['http_headers'])
2872 info_dict
['http_headers'].pop('Cookie', None)
2874 # This is copied to http_headers by the above _calc_headers and can now be removed
2875 if '__x_forwarded_for_ip' in info_dict
:
2876 del info_dict
['__x_forwarded_for_ip']
2880 '_format_sort_fields': info_dict
.get('_format_sort_fields'),
2883 # Sanitize and group by format_id
2885 for i
, fmt
in enumerate(formats
):
2886 if not fmt
.get('format_id'):
2887 fmt
['format_id'] = str(i
)
2889 # Sanitize format_id from characters used in format selector expression
2890 fmt
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', fmt
['format_id'])
2891 formats_dict
.setdefault(fmt
['format_id'], []).append(fmt
)
2893 # Make sure all formats have unique format_id
2894 common_exts
= set(itertools
.chain(*self
._format
_selection
_exts
.values()))
2895 for format_id
, ambiguous_formats
in formats_dict
.items():
2896 ambigious_id
= len(ambiguous_formats
) > 1
2897 for i
, fmt
in enumerate(ambiguous_formats
):
2899 fmt
['format_id'] = f
'{format_id}-{i}'
2900 # Ensure there is no conflict between id and ext in format selection
2901 # See https://github.com/yt-dlp/yt-dlp/issues/1282
2902 if fmt
['format_id'] != fmt
['ext'] and fmt
['format_id'] in common_exts
:
2903 fmt
['format_id'] = 'f{}'.format(fmt
['format_id'])
2905 if fmt
.get('format') is None:
2906 fmt
['format'] = '{id} - {res}{note}'.format(
2907 id=fmt
['format_id'],
2908 res
=self
.format_resolution(fmt
),
2909 note
=format_field(fmt
, 'format_note', ' (%s)'),
2912 if self
.params
.get('check_formats') is True:
2913 formats
= LazyList(self
._check
_formats
(formats
[::-1]), reverse
=True)
2915 if not formats
or formats
[0] is not info_dict
:
2916 # only set the 'formats' fields if the original info_dict list them
2917 # otherwise we end up with a circular reference, the first (and unique)
2918 # element in the 'formats' field in info_dict is info_dict itself,
2919 # which can't be exported to json
2920 info_dict
['formats'] = formats
2922 info_dict
, _
= self
.pre_process(info_dict
)
2924 if self
._match
_entry
(info_dict
, incomplete
=self
._format
_fields
) is not None:
2927 self
.post_extract(info_dict
)
2928 info_dict
, _
= self
.pre_process(info_dict
, 'after_filter')
2930 # The pre-processors may have modified the formats
2931 formats
= self
._get
_formats
(info_dict
)
2933 list_only
= self
.params
.get('simulate') == 'list_only'
2934 interactive_format_selection
= not list_only
and self
.format_selector
== '-'
2935 if self
.params
.get('list_thumbnails'):
2936 self
.list_thumbnails(info_dict
)
2937 if self
.params
.get('listsubtitles'):
2938 if 'automatic_captions' in info_dict
:
2939 self
.list_subtitles(
2940 info_dict
['id'], automatic_captions
, 'automatic captions')
2941 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
2942 if self
.params
.get('listformats') or interactive_format_selection
:
2943 self
.list_formats(info_dict
)
2945 # Without this printing, -F --print-json will not work
2946 self
.__forced
_printings
(info_dict
)
2949 format_selector
= self
.format_selector
2951 if interactive_format_selection
:
2952 req_format
= input(self
._format
_screen
('\nEnter format selector ', self
.Styles
.EMPHASIS
)
2953 + '(Press ENTER for default, or Ctrl+C to quit)'
2954 + self
._format
_screen
(': ', self
.Styles
.EMPHASIS
))
2956 format_selector
= self
.build_format_selector(req_format
) if req_format
else None
2957 except SyntaxError as err
:
2958 self
.report_error(err
, tb
=False, is_error
=False)
2961 if format_selector
is None:
2962 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
2963 self
.write_debug(f
'Default format spec: {req_format}')
2964 format_selector
= self
.build_format_selector(req_format
)
2966 formats_to_download
= self
._select
_formats
(formats
, format_selector
)
2967 if interactive_format_selection
and not formats_to_download
:
2968 self
.report_error('Requested format is not available', tb
=False, is_error
=False)
2972 if not formats_to_download
:
2973 if not self
.params
.get('ignore_no_formats_error'):
2974 raise ExtractorError(
2975 'Requested format is not available. Use --list-formats for a list of available formats',
2976 expected
=True, video_id
=info_dict
['id'], ie
=info_dict
['extractor'])
2977 self
.report_warning('Requested format is not available')
2978 # Process what we can, even without any available formats.
2979 formats_to_download
= [{}]
2981 requested_ranges
= tuple(self
.params
.get('download_ranges', lambda *_
: [{}])(info_dict
, self
))
2982 best_format
, downloaded_formats
= formats_to_download
[-1], []
2984 if best_format
and requested_ranges
:
2985 def to_screen(*msg
):
2986 self
.to_screen(f
'[info] {info_dict["id"]}: {" ".join(", ".join(variadic(m)) for m in msg)}')
2988 to_screen(f
'Downloading {len(formats_to_download)} format(s):',
2989 (f
['format_id'] for f
in formats_to_download
))
2990 if requested_ranges
!= ({}, ):
2991 to_screen(f
'Downloading {len(requested_ranges)} time ranges:',
2992 (f
'{c["start_time"]:.1f}-{c["end_time"]:.1f}' for c
in requested_ranges
))
2993 max_downloads_reached
= False
2995 for fmt
, chapter
in itertools
.product(formats_to_download
, requested_ranges
):
2996 new_info
= self
._copy
_infodict
(info_dict
)
2997 new_info
.update(fmt
)
2998 offset
, duration
= info_dict
.get('section_start') or 0, info_dict
.get('duration') or float('inf')
2999 end_time
= offset
+ min(chapter
.get('end_time', duration
), duration
)
3000 # duration may not be accurate. So allow deviations <1sec
3001 if end_time
== float('inf') or end_time
> offset
+ duration
+ 1:
3003 if chapter
or offset
:
3005 'section_start': offset
+ chapter
.get('start_time', 0),
3006 'section_end': end_time
,
3007 'section_title': chapter
.get('title'),
3008 'section_number': chapter
.get('index'),
3010 downloaded_formats
.append(new_info
)
3012 self
.process_info(new_info
)
3013 except MaxDownloadsReached
:
3014 max_downloads_reached
= True
3015 self
._raise
_pending
_errors
(new_info
)
3016 # Remove copied info
3017 for key
, val
in tuple(new_info
.items()):
3018 if info_dict
.get(key
) == val
:
3020 if max_downloads_reached
:
3023 write_archive
= {f.get('__write_download_archive', False) for f in downloaded_formats}
3024 assert write_archive
.issubset({True, False, 'ignore'}
)
3025 if True in write_archive
and False not in write_archive
:
3026 self
.record_download_archive(info_dict
)
3028 info_dict
['requested_downloads'] = downloaded_formats
3029 info_dict
= self
.run_all_pps('after_video', info_dict
)
3030 if max_downloads_reached
:
3031 raise MaxDownloadsReached
3033 # We update the info dict with the selected best quality format (backwards compatibility)
3034 info_dict
.update(best_format
)
3037 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
3038 """Select the requested subtitles and their format"""
3039 available_subs
, normal_sub_langs
= {}, []
3040 if normal_subtitles
and self
.params
.get('writesubtitles'):
3041 available_subs
.update(normal_subtitles
)
3042 normal_sub_langs
= tuple(normal_subtitles
.keys())
3043 if automatic_captions
and self
.params
.get('writeautomaticsub'):
3044 for lang
, cap_info
in automatic_captions
.items():
3045 if lang
not in available_subs
:
3046 available_subs
[lang
] = cap_info
3048 if not available_subs
or (
3049 not self
.params
.get('writesubtitles')
3050 and not self
.params
.get('writeautomaticsub')):
3053 all_sub_langs
= tuple(available_subs
.keys())
3054 if self
.params
.get('allsubtitles', False):
3055 requested_langs
= all_sub_langs
3056 elif self
.params
.get('subtitleslangs', False):
3058 requested_langs
= orderedSet_from_options(
3059 self
.params
.get('subtitleslangs'), {'all': all_sub_langs}
, use_regex
=True)
3060 except re
.error
as e
:
3061 raise ValueError(f
'Wrong regex for subtitlelangs: {e.pattern}')
3063 requested_langs
= LazyList(itertools
.chain(
3064 ['en'] if 'en' in normal_sub_langs
else [],
3065 filter(lambda f
: f
.startswith('en'), normal_sub_langs
),
3066 ['en'] if 'en' in all_sub_langs
else [],
3067 filter(lambda f
: f
.startswith('en'), all_sub_langs
),
3068 normal_sub_langs
, all_sub_langs
,
3071 self
.to_screen(f
'[info] {video_id}: Downloading subtitles: {", ".join(requested_langs)}')
3073 formats_query
= self
.params
.get('subtitlesformat', 'best')
3074 formats_preference
= formats_query
.split('/') if formats_query
else []
3076 for lang
in requested_langs
:
3077 formats
= available_subs
.get(lang
)
3079 self
.report_warning(f
'{lang} subtitles not available for {video_id}')
3081 for ext
in formats_preference
:
3085 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
3091 self
.report_warning(
3092 'No subtitle format found matching "{}" for language {}, '
3093 'using {}. Use --list-subs for a list of available subtitles'.format(formats_query
, lang
, f
['ext']))
3097 def _forceprint(self
, key
, info_dict
):
3098 if info_dict
is None:
3100 info_copy
= info_dict
.copy()
3101 info_copy
.setdefault('filename', self
.prepare_filename(info_dict
))
3102 if info_dict
.get('requested_formats') is not None:
3103 # For RTMP URLs, also include the playpath
3104 info_copy
['urls'] = '\n'.join(f
['url'] + f
.get('play_path', '') for f
in info_dict
['requested_formats'])
3105 elif info_dict
.get('url'):
3106 info_copy
['urls'] = info_dict
['url'] + info_dict
.get('play_path', '')
3107 info_copy
['formats_table'] = self
.render_formats_table(info_dict
)
3108 info_copy
['thumbnails_table'] = self
.render_thumbnails_table(info_dict
)
3109 info_copy
['subtitles_table'] = self
.render_subtitles_table(info_dict
.get('id'), info_dict
.get('subtitles'))
3110 info_copy
['automatic_captions_table'] = self
.render_subtitles_table(info_dict
.get('id'), info_dict
.get('automatic_captions'))
3112 def format_tmpl(tmpl
):
3113 mobj
= re
.fullmatch(r
'([\w.:,]|-\d|(?P<dict>{([\w.:,]|-\d)+}))+=?', tmpl
)
3118 if tmpl
.startswith('{'):
3119 tmpl
, fmt
= f
'.{tmpl}', '%({})j'
3120 if tmpl
.endswith('='):
3121 tmpl
, fmt
= tmpl
[:-1], '{0} = %({0})#j'
3122 return '\n'.join(map(fmt
.format
, [tmpl
] if mobj
.group('dict') else tmpl
.split(',')))
3124 for tmpl
in self
.params
['forceprint'].get(key
, []):
3125 self
.to_stdout(self
.evaluate_outtmpl(format_tmpl(tmpl
), info_copy
))
3127 for tmpl
, file_tmpl
in self
.params
['print_to_file'].get(key
, []):
3128 filename
= self
.prepare_filename(info_dict
, outtmpl
=file_tmpl
)
3129 tmpl
= format_tmpl(tmpl
)
3130 self
.to_screen(f
'[info] Writing {tmpl!r} to: {filename}')
3131 if self
._ensure
_dir
_exists
(filename
):
3132 with open(filename
, 'a', encoding
='utf-8', newline
='') as f
:
3133 f
.write(self
.evaluate_outtmpl(tmpl
, info_copy
) + os
.linesep
)
3137 def __forced_printings(self
, info_dict
, filename
=None, incomplete
=True):
3138 if (self
.params
.get('forcejson')
3139 or self
.params
['forceprint'].get('video')
3140 or self
.params
['print_to_file'].get('video')):
3141 self
.post_extract(info_dict
)
3143 info_dict
['filename'] = filename
3144 info_copy
= self
._forceprint
('video', info_dict
)
3146 def print_field(field
, actual_field
=None, optional
=False):
3147 if actual_field
is None:
3148 actual_field
= field
3149 if self
.params
.get(f
'force{field}') and (
3150 info_copy
.get(field
) is not None or (not optional
and not incomplete
)):
3151 self
.to_stdout(info_copy
[actual_field
])
3153 print_field('title')
3155 print_field('url', 'urls')
3156 print_field('thumbnail', optional
=True)
3157 print_field('description', optional
=True)
3158 print_field('filename')
3159 if self
.params
.get('forceduration') and info_copy
.get('duration') is not None:
3160 self
.to_stdout(formatSeconds(info_copy
['duration']))
3161 print_field('format')
3163 if self
.params
.get('forcejson'):
3164 self
.to_stdout(json
.dumps(self
.sanitize_info(info_dict
)))
3166 def dl(self
, name
, info
, subtitle
=False, test
=False):
3167 if not info
.get('url'):
3168 self
.raise_no_formats(info
, True)
3171 verbose
= self
.params
.get('verbose')
3174 'quiet': self
.params
.get('quiet') or not verbose
,
3176 'noprogress': not verbose
,
3178 'skip_unavailable_fragments': False,
3179 'keep_fragments': False,
3181 '_no_ytdl_file': True,
3184 params
= self
.params
3185 fd
= get_suitable_downloader(info
, params
, to_stdout
=(name
== '-'))(self
, params
)
3187 for ph
in self
._progress
_hooks
:
3188 fd
.add_progress_hook(ph
)
3190 (f
['url'].split(',')[0] + ',<data>' if f
['url'].startswith('data:') else f
['url'])
3191 for f
in info
.get('requested_formats', []) or [info
])
3192 self
.write_debug(f
'Invoking {fd.FD_NAME} downloader on "{urls}"')
3194 # Note: Ideally info should be a deep-copied so that hooks cannot modify it.
3195 # But it may contain objects that are not deep-copyable
3196 new_info
= self
._copy
_infodict
(info
)
3197 if new_info
.get('http_headers') is None:
3198 new_info
['http_headers'] = self
._calc
_headers
(new_info
)
3199 return fd
.download(name
, new_info
, subtitle
)
3201 def existing_file(self
, filepaths
, *, default_overwrite
=True):
3202 existing_files
= list(filter(os
.path
.exists
, orderedSet(filepaths
)))
3203 if existing_files
and not self
.params
.get('overwrites', default_overwrite
):
3204 return existing_files
[0]
3206 for file in existing_files
:
3207 self
.report_file_delete(file)
3211 @_catch_unsafe_extension_error
3212 def process_info(self
, info_dict
):
3213 """Process a single resolved IE result. (Modifies it in-place)"""
3215 assert info_dict
.get('_type', 'video') == 'video'
3216 original_infodict
= info_dict
3218 if 'format' not in info_dict
and 'ext' in info_dict
:
3219 info_dict
['format'] = info_dict
['ext']
3221 if self
._match
_entry
(info_dict
) is not None:
3222 info_dict
['__write_download_archive'] = 'ignore'
3225 # Does nothing under normal operation - for backward compatibility of process_info
3226 self
.post_extract(info_dict
)
3228 def replace_info_dict(new_info
):
3230 if new_info
== info_dict
:
3233 info_dict
.update(new_info
)
3235 new_info
, _
= self
.pre_process(info_dict
, 'video')
3236 replace_info_dict(new_info
)
3237 self
._num
_downloads
+= 1
3239 # info_dict['_filename'] needs to be set for backward compatibility
3240 info_dict
['_filename'] = full_filename
= self
.prepare_filename(info_dict
, warn
=True)
3241 temp_filename
= self
.prepare_filename(info_dict
, 'temp')
3245 self
.__forced
_printings
(info_dict
, full_filename
, incomplete
=('format' not in info_dict
))
3247 def check_max_downloads():
3248 if self
._num
_downloads
>= float(self
.params
.get('max_downloads') or 'inf'):
3249 raise MaxDownloadsReached
3251 if self
.params
.get('simulate'):
3252 info_dict
['__write_download_archive'] = self
.params
.get('force_write_download_archive')
3253 check_max_downloads()
3256 if full_filename
is None:
3258 if not self
._ensure
_dir
_exists
(encodeFilename(full_filename
)):
3260 if not self
._ensure
_dir
_exists
(encodeFilename(temp_filename
)):
3263 if self
._write
_description
('video', info_dict
,
3264 self
.prepare_filename(info_dict
, 'description')) is None:
3267 sub_files
= self
._write
_subtitles
(info_dict
, temp_filename
)
3268 if sub_files
is None:
3270 files_to_move
.update(dict(sub_files
))
3272 thumb_files
= self
._write
_thumbnails
(
3273 'video', info_dict
, temp_filename
, self
.prepare_filename(info_dict
, 'thumbnail'))
3274 if thumb_files
is None:
3276 files_to_move
.update(dict(thumb_files
))
3278 infofn
= self
.prepare_filename(info_dict
, 'infojson')
3279 _infojson_written
= self
._write
_info
_json
('video', info_dict
, infofn
)
3280 if _infojson_written
:
3281 info_dict
['infojson_filename'] = infofn
3282 # For backward compatibility, even though it was a private field
3283 info_dict
['__infojson_filename'] = infofn
3284 elif _infojson_written
is None:
3287 # Note: Annotations are deprecated
3289 if self
.params
.get('writeannotations', False):
3290 annofn
= self
.prepare_filename(info_dict
, 'annotation')
3292 if not self
._ensure
_dir
_exists
(encodeFilename(annofn
)):
3294 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(annofn
)):
3295 self
.to_screen('[info] Video annotations are already present')
3296 elif not info_dict
.get('annotations'):
3297 self
.report_warning('There are no annotations to write.')
3300 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
3301 with open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
3302 annofile
.write(info_dict
['annotations'])
3303 except (KeyError, TypeError):
3304 self
.report_warning('There are no annotations to write.')
3306 self
.report_error('Cannot write annotations file: ' + annofn
)
3309 # Write internet shortcut files
3310 def _write_link_file(link_type
):
3311 url
= try_get(info_dict
['webpage_url'], iri_to_uri
)
3313 self
.report_warning(
3314 f
'Cannot write internet shortcut file because the actual URL of "{info_dict["webpage_url"]}" is unknown')
3316 linkfn
= replace_extension(self
.prepare_filename(info_dict
, 'link'), link_type
, info_dict
.get('ext'))
3317 if not self
._ensure
_dir
_exists
(encodeFilename(linkfn
)):
3319 if self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(linkfn
)):
3320 self
.to_screen(f
'[info] Internet shortcut (.{link_type}) is already present')
3323 self
.to_screen(f
'[info] Writing internet shortcut (.{link_type}) to: {linkfn}')
3324 with open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8',
3325 newline
='\r\n' if link_type
== 'url' else '\n') as linkfile
:
3326 template_vars
= {'url': url}
3327 if link_type
== 'desktop':
3328 template_vars
['filename'] = linkfn
[:-(len(link_type
) + 1)]
3329 linkfile
.write(LINK_TEMPLATES
[link_type
] % template_vars
)
3331 self
.report_error(f
'Cannot write internet shortcut {linkfn}')
3336 'url': self
.params
.get('writeurllink'),
3337 'webloc': self
.params
.get('writewebloclink'),
3338 'desktop': self
.params
.get('writedesktoplink'),
3340 if self
.params
.get('writelink'):
3341 link_type
= ('webloc' if sys
.platform
== 'darwin'
3342 else 'desktop' if sys
.platform
.startswith('linux')
3344 write_links
[link_type
] = True
3346 if any(should_write
and not _write_link_file(link_type
)
3347 for link_type
, should_write
in write_links
.items()):
3350 new_info
, files_to_move
= self
.pre_process(info_dict
, 'before_dl', files_to_move
)
3351 replace_info_dict(new_info
)
3353 if self
.params
.get('skip_download'):
3354 info_dict
['filepath'] = temp_filename
3355 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
3356 info_dict
['__files_to_move'] = files_to_move
3357 replace_info_dict(self
.run_pp(MoveFilesAfterDownloadPP(self
, False), info_dict
))
3358 info_dict
['__write_download_archive'] = self
.params
.get('force_write_download_archive')
3361 info_dict
.setdefault('__postprocessors', [])
3364 def existing_video_file(*filepaths
):
3365 ext
= info_dict
.get('ext')
3366 converted
= lambda file: replace_extension(file, self
.params
.get('final_ext') or ext
, ext
)
3367 file = self
.existing_file(itertools
.chain(*zip(map(converted
, filepaths
), filepaths
)),
3368 default_overwrite
=False)
3370 info_dict
['ext'] = os
.path
.splitext(file)[1][1:]
3373 fd
, success
= None, True
3374 if info_dict
.get('protocol') or info_dict
.get('url'):
3375 fd
= get_suitable_downloader(info_dict
, self
.params
, to_stdout
=temp_filename
== '-')
3376 if fd
!= FFmpegFD
and 'no-direct-merge' not in self
.params
['compat_opts'] and (
3377 info_dict
.get('section_start') or info_dict
.get('section_end')):
3378 msg
= ('This format cannot be partially downloaded' if FFmpegFD
.available()
3379 else 'You have requested downloading the video partially, but ffmpeg is not installed')
3380 self
.report_error(f
'{msg}. Aborting')
3383 if info_dict
.get('requested_formats') is not None:
3384 old_ext
= info_dict
['ext']
3385 if self
.params
.get('merge_output_format') is None:
3386 if (info_dict
['ext'] == 'webm'
3387 and info_dict
.get('thumbnails')
3388 # check with type instead of pp_key, __name__, or isinstance
3389 # since we dont want any custom PPs to trigger this
3390 and any(type(pp
) == EmbedThumbnailPP
for pp
in self
._pps
['post_process'])): # noqa: E721
3391 info_dict
['ext'] = 'mkv'
3392 self
.report_warning(
3393 'webm doesn\'t support embedding a thumbnail, mkv will be used')
3394 new_ext
= info_dict
['ext']
3396 def correct_ext(filename
, ext
=new_ext
):
3399 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
3401 os
.path
.splitext(filename
)[0]
3402 if filename_real_ext
in (old_ext
, new_ext
)
3404 return f
'{filename_wo_ext}.{ext}'
3406 # Ensure filename always has a correct extension for successful merge
3407 full_filename
= correct_ext(full_filename
)
3408 temp_filename
= correct_ext(temp_filename
)
3409 dl_filename
= existing_video_file(full_filename
, temp_filename
)
3411 info_dict
['__real_download'] = False
3412 # NOTE: Copy so that original format dicts are not modified
3413 info_dict
['requested_formats'] = list(map(dict, info_dict
['requested_formats']))
3415 merger
= FFmpegMergerPP(self
)
3417 if dl_filename
is not None:
3418 self
.report_file_already_downloaded(dl_filename
)
3420 for f
in info_dict
['requested_formats'] if fd
!= FFmpegFD
else []:
3421 f
['filepath'] = fname
= prepend_extension(
3422 correct_ext(temp_filename
, info_dict
['ext']),
3423 'f{}'.format(f
['format_id']), info_dict
['ext'])
3424 downloaded
.append(fname
)
3425 info_dict
['url'] = '\n'.join(f
['url'] for f
in info_dict
['requested_formats'])
3426 success
, real_download
= self
.dl(temp_filename
, info_dict
)
3427 info_dict
['__real_download'] = real_download
3429 if self
.params
.get('allow_unplayable_formats'):
3430 self
.report_warning(
3431 'You have requested merging of multiple formats '
3432 'while also allowing unplayable formats to be downloaded. '
3433 'The formats won\'t be merged to prevent data corruption.')
3434 elif not merger
.available
:
3435 msg
= 'You have requested merging of multiple formats but ffmpeg is not installed'
3436 if not self
.params
.get('ignoreerrors'):
3437 self
.report_error(f
'{msg}. Aborting due to --abort-on-error')
3439 self
.report_warning(f
'{msg}. The formats won\'t be merged')
3441 if temp_filename
== '-':
3442 reason
= ('using a downloader other than ffmpeg' if FFmpegFD
.can_merge_formats(info_dict
, self
.params
)
3443 else 'but the formats are incompatible for simultaneous download' if merger
.available
3444 else 'but ffmpeg is not installed')
3445 self
.report_warning(
3446 f
'You have requested downloading multiple formats to stdout {reason}. '
3447 'The formats will be streamed one after the other')
3448 fname
= temp_filename
3449 for f
in info_dict
['requested_formats']:
3450 new_info
= dict(info_dict
)
3451 del new_info
['requested_formats']
3453 if temp_filename
!= '-':
3454 fname
= prepend_extension(
3455 correct_ext(temp_filename
, new_info
['ext']),
3456 'f{}'.format(f
['format_id']), new_info
['ext'])
3457 if not self
._ensure
_dir
_exists
(fname
):
3459 f
['filepath'] = fname
3460 downloaded
.append(fname
)
3461 partial_success
, real_download
= self
.dl(fname
, new_info
)
3462 info_dict
['__real_download'] = info_dict
['__real_download'] or real_download
3463 success
= success
and partial_success
3465 if downloaded
and merger
.available
and not self
.params
.get('allow_unplayable_formats'):
3466 info_dict
['__postprocessors'].append(merger
)
3467 info_dict
['__files_to_merge'] = downloaded
3468 # Even if there were no downloads, it is being merged only now
3469 info_dict
['__real_download'] = True
3471 for file in downloaded
:
3472 files_to_move
[file] = None
3474 # Just a single file
3475 dl_filename
= existing_video_file(full_filename
, temp_filename
)
3476 if dl_filename
is None or dl_filename
== temp_filename
:
3477 # dl_filename == temp_filename could mean that the file was partially downloaded with --no-part.
3478 # So we should try to resume the download
3479 success
, real_download
= self
.dl(temp_filename
, info_dict
)
3480 info_dict
['__real_download'] = real_download
3482 self
.report_file_already_downloaded(dl_filename
)
3484 dl_filename
= dl_filename
or temp_filename
3485 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
3487 except network_exceptions
as err
:
3488 self
.report_error(f
'unable to download video data: {err}')
3490 except OSError as err
:
3491 raise UnavailableVideoError(err
)
3492 except ContentTooShortError
as err
:
3493 self
.report_error(f
'content too short (expected {err.expected} bytes and served {err.downloaded})')
3496 self
._raise
_pending
_errors
(info_dict
)
3497 if success
and full_filename
!= '-':
3501 fixup_policy
= self
.params
.get('fixup')
3502 vid
= info_dict
['id']
3504 if fixup_policy
in ('ignore', 'never'):
3506 elif fixup_policy
== 'warn':
3508 elif fixup_policy
!= 'force':
3509 assert fixup_policy
in ('detect_or_warn', None)
3510 if not info_dict
.get('__real_download'):
3513 def ffmpeg_fixup(cndn
, msg
, cls
):
3514 if not (do_fixup
and cndn
):
3516 elif do_fixup
== 'warn':
3517 self
.report_warning(f
'{vid}: {msg}')
3521 info_dict
['__postprocessors'].append(pp
)
3523 self
.report_warning(f
'{vid}: {msg}. Install ffmpeg to fix this automatically')
3525 stretched_ratio
= info_dict
.get('stretched_ratio')
3526 ffmpeg_fixup(stretched_ratio
not in (1, None),
3527 f
'Non-uniform pixel ratio {stretched_ratio}',
3528 FFmpegFixupStretchedPP
)
3530 downloader
= get_suitable_downloader(info_dict
, self
.params
) if 'protocol' in info_dict
else None
3531 downloader
= downloader
.FD_NAME
if downloader
else None
3533 ext
= info_dict
.get('ext')
3534 postprocessed_by_ffmpeg
= info_dict
.get('requested_formats') or any((
3535 isinstance(pp
, FFmpegVideoConvertorPP
)
3536 and resolve_recode_mapping(ext
, pp
.mapping
)[0] not in (ext
, None)
3537 ) for pp
in self
._pps
['post_process'])
3539 if not postprocessed_by_ffmpeg
:
3540 ffmpeg_fixup(fd
!= FFmpegFD
and ext
== 'm4a'
3541 and info_dict
.get('container') == 'm4a_dash',
3542 'writing DASH m4a. Only some players support this container',
3544 ffmpeg_fixup(downloader
== 'hlsnative' and not self
.params
.get('hls_use_mpegts')
3545 or info_dict
.get('is_live') and self
.params
.get('hls_use_mpegts') is None,
3546 'Possible MPEG-TS in MP4 container or malformed AAC timestamps',
3548 ffmpeg_fixup(downloader
== 'dashsegments'
3549 and (info_dict
.get('is_live') or info_dict
.get('is_dash_periods')),
3550 'Possible duplicate MOOV atoms', FFmpegFixupDuplicateMoovPP
)
3552 ffmpeg_fixup(downloader
== 'web_socket_fragment', 'Malformed timestamps detected', FFmpegFixupTimestampPP
)
3553 ffmpeg_fixup(downloader
== 'web_socket_fragment', 'Malformed duration detected', FFmpegFixupDurationPP
)
3557 replace_info_dict(self
.post_process(dl_filename
, info_dict
, files_to_move
))
3558 except PostProcessingError
as err
:
3559 self
.report_error(f
'Postprocessing: {err}')
3562 for ph
in self
._post
_hooks
:
3563 ph(info_dict
['filepath'])
3564 except Exception as err
:
3565 self
.report_error(f
'post hooks: {err}')
3567 info_dict
['__write_download_archive'] = True
3569 assert info_dict
is original_infodict
# Make sure the info_dict was modified in-place
3570 if self
.params
.get('force_write_download_archive'):
3571 info_dict
['__write_download_archive'] = True
3572 check_max_downloads()
3574 def __download_wrapper(self
, func
):
3575 @functools.wraps(func
)
3576 def wrapper(*args
, **kwargs
):
3578 res
= func(*args
, **kwargs
)
3579 except UnavailableVideoError
as e
:
3580 self
.report_error(e
)
3581 except DownloadCancelled
as e
:
3582 self
.to_screen(f
'[info] {e}')
3583 if not self
.params
.get('break_per_url'):
3585 self
._num
_downloads
= 0
3587 if self
.params
.get('dump_single_json', False):
3588 self
.post_extract(res
)
3589 self
.to_stdout(json
.dumps(self
.sanitize_info(res
)))
3592 def download(self
, url_list
):
3593 """Download a given list of URLs."""
3594 url_list
= variadic(url_list
) # Passing a single URL is a common mistake
3595 outtmpl
= self
.params
['outtmpl']['default']
3596 if (len(url_list
) > 1
3598 and '%' not in outtmpl
3599 and self
.params
.get('max_downloads') != 1):
3600 raise SameFileError(outtmpl
)
3602 for url
in url_list
:
3603 self
.__download
_wrapper
(self
.extract_info
)(
3604 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
3606 return self
._download
_retcode
3608 def download_with_info_file(self
, info_filename
):
3609 with contextlib
.closing(fileinput
.FileInput(
3610 [info_filename
], mode
='r',
3611 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
3612 # FileInput doesn't have a read method, we can't call json.load
3613 infos
= [self
.sanitize_info(info
, self
.params
.get('clean_infojson', True))
3614 for info
in variadic(json
.loads('\n'.join(f
)))]
3617 self
.__download
_wrapper
(self
.process_ie_result
)(info
, download
=True)
3618 except (DownloadError
, EntryNotInPlaylist
, ReExtractInfo
) as e
:
3619 if not isinstance(e
, EntryNotInPlaylist
):
3620 self
.to_stderr('\r')
3621 webpage_url
= info
.get('webpage_url')
3622 if webpage_url
is None:
3624 self
.report_warning(f
'The info failed to download: {e}; trying with URL {webpage_url}')
3625 self
.download([webpage_url
])
3626 except ExtractorError
as e
:
3627 self
.report_error(e
)
3628 return self
._download
_retcode
3631 def sanitize_info(info_dict
, remove_private_keys
=False):
3632 """ Sanitize the infodict for converting to json """
3633 if info_dict
is None:
3635 info_dict
.setdefault('epoch', int(time
.time()))
3636 info_dict
.setdefault('_type', 'video')
3637 info_dict
.setdefault('_version', {
3638 'version': __version__
,
3639 'current_git_head': current_git_head(),
3640 'release_git_head': RELEASE_GIT_HEAD
,
3641 'repository': ORIGIN
,
3644 if remove_private_keys
:
3645 reject
= lambda k
, v
: v
is None or k
.startswith('__') or k
in {
3646 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
3647 'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url',
3648 'playlist_autonumber',
3651 reject
= lambda k
, v
: False
3654 if isinstance(obj
, dict):
3655 return {k: filter_fn(v) for k, v in obj.items() if not reject(k, v)}
3656 elif isinstance(obj
, (list, tuple, set, LazyList
)):
3657 return list(map(filter_fn
, obj
))
3658 elif obj
is None or isinstance(obj
, (str, int, float, bool)):
3663 return filter_fn(info_dict
)
3666 def filter_requested_info(info_dict
, actually_filter
=True):
3667 """ Alias of sanitize_info for backward compatibility """
3668 return YoutubeDL
.sanitize_info(info_dict
, actually_filter
)
3670 def _delete_downloaded_files(self
, *files_to_delete
, info
={}, msg
=None):
3671 for filename
in set(filter(None, files_to_delete
)):
3673 self
.to_screen(msg
% filename
)
3677 self
.report_warning(f
'Unable to delete file {filename}')
3678 if filename
in info
.get('__files_to_move', []): # NB: Delete even if None
3679 del info
['__files_to_move'][filename
]
3682 def post_extract(info_dict
):
3683 def actual_post_extract(info_dict
):
3684 if info_dict
.get('_type') in ('playlist', 'multi_video'):
3685 for video_dict
in info_dict
.get('entries', {}):
3686 actual_post_extract(video_dict
or {})
3689 post_extractor
= info_dict
.pop('__post_extractor', None) or dict
3690 info_dict
.update(post_extractor())
3692 actual_post_extract(info_dict
or {})
3694 def run_pp(self
, pp
, infodict
):
3695 files_to_delete
= []
3696 if '__files_to_move' not in infodict
:
3697 infodict
['__files_to_move'] = {}
3699 files_to_delete
, infodict
= pp
.run(infodict
)
3700 except PostProcessingError
as e
:
3701 # Must be True and not 'only_download'
3702 if self
.params
.get('ignoreerrors') is True:
3703 self
.report_error(e
)
3707 if not files_to_delete
:
3709 if self
.params
.get('keepvideo', False):
3710 for f
in files_to_delete
:
3711 infodict
['__files_to_move'].setdefault(f
, '')
3713 self
._delete
_downloaded
_files
(
3714 *files_to_delete
, info
=infodict
, msg
='Deleting original file %s (pass -k to keep)')
3717 def run_all_pps(self
, key
, info
, *, additional_pps
=None):
3719 self
._forceprint
(key
, info
)
3720 for pp
in (additional_pps
or []) + self
._pps
[key
]:
3721 info
= self
.run_pp(pp
, info
)
3724 def pre_process(self
, ie_info
, key
='pre_process', files_to_move
=None):
3725 info
= dict(ie_info
)
3726 info
['__files_to_move'] = files_to_move
or {}
3728 info
= self
.run_all_pps(key
, info
)
3729 except PostProcessingError
as err
:
3730 msg
= f
'Preprocessing: {err}'
3731 info
.setdefault('__pending_error', msg
)
3732 self
.report_error(msg
, is_error
=False)
3733 return info
, info
.pop('__files_to_move', None)
3735 def post_process(self
, filename
, info
, files_to_move
=None):
3736 """Run all the postprocessors on the given file."""
3737 info
['filepath'] = filename
3738 info
['__files_to_move'] = files_to_move
or {}
3739 info
= self
.run_all_pps('post_process', info
, additional_pps
=info
.get('__postprocessors'))
3740 info
= self
.run_pp(MoveFilesAfterDownloadPP(self
), info
)
3741 del info
['__files_to_move']
3742 return self
.run_all_pps('after_move', info
)
3744 def _make_archive_id(self
, info_dict
):
3745 video_id
= info_dict
.get('id')
3748 # Future-proof against any change in case
3749 # and backwards compatibility with prior versions
3750 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
3751 if extractor
is None:
3752 url
= str_or_none(info_dict
.get('url'))
3755 # Try to find matching extractor for the URL and take its ie_key
3756 for ie_key
, ie
in self
._ies
.items():
3757 if ie
.suitable(url
):
3762 return make_archive_id(extractor
, video_id
)
3764 def in_download_archive(self
, info_dict
):
3765 if not self
.archive
:
3768 vid_ids
= [self
._make
_archive
_id
(info_dict
)]
3769 vid_ids
.extend(info_dict
.get('_old_archive_ids') or [])
3770 return any(id_
in self
.archive
for id_
in vid_ids
)
3772 def record_download_archive(self
, info_dict
):
3773 fn
= self
.params
.get('download_archive')
3776 vid_id
= self
._make
_archive
_id
(info_dict
)
3779 self
.write_debug(f
'Adding to archive: {vid_id}')
3780 if is_path_like(fn
):
3781 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
3782 archive_file
.write(vid_id
+ '\n')
3783 self
.archive
.add(vid_id
)
3786 def format_resolution(format
, default
='unknown'):
3787 if format
.get('vcodec') == 'none' and format
.get('acodec') != 'none':
3789 if format
.get('resolution') is not None:
3790 return format
['resolution']
3791 if format
.get('width') and format
.get('height'):
3792 return '%dx%d' % (format
['width'], format
['height'])
3793 elif format
.get('height'):
3794 return '{}p'.format(format
['height'])
3795 elif format
.get('width'):
3796 return '%dx?' % format
['width']
3799 def _list_format_headers(self
, *headers
):
3800 if self
.params
.get('listformats_table', True) is not False:
3801 return [self
._format
_out
(header
, self
.Styles
.HEADERS
) for header
in headers
]
3804 def _format_note(self
, fdict
):
3806 if fdict
.get('ext') in ['f4f', 'f4m']:
3807 res
+= '(unsupported)'
3808 if fdict
.get('language'):
3811 res
+= '[{}]'.format(fdict
['language'])
3812 if fdict
.get('format_note') is not None:
3815 res
+= fdict
['format_note']
3816 if fdict
.get('tbr') is not None:
3819 res
+= '%4dk' % fdict
['tbr']
3820 if fdict
.get('container') is not None:
3823 res
+= '{} container'.format(fdict
['container'])
3824 if (fdict
.get('vcodec') is not None
3825 and fdict
.get('vcodec') != 'none'):
3828 res
+= fdict
['vcodec']
3829 if fdict
.get('vbr') is not None:
3831 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
3833 if fdict
.get('vbr') is not None:
3834 res
+= '%4dk' % fdict
['vbr']
3835 if fdict
.get('fps') is not None:
3838 res
+= '{}fps'.format(fdict
['fps'])
3839 if fdict
.get('acodec') is not None:
3842 if fdict
['acodec'] == 'none':
3845 res
+= '%-5s' % fdict
['acodec']
3846 elif fdict
.get('abr') is not None:
3850 if fdict
.get('abr') is not None:
3851 res
+= '@%3dk' % fdict
['abr']
3852 if fdict
.get('asr') is not None:
3853 res
+= ' (%5dHz)' % fdict
['asr']
3854 if fdict
.get('filesize') is not None:
3857 res
+= format_bytes(fdict
['filesize'])
3858 elif fdict
.get('filesize_approx') is not None:
3861 res
+= '~' + format_bytes(fdict
['filesize_approx'])
3864 def _get_formats(self
, info_dict
):
3865 if info_dict
.get('formats') is None:
3866 if info_dict
.get('url') and info_dict
.get('_type', 'video') == 'video':
3869 return info_dict
['formats']
3871 def render_formats_table(self
, info_dict
):
3872 formats
= self
._get
_formats
(info_dict
)
3875 if not self
.params
.get('listformats_table', True) is not False:
3878 format_field(f
, 'format_id'),
3879 format_field(f
, 'ext'),
3880 self
.format_resolution(f
),
3881 self
._format
_note
(f
),
3882 ] for f
in formats
if (f
.get('preference') or 0) >= -1000]
3883 return render_table(['format code', 'extension', 'resolution', 'note'], table
, extra_gap
=1)
3885 def simplified_codec(f
, field
):
3886 assert field
in ('acodec', 'vcodec')
3887 codec
= f
.get(field
)
3890 elif codec
!= 'none':
3891 return '.'.join(codec
.split('.')[:4])
3893 if field
== 'vcodec' and f
.get('acodec') == 'none':
3895 elif field
== 'acodec' and f
.get('vcodec') == 'none':
3897 return self
._format
_out
('audio only' if field
== 'vcodec' else 'video only',
3898 self
.Styles
.SUPPRESS
)
3900 delim
= self
._format
_out
('\u2502', self
.Styles
.DELIM
, '|', test_encoding
=True)
3903 self
._format
_out
(format_field(f
, 'format_id'), self
.Styles
.ID
),
3904 format_field(f
, 'ext'),
3905 format_field(f
, func
=self
.format_resolution
, ignore
=('audio only', 'images')),
3906 format_field(f
, 'fps', '\t%d', func
=round),
3907 format_field(f
, 'dynamic_range', '%s', ignore
=(None, 'SDR')).replace('HDR', ''),
3908 format_field(f
, 'audio_channels', '\t%s'),
3910 format_field(f
, 'filesize', ' \t%s', func
=format_bytes
)
3911 or format_field(f
, 'filesize_approx', '≈\t%s', func
=format_bytes
)
3912 or format_field(filesize_from_tbr(f
.get('tbr'), info_dict
.get('duration')), None,
3913 self
._format
_out
('~\t%s', self
.Styles
.SUPPRESS
), func
=format_bytes
)),
3914 format_field(f
, 'tbr', '\t%dk', func
=round),
3915 shorten_protocol_name(f
.get('protocol', '')),
3917 simplified_codec(f
, 'vcodec'),
3918 format_field(f
, 'vbr', '\t%dk', func
=round),
3919 simplified_codec(f
, 'acodec'),
3920 format_field(f
, 'abr', '\t%dk', func
=round),
3921 format_field(f
, 'asr', '\t%s', func
=format_decimal_suffix
),
3922 join_nonempty(format_field(f
, 'language', '[%s]'), join_nonempty(
3923 self
._format
_out
('UNSUPPORTED', self
.Styles
.BAD_FORMAT
) if f
.get('ext') in ('f4f', 'f4m') else None,
3924 (self
._format
_out
('Maybe DRM', self
.Styles
.WARNING
) if f
.get('has_drm') == 'maybe'
3925 else self
._format
_out
('DRM', self
.Styles
.BAD_FORMAT
) if f
.get('has_drm') else None),
3926 format_field(f
, 'format_note'),
3927 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
3928 delim
=', '), delim
=' '),
3929 ] for f
in formats
if f
.get('preference') is None or f
['preference'] >= -1000]
3930 header_line
= self
._list
_format
_headers
(
3931 'ID', 'EXT', 'RESOLUTION', '\tFPS', 'HDR', 'CH', delim
, '\tFILESIZE', '\tTBR', 'PROTO',
3932 delim
, 'VCODEC', '\tVBR', 'ACODEC', '\tABR', '\tASR', 'MORE INFO')
3934 return render_table(
3935 header_line
, table
, hide_empty
=True,
3936 delim
=self
._format
_out
('\u2500', self
.Styles
.DELIM
, '-', test_encoding
=True))
3938 def render_thumbnails_table(self
, info_dict
):
3939 thumbnails
= list(info_dict
.get('thumbnails') or [])
3942 return render_table(
3943 self
._list
_format
_headers
('ID', 'Width', 'Height', 'URL'),
3944 [[t
.get('id'), t
.get('width') or 'unknown', t
.get('height') or 'unknown', t
['url']] for t
in thumbnails
])
3946 def render_subtitles_table(self
, video_id
, subtitles
):
3947 def _row(lang
, formats
):
3948 exts
, names
= zip(*((f
['ext'], f
.get('name') or 'unknown') for f
in reversed(formats
)))
3949 if len(set(names
)) == 1:
3950 names
= [] if names
[0] == 'unknown' else names
[:1]
3951 return [lang
, ', '.join(names
), ', '.join(exts
)]
3955 return render_table(
3956 self
._list
_format
_headers
('Language', 'Name', 'Formats'),
3957 [_row(lang
, formats
) for lang
, formats
in subtitles
.items()],
3960 def __list_table(self
, video_id
, name
, func
, *args
):
3963 self
.to_screen(f
'{video_id} has no {name}')
3965 self
.to_screen(f
'[info] Available {name} for {video_id}:')
3966 self
.to_stdout(table
)
3968 def list_formats(self
, info_dict
):
3969 self
.__list
_table
(info_dict
['id'], 'formats', self
.render_formats_table
, info_dict
)
3971 def list_thumbnails(self
, info_dict
):
3972 self
.__list
_table
(info_dict
['id'], 'thumbnails', self
.render_thumbnails_table
, info_dict
)
3974 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
3975 self
.__list
_table
(video_id
, name
, self
.render_subtitles_table
, video_id
, subtitles
)
3977 def print_debug_header(self
):
3978 if not self
.params
.get('verbose'):
3981 from . import _IN_CLI
# Must be delayed import
3983 # These imports can be slow. So import them only as needed
3984 from .extractor
.extractors
import _LAZY_LOADER
3985 from .extractor
.extractors
import (
3986 _PLUGIN_CLASSES
as plugin_ies
,
3987 _PLUGIN_OVERRIDES
as plugin_ie_overrides
,
3990 def get_encoding(stream
):
3991 ret
= str(getattr(stream
, 'encoding', f
'missing ({type(stream).__name__})'))
3992 additional_info
= []
3993 if os
.environ
.get('TERM', '').lower() == 'dumb':
3994 additional_info
.append('dumb')
3995 if not supports_terminal_sequences(stream
):
3996 from .utils
import WINDOWS_VT_MODE
# Must be imported locally
3997 additional_info
.append('No VT' if WINDOWS_VT_MODE
is False else 'No ANSI')
3999 ret
= f
'{ret} ({",".join(additional_info)})'
4002 encoding_str
= 'Encodings: locale {}, fs {}, pref {}, {}'.format(
4003 locale
.getpreferredencoding(),
4004 sys
.getfilesystemencoding(),
4005 self
.get_encoding(),
4007 f
'{key} {get_encoding(stream)}' for key
, stream
in self
._out
_files
.items_
4008 if stream
is not None and key
!= 'console'),
4011 logger
= self
.params
.get('logger')
4013 write_debug
= lambda msg
: logger
.debug(f
'[debug] {msg}')
4014 write_debug(encoding_str
)
4016 write_string(f
'[debug] {encoding_str}\n', encoding
=None)
4017 write_debug
= lambda msg
: self
._write
_string
(f
'[debug] {msg}\n')
4019 source
= detect_variant()
4020 if VARIANT
not in (None, 'pip'):
4023 write_debug(join_nonempty(
4024 f
'{REPOSITORY.rpartition("/")[2]} version',
4025 _make_label(ORIGIN
, CHANNEL
.partition('@')[2] or __version__
, __version__
),
4026 f
'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD
else '',
4027 '' if source
== 'unknown' else f
'({source})',
4028 '' if _IN_CLI
else 'API' if klass
== YoutubeDL
else f
'API:{self.__module__}.{klass.__qualname__}',
4032 write_debug(f
'params: {self.params}')
4034 if not _LAZY_LOADER
:
4035 if os
.environ
.get('YTDLP_NO_LAZY_EXTRACTORS'):
4036 write_debug('Lazy loading extractors is forcibly disabled')
4038 write_debug('Lazy loading extractors is disabled')
4039 if self
.params
['compat_opts']:
4040 write_debug('Compatibility options: {}'.format(', '.join(self
.params
['compat_opts'])))
4042 if current_git_head():
4043 write_debug(f
'Git HEAD: {current_git_head()}')
4044 write_debug(system_identifier())
4046 exe_versions
, ffmpeg_features
= FFmpegPostProcessor
.get_versions_and_features(self
)
4047 ffmpeg_features
= {key for key, val in ffmpeg_features.items() if val}
4049 exe_versions
['ffmpeg'] += ' ({})'.format(','.join(sorted(ffmpeg_features
)))
4051 exe_versions
['rtmpdump'] = rtmpdump_version()
4052 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
4053 exe_str
= ', '.join(
4054 f
'{exe} {v}' for exe
, v
in sorted(exe_versions
.items()) if v
4056 write_debug(f
'exe versions: {exe_str}')
4058 from .compat
.compat_utils
import get_package_info
4059 from .dependencies
import available_dependencies
4061 write_debug('Optional libraries: %s' % (', '.join(sorted({
4062 join_nonempty(*get_package_info(m
)) for m
in available_dependencies
.values()
4065 write_debug(f
'Proxy map: {self.proxies}')
4066 write_debug(f
'Request Handlers: {", ".join(rh.RH_NAME for rh in self._request_director.handlers.values())}')
4067 for plugin_type
, plugins
in {'Extractor': plugin_ies, 'Post-Processor': plugin_pps}
.items():
4068 display_list
= ['{}{}'.format(
4069 klass
.__name
__, '' if klass
.__name
__ == name
else f
' as {name}')
4070 for name
, klass
in plugins
.items()]
4071 if plugin_type
== 'Extractor':
4072 display_list
.extend(f
'{plugins[-1].IE_NAME.partition("+")[2]} ({parent.__name__})'
4073 for parent
, plugins
in plugin_ie_overrides
.items())
4074 if not display_list
:
4076 write_debug(f
'{plugin_type} Plugins: {", ".join(sorted(display_list))}')
4078 plugin_dirs
= plugin_directories()
4080 write_debug(f
'Plugin directories: {plugin_dirs}')
4083 if False and self
.params
.get('call_home'):
4084 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode()
4085 write_debug(f
'Public IP address: {ipaddr}')
4086 latest_version
= self
.urlopen(
4087 'https://yt-dl.org/latest/version').read().decode()
4088 if version_tuple(latest_version
) > version_tuple(__version__
):
4089 self
.report_warning(
4090 f
'You are using an outdated version (newest version: {latest_version})! '
4091 'See https://yt-dl.org/update if you need help updating.')
4093 @functools.cached_property
4095 """Global proxy configuration"""
4096 opts_proxy
= self
.params
.get('proxy')
4097 if opts_proxy
is not None:
4098 if opts_proxy
== '':
4099 opts_proxy
= '__noproxy__'
4100 proxies
= {'all': opts_proxy}
4102 proxies
= urllib
.request
.getproxies()
4103 # compat. Set HTTPS_PROXY to __noproxy__ to revert
4104 if 'http' in proxies
and 'https' not in proxies
:
4105 proxies
['https'] = proxies
['http']
4109 @functools.cached_property
4110 def cookiejar(self
):
4111 """Global cookiejar instance"""
4112 return load_cookies(
4113 self
.params
.get('cookiefile'), self
.params
.get('cookiesfrombrowser'), self
)
4118 Get a urllib OpenerDirector from the Urllib handler (deprecated).
4120 self
.deprecation_warning('YoutubeDL._opener is deprecated, use YoutubeDL.urlopen()')
4121 handler
= self
._request
_director
.handlers
['Urllib']
4122 return handler
._get
_instance
(cookiejar
=self
.cookiejar
, proxies
=self
.proxies
)
4124 def _get_available_impersonate_targets(self
):
4125 # TODO(future): make available as public API
4127 (target
, rh
.RH_NAME
)
4128 for rh
in self
._request
_director
.handlers
.values()
4129 if isinstance(rh
, ImpersonateRequestHandler
)
4130 for target
in rh
.supported_targets
4133 def _impersonate_target_available(self
, target
):
4134 # TODO(future): make available as public API
4136 rh
.is_supported_target(target
)
4137 for rh
in self
._request
_director
.handlers
.values()
4138 if isinstance(rh
, ImpersonateRequestHandler
))
4140 def urlopen(self
, req
):
4141 """ Start an HTTP download """
4142 if isinstance(req
, str):
4144 elif isinstance(req
, urllib
.request
.Request
):
4145 self
.deprecation_warning(
4146 'Passing a urllib.request.Request object to YoutubeDL.urlopen() is deprecated. '
4147 'Use yt_dlp.networking.common.Request instead.')
4148 req
= urllib_req_to_req(req
)
4149 assert isinstance(req
, Request
)
4151 # compat: Assume user:pass url params are basic auth
4152 url
, basic_auth_header
= extract_basic_auth(req
.url
)
4153 if basic_auth_header
:
4154 req
.headers
['Authorization'] = basic_auth_header
4155 req
.url
= sanitize_url(url
)
4157 clean_proxies(proxies
=req
.proxies
, headers
=req
.headers
)
4158 clean_headers(req
.headers
)
4161 return self
._request
_director
.send(req
)
4162 except NoSupportingHandlers
as e
:
4163 for ue
in e
.unsupported_errors
:
4164 # FIXME: This depends on the order of errors.
4165 if not (ue
.handler
and ue
.msg
):
4167 if ue
.handler
.RH_KEY
== 'Urllib' and 'unsupported url scheme: "file"' in ue
.msg
.lower():
4169 'file:// URLs are disabled by default in yt-dlp for security reasons. '
4170 'Use --enable-file-urls to enable at your own risk.', cause
=ue
) from ue
4172 'unsupported proxy type: "https"' in ue
.msg
.lower()
4173 and 'requests' not in self
._request
_director
.handlers
4174 and 'curl_cffi' not in self
._request
_director
.handlers
4177 'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests, curl_cffi')
4180 re
.match(r
'unsupported url scheme: "wss?"', ue
.msg
.lower())
4181 and 'websockets' not in self
._request
_director
.handlers
4184 'This request requires WebSocket support. '
4185 'Ensure one of the following dependencies are installed: websockets',
4188 elif re
.match(r
'unsupported (?:extensions: impersonate|impersonate target)', ue
.msg
.lower()):
4190 f
'Impersonate target "{req.extensions["impersonate"]}" is not available.'
4191 f
' See --list-impersonate-targets for available targets.'
4192 f
' This request requires browser impersonation, however you may be missing dependencies'
4193 f
' required to support this target.')
4195 except SSLError
as e
:
4196 if 'UNSAFE_LEGACY_RENEGOTIATION_DISABLED' in str(e
):
4197 raise RequestError('UNSAFE_LEGACY_RENEGOTIATION_DISABLED: Try using --legacy-server-connect', cause
=e
) from e
4198 elif 'SSLV3_ALERT_HANDSHAKE_FAILURE' in str(e
):
4200 'SSLV3_ALERT_HANDSHAKE_FAILURE: The server may not support the current cipher list. '
4201 'Try using --legacy-server-connect', cause
=e
) from e
4204 def build_request_director(self
, handlers
, preferences
=None):
4205 logger
= _YDLLogger(self
)
4206 headers
= self
.params
['http_headers'].copy()
4207 proxies
= self
.proxies
.copy()
4208 clean_headers(headers
)
4209 clean_proxies(proxies
, headers
)
4211 director
= RequestDirector(logger
=logger
, verbose
=self
.params
.get('debug_printtraffic'))
4212 for handler
in handlers
:
4213 director
.add_handler(handler(
4216 cookiejar
=self
.cookiejar
,
4218 prefer_system_certs
='no-certifi' in self
.params
['compat_opts'],
4219 verify
=not self
.params
.get('nocheckcertificate'),
4220 **traverse_obj(self
.params
, {
4221 'verbose': 'debug_printtraffic',
4222 'source_address': 'source_address',
4223 'timeout': 'socket_timeout',
4224 'legacy_ssl_support': 'legacyserverconnect',
4225 'enable_file_urls': 'enable_file_urls',
4226 'impersonate': 'impersonate',
4228 'client_certificate': 'client_certificate',
4229 'client_certificate_key': 'client_certificate_key',
4230 'client_certificate_password': 'client_certificate_password',
4234 director
.preferences
.update(preferences
or [])
4235 if 'prefer-legacy-http-handler' in self
.params
['compat_opts']:
4236 director
.preferences
.add(lambda rh
, _
: 500 if rh
.RH_KEY
== 'Urllib' else 0)
4239 @functools.cached_property
4240 def _request_director(self
):
4241 return self
.build_request_director(_REQUEST_HANDLERS
.values(), _RH_PREFERENCES
)
4243 def encode(self
, s
):
4244 if isinstance(s
, bytes):
4245 return s
# Already encoded
4248 return s
.encode(self
.get_encoding())
4249 except UnicodeEncodeError as err
:
4250 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
4253 def get_encoding(self
):
4254 encoding
= self
.params
.get('encoding')
4255 if encoding
is None:
4256 encoding
= preferredencoding()
4259 def _write_info_json(self
, label
, ie_result
, infofn
, overwrite
=None):
4260 """ Write infojson and returns True = written, 'exists' = Already exists, False = skip, None = error """
4261 if overwrite
is None:
4262 overwrite
= self
.params
.get('overwrites', True)
4263 if not self
.params
.get('writeinfojson'):
4266 self
.write_debug(f
'Skipping writing {label} infojson')
4268 elif not self
._ensure
_dir
_exists
(infofn
):
4270 elif not overwrite
and os
.path
.exists(infofn
):
4271 self
.to_screen(f
'[info] {label.title()} metadata is already present')
4274 self
.to_screen(f
'[info] Writing {label} metadata as JSON to: {infofn}')
4276 write_json_file(self
.sanitize_info(ie_result
, self
.params
.get('clean_infojson', True)), infofn
)
4279 self
.report_error(f
'Cannot write {label} metadata to JSON file {infofn}')
4282 def _write_description(self
, label
, ie_result
, descfn
):
4283 """ Write description and returns True = written, False = skip, None = error """
4284 if not self
.params
.get('writedescription'):
4287 self
.write_debug(f
'Skipping writing {label} description')
4289 elif not self
._ensure
_dir
_exists
(descfn
):
4291 elif not self
.params
.get('overwrites', True) and os
.path
.exists(descfn
):
4292 self
.to_screen(f
'[info] {label.title()} description is already present')
4293 elif ie_result
.get('description') is None:
4294 self
.to_screen(f
'[info] There\'s no {label} description to write')
4298 self
.to_screen(f
'[info] Writing {label} description to: {descfn}')
4299 with open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
4300 descfile
.write(ie_result
['description'])
4302 self
.report_error(f
'Cannot write {label} description file {descfn}')
4306 def _write_subtitles(self
, info_dict
, filename
):
4307 """ Write subtitles to file and return list of (sub_filename, final_sub_filename); or None if error"""
4309 subtitles
= info_dict
.get('requested_subtitles')
4310 if not (self
.params
.get('writesubtitles') or self
.params
.get('writeautomaticsub')):
4311 # subtitles download errors are already managed as troubles in relevant IE
4312 # that way it will silently go on when used with unsupporting IE
4315 self
.to_screen('[info] There are no subtitles for the requested languages')
4317 sub_filename_base
= self
.prepare_filename(info_dict
, 'subtitle')
4318 if not sub_filename_base
:
4319 self
.to_screen('[info] Skipping writing video subtitles')
4322 for sub_lang
, sub_info
in subtitles
.items():
4323 sub_format
= sub_info
['ext']
4324 sub_filename
= subtitles_filename(filename
, sub_lang
, sub_format
, info_dict
.get('ext'))
4325 sub_filename_final
= subtitles_filename(sub_filename_base
, sub_lang
, sub_format
, info_dict
.get('ext'))
4326 existing_sub
= self
.existing_file((sub_filename_final
, sub_filename
))
4328 self
.to_screen(f
'[info] Video subtitle {sub_lang}.{sub_format} is already present')
4329 sub_info
['filepath'] = existing_sub
4330 ret
.append((existing_sub
, sub_filename_final
))
4333 self
.to_screen(f
'[info] Writing video subtitles to: {sub_filename}')
4334 if sub_info
.get('data') is not None:
4336 # Use newline='' to prevent conversion of newline characters
4337 # See https://github.com/ytdl-org/youtube-dl/issues/10268
4338 with open(sub_filename
, 'w', encoding
='utf-8', newline
='') as subfile
:
4339 subfile
.write(sub_info
['data'])
4340 sub_info
['filepath'] = sub_filename
4341 ret
.append((sub_filename
, sub_filename_final
))
4344 self
.report_error(f
'Cannot write video subtitles file {sub_filename}')
4348 sub_copy
= sub_info
.copy()
4349 sub_copy
.setdefault('http_headers', info_dict
.get('http_headers'))
4350 self
.dl(sub_filename
, sub_copy
, subtitle
=True)
4351 sub_info
['filepath'] = sub_filename
4352 ret
.append((sub_filename
, sub_filename_final
))
4353 except (DownloadError
, ExtractorError
, OSError, ValueError, *network_exceptions
) as err
:
4354 msg
= f
'Unable to download video subtitles for {sub_lang!r}: {err}'
4355 if self
.params
.get('ignoreerrors') is not True: # False or 'only_download'
4356 if not self
.params
.get('ignoreerrors'):
4357 self
.report_error(msg
)
4358 raise DownloadError(msg
)
4359 self
.report_warning(msg
)
4362 def _write_thumbnails(self
, label
, info_dict
, filename
, thumb_filename_base
=None):
4363 """ Write thumbnails to file and return list of (thumb_filename, final_thumb_filename); or None if error """
4364 write_all
= self
.params
.get('write_all_thumbnails', False)
4365 thumbnails
, ret
= [], []
4366 if write_all
or self
.params
.get('writethumbnail', False):
4367 thumbnails
= info_dict
.get('thumbnails') or []
4369 self
.to_screen(f
'[info] There are no {label} thumbnails to download')
4371 multiple
= write_all
and len(thumbnails
) > 1
4373 if thumb_filename_base
is None:
4374 thumb_filename_base
= filename
4375 if thumbnails
and not thumb_filename_base
:
4376 self
.write_debug(f
'Skipping writing {label} thumbnail')
4379 if thumbnails
and not self
._ensure
_dir
_exists
(filename
):
4382 for idx
, t
in list(enumerate(thumbnails
))[::-1]:
4383 thumb_ext
= (f
'{t["id"]}.' if multiple
else '') + determine_ext(t
['url'], 'jpg')
4384 thumb_display_id
= f
'{label} thumbnail {t["id"]}'
4385 thumb_filename
= replace_extension(filename
, thumb_ext
, info_dict
.get('ext'))
4386 thumb_filename_final
= replace_extension(thumb_filename_base
, thumb_ext
, info_dict
.get('ext'))
4388 existing_thumb
= self
.existing_file((thumb_filename_final
, thumb_filename
))
4390 self
.to_screen('[info] {} is already present'.format((
4391 thumb_display_id
if multiple
else f
'{label} thumbnail').capitalize()))
4392 t
['filepath'] = existing_thumb
4393 ret
.append((existing_thumb
, thumb_filename_final
))
4395 self
.to_screen(f
'[info] Downloading {thumb_display_id} ...')
4397 uf
= self
.urlopen(Request(t
['url'], headers
=t
.get('http_headers', {})))
4398 self
.to_screen(f
'[info] Writing {thumb_display_id} to: {thumb_filename}')
4399 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
4400 shutil
.copyfileobj(uf
, thumbf
)
4401 ret
.append((thumb_filename
, thumb_filename_final
))
4402 t
['filepath'] = thumb_filename
4403 except network_exceptions
as err
:
4404 if isinstance(err
, HTTPError
) and err
.status
== 404:
4405 self
.to_screen(f
'[info] {thumb_display_id.title()} does not exist')
4407 self
.report_warning(f
'Unable to download {thumb_display_id}: {err}')
4409 if ret
and not write_all
: