24 from string
import ascii_letters
26 from .cache
import Cache
27 from .compat
import compat_os_name
, compat_shlex_quote
28 from .cookies
import load_cookies
29 from .downloader
import FFmpegFD
, get_suitable_downloader
, shorten_protocol_name
30 from .downloader
.rtmp
import rtmpdump_version
31 from .extractor
import gen_extractor_classes
, get_info_extractor
32 from .extractor
.common
import UnsupportedURLIE
33 from .extractor
.openload
import PhantomJSwrapper
34 from .minicurses
import format_text
35 from .postprocessor
import _PLUGIN_CLASSES
as plugin_postprocessors
36 from .postprocessor
import (
38 FFmpegFixupDuplicateMoovPP
,
39 FFmpegFixupDurationPP
,
42 FFmpegFixupStretchedPP
,
43 FFmpegFixupTimestampPP
,
46 FFmpegVideoConvertorPP
,
47 MoveFilesAfterDownloadPP
,
50 from .postprocessor
.ffmpeg
import resolve_mapping
as resolve_recode_mapping
51 from .update
import REPOSITORY
, current_git_head
, detect_variant
77 PerRequestProxyHandler
,
84 UnavailableVideoError
,
86 YoutubeDLCookieProcessor
,
88 YoutubeDLRedirectHandler
,
104 format_decimal_suffix
,
121 orderedSet_from_options
,
125 register_socks_protocols
,
126 remove_terminal_sequences
,
137 supports_terminal_sequences
,
147 windows_enable_vt_mode
,
151 from .version
import RELEASE_GIT_HEAD
, VARIANT
, __version__
153 if compat_os_name
== 'nt':
160 YoutubeDL objects are the ones responsible of downloading the
161 actual video file and writing it to disk if the user has requested
162 it, among some other tasks. In most cases there should be one per
163 program. As, given a video URL, the downloader doesn't know how to
164 extract all the needed information, task that InfoExtractors do, it
165 has to pass the URL to one of them.
167 For this, YoutubeDL objects have a method that allows
168 InfoExtractors to be registered in a given order. When it is passed
169 a URL, the YoutubeDL object handles it to the first InfoExtractor it
170 finds that reports being able to handle it. The InfoExtractor extracts
171 all the information about the video or videos the URL refers to, and
172 YoutubeDL process the extracted information, possibly using a File
173 Downloader to download the video.
175 YoutubeDL objects accept a lot of parameters. In order not to saturate
176 the object constructor with arguments, it receives a dictionary of
177 options instead. These options are available through the params
178 attribute for the InfoExtractors to use. The YoutubeDL also
179 registers itself as the downloader in charge for the InfoExtractors
180 that are added to it, so this is a "mutual registration".
184 username: Username for authentication purposes.
185 password: Password for authentication purposes.
186 videopassword: Password for accessing a video.
187 ap_mso: Adobe Pass multiple-system operator identifier.
188 ap_username: Multiple-system operator account username.
189 ap_password: Multiple-system operator account password.
190 usenetrc: Use netrc for authentication instead.
191 verbose: Print additional info to stdout.
192 quiet: Do not print messages to stdout.
193 no_warnings: Do not print out anything for warnings.
194 forceprint: A dict with keys WHEN mapped to a list of templates to
195 print to stdout. The allowed keys are video or any of the
196 items in utils.POSTPROCESS_WHEN.
197 For compatibility, a single list is also accepted
198 print_to_file: A dict with keys WHEN (same as forceprint) mapped to
199 a list of tuples with (template, filename)
200 forcejson: Force printing info_dict as JSON.
201 dump_single_json: Force printing the info_dict of the whole playlist
202 (or video) as a single JSON line.
203 force_write_download_archive: Force writing download archive regardless
204 of 'skip_download' or 'simulate'.
205 simulate: Do not download the video files. If unset (or None),
206 simulate only if listsubtitles, listformats or list_thumbnails is used
207 format: Video format code. see "FORMAT SELECTION" for more details.
208 You can also pass a function. The function takes 'ctx' as
209 argument and returns the formats to download.
210 See "build_format_selector" for an implementation
211 allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
212 ignore_no_formats_error: Ignore "No video formats" error. Usefull for
213 extracting metadata even if the video is not actually
214 available for download (experimental)
215 format_sort: A list of fields by which to sort the video formats.
216 See "Sorting Formats" for more details.
217 format_sort_force: Force the given format_sort. see "Sorting Formats"
219 prefer_free_formats: Whether to prefer video formats with free containers
220 over non-free ones of same quality.
221 allow_multiple_video_streams: Allow multiple video streams to be merged
223 allow_multiple_audio_streams: Allow multiple audio streams to be merged
225 check_formats Whether to test if the formats are downloadable.
226 Can be True (check all), False (check none),
227 'selected' (check selected formats),
228 or None (check only if requested by extractor)
229 paths: Dictionary of output paths. The allowed keys are 'home'
230 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
231 outtmpl: Dictionary of templates for output names. Allowed keys
232 are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
233 For compatibility with youtube-dl, a single string can also be used
234 outtmpl_na_placeholder: Placeholder for unavailable meta fields.
235 restrictfilenames: Do not allow "&" and spaces in file names
236 trim_file_name: Limit length of filename (extension excluded)
237 windowsfilenames: Force the filenames to be windows compatible
238 ignoreerrors: Do not stop on download/postprocessing errors.
239 Can be 'only_download' to ignore only download errors.
240 Default is 'only_download' for CLI, but False for API
241 skip_playlist_after_errors: Number of allowed failures until the rest of
242 the playlist is skipped
243 allowed_extractors: List of regexes to match against extractor names that are allowed
244 overwrites: Overwrite all video and metadata files if True,
245 overwrite only non-video files if None
246 and don't overwrite any file if False
247 For compatibility with youtube-dl,
248 "nooverwrites" may also be used instead
249 playlist_items: Specific indices of playlist to download.
250 playlistrandom: Download playlist items in random order.
251 lazy_playlist: Process playlist entries as they are received.
252 matchtitle: Download only matching titles.
253 rejecttitle: Reject downloads for matching titles.
254 logger: Log messages to a logging.Logger instance.
255 logtostderr: Print everything to stderr instead of stdout.
256 consoletitle: Display progress in console window's titlebar.
257 writedescription: Write the video description to a .description file
258 writeinfojson: Write the video description to a .info.json file
259 clean_infojson: Remove private fields from the infojson
260 getcomments: Extract video comments. This will not be written to disk
261 unless writeinfojson is also given
262 writeannotations: Write the video annotations to a .annotations.xml file
263 writethumbnail: Write the thumbnail image to a file
264 allow_playlist_files: Whether to write playlists' description, infojson etc
265 also to disk when using the 'write*' options
266 write_all_thumbnails: Write all thumbnail formats to files
267 writelink: Write an internet shortcut file, depending on the
268 current platform (.url/.webloc/.desktop)
269 writeurllink: Write a Windows internet shortcut file (.url)
270 writewebloclink: Write a macOS internet shortcut file (.webloc)
271 writedesktoplink: Write a Linux internet shortcut file (.desktop)
272 writesubtitles: Write the video subtitles to a file
273 writeautomaticsub: Write the automatically generated subtitles to a file
274 listsubtitles: Lists all available subtitles for the video
275 subtitlesformat: The format code for subtitles
276 subtitleslangs: List of languages of the subtitles to download (can be regex).
277 The list may contain "all" to refer to all the available
278 subtitles. The language can be prefixed with a "-" to
279 exclude it from the requested languages, e.g. ['all', '-live_chat']
280 keepvideo: Keep the video file after post-processing
281 daterange: A DateRange object, download only if the upload_date is in the range.
282 skip_download: Skip the actual download of the video file
283 cachedir: Location of the cache files in the filesystem.
284 False to disable filesystem cache.
285 noplaylist: Download single video instead of a playlist if in doubt.
286 age_limit: An integer representing the user's age in years.
287 Unsuitable videos for the given age are skipped.
288 min_views: An integer representing the minimum view count the video
289 must have in order to not be skipped.
290 Videos without view count information are always
291 downloaded. None for no limit.
292 max_views: An integer representing the maximum view count.
293 Videos that are more popular than that are not
295 Videos without view count information are always
296 downloaded. None for no limit.
297 download_archive: A set, or the name of a file where all downloads are recorded.
298 Videos already present in the file are not downloaded again.
299 break_on_existing: Stop the download process after attempting to download a
300 file that is in the archive.
301 break_on_reject: Stop the download process when encountering a video that
302 has been filtered out.
303 break_per_url: Whether break_on_reject and break_on_existing
304 should act on each input URL as opposed to for the entire queue
305 cookiefile: File name or text stream from where cookies should be read and dumped to
306 cookiesfrombrowser: A tuple containing the name of the browser, the profile
307 name/path from where cookies are loaded, the name of the keyring,
308 and the container name, e.g. ('chrome', ) or
309 ('vivaldi', 'default', 'BASICTEXT') or ('firefox', 'default', None, 'Meta')
310 legacyserverconnect: Explicitly allow HTTPS connection to servers that do not
311 support RFC 5746 secure renegotiation
312 nocheckcertificate: Do not verify SSL certificates
313 client_certificate: Path to client certificate file in PEM format. May include the private key
314 client_certificate_key: Path to private key file for client certificate
315 client_certificate_password: Password for client certificate private key, if encrypted.
316 If not provided and the key is encrypted, yt-dlp will ask interactively
317 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
318 (Only supported by some extractors)
319 http_headers: A dictionary of custom headers to be used for all requests
320 proxy: URL of the proxy server to use
321 geo_verification_proxy: URL of the proxy to use for IP address verification
322 on geo-restricted sites.
323 socket_timeout: Time to wait for unresponsive hosts, in seconds
324 bidi_workaround: Work around buggy terminals without bidirectional text
325 support, using fridibi
326 debug_printtraffic:Print out sent and received HTTP traffic
327 default_search: Prepend this string if an input url is not valid.
328 'auto' for elaborate guessing
329 encoding: Use this encoding instead of the system-specified.
330 extract_flat: Whether to resolve and process url_results further
331 * False: Always process (default)
332 * True: Never process
333 * 'in_playlist': Do not process inside playlist/multi_video
334 * 'discard': Always process, but don't return the result
335 from inside playlist/multi_video
336 * 'discard_in_playlist': Same as "discard", but only for
337 playlists (not multi_video)
338 wait_for_video: If given, wait for scheduled streams to become available.
339 The value should be a tuple containing the range
340 (min_secs, max_secs) to wait between retries
341 postprocessors: A list of dictionaries, each with an entry
342 * key: The name of the postprocessor. See
343 yt_dlp/postprocessor/__init__.py for a list.
344 * when: When to run the postprocessor. Allowed values are
345 the entries of utils.POSTPROCESS_WHEN
346 Assumed to be 'post_process' if not given
347 progress_hooks: A list of functions that get called on download
348 progress, with a dictionary with the entries
349 * status: One of "downloading", "error", or "finished".
350 Check this first and ignore unknown values.
351 * info_dict: The extracted info_dict
353 If status is one of "downloading", or "finished", the
354 following properties may also be present:
355 * filename: The final filename (always present)
356 * tmpfilename: The filename we're currently writing to
357 * downloaded_bytes: Bytes on disk
358 * total_bytes: Size of the whole file, None if unknown
359 * total_bytes_estimate: Guess of the eventual file size,
361 * elapsed: The number of seconds since download started.
362 * eta: The estimated time in seconds, None if unknown
363 * speed: The download speed in bytes/second, None if
365 * fragment_index: The counter of the currently
366 downloaded video fragment.
367 * fragment_count: The number of fragments (= individual
368 files that will be merged)
370 Progress hooks are guaranteed to be called at least once
371 (with status "finished") if the download is successful.
372 postprocessor_hooks: A list of functions that get called on postprocessing
373 progress, with a dictionary with the entries
374 * status: One of "started", "processing", or "finished".
375 Check this first and ignore unknown values.
376 * postprocessor: Name of the postprocessor
377 * info_dict: The extracted info_dict
379 Progress hooks are guaranteed to be called at least twice
380 (with status "started" and "finished") if the processing is successful.
381 merge_output_format: "/" separated list of extensions to use when merging formats.
382 final_ext: Expected final extension; used to detect when the file was
383 already downloaded and converted
384 fixup: Automatically correct known faults of the file.
386 - "never": do nothing
387 - "warn": only emit a warning
388 - "detect_or_warn": check whether we can do anything
389 about it, warn otherwise (default)
390 source_address: Client-side IP address to bind to.
391 sleep_interval_requests: Number of seconds to sleep between requests
393 sleep_interval: Number of seconds to sleep before each download when
394 used alone or a lower bound of a range for randomized
395 sleep before each download (minimum possible number
396 of seconds to sleep) when used along with
398 max_sleep_interval:Upper bound of a range for randomized sleep before each
399 download (maximum possible number of seconds to sleep).
400 Must only be used along with sleep_interval.
401 Actual sleep time will be a random float from range
402 [sleep_interval; max_sleep_interval].
403 sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
404 listformats: Print an overview of available video formats and exit.
405 list_thumbnails: Print a table of all thumbnails and exit.
406 match_filter: A function that gets called for every video with the signature
407 (info_dict, *, incomplete: bool) -> Optional[str]
408 For backward compatibility with youtube-dl, the signature
409 (info_dict) -> Optional[str] is also allowed.
410 - If it returns a message, the video is ignored.
411 - If it returns None, the video is downloaded.
412 - If it returns utils.NO_DEFAULT, the user is interactively
413 asked whether to download the video.
414 match_filter_func in utils.py is one example for this.
415 no_color: Do not emit color codes in output.
416 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
419 Two-letter ISO 3166-2 country code that will be used for
420 explicit geographic restriction bypassing via faking
421 X-Forwarded-For HTTP header
423 IP range in CIDR notation that will be used similarly to
425 external_downloader: A dictionary of protocol keys and the executable of the
426 external downloader to use for it. The allowed protocols
427 are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
428 Set the value to 'native' to use the native downloader
429 compat_opts: Compatibility options. See "Differences in default behavior".
430 The following options do not work when used through the API:
431 filename, abort-on-error, multistreams, no-live-chat, format-sort
432 no-clean-infojson, no-playlist-metafiles, no-keep-subs, no-attach-info-json.
433 Refer __init__.py for their implementation
434 progress_template: Dictionary of templates for progress outputs.
435 Allowed keys are 'download', 'postprocess',
436 'download-title' (console title) and 'postprocess-title'.
437 The template is mapped on a dictionary with keys 'progress' and 'info'
438 retry_sleep_functions: Dictionary of functions that takes the number of attempts
439 as argument and returns the time to sleep in seconds.
440 Allowed keys are 'http', 'fragment', 'file_access'
441 download_ranges: A callback function that gets called for every video with
442 the signature (info_dict, ydl) -> Iterable[Section].
443 Only the returned sections will be downloaded.
444 Each Section is a dict with the following keys:
445 * start_time: Start time of the section in seconds
446 * end_time: End time of the section in seconds
447 * title: Section title (Optional)
448 * index: Section number (Optional)
449 force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts
450 noprogress: Do not print the progress bar
451 live_from_start: Whether to download livestreams videos from the start
453 The following parameters are not used by YoutubeDL itself, they are used by
454 the downloader (see yt_dlp/downloader/common.py):
455 nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
456 max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries,
457 continuedl, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
458 external_downloader_args, concurrent_fragment_downloads.
460 The following options are used by the post processors:
461 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
462 to the binary or its containing directory.
463 postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
464 and a list of additional command-line arguments for the
465 postprocessor/executable. The dict can also have "PP+EXE" keys
466 which are used when the given exe is used by the given PP.
467 Use 'default' as the name for arguments to passed to all PP
468 For compatibility with youtube-dl, a single list of args
471 The following options are used by the extractors:
472 extractor_retries: Number of times to retry for known errors
473 dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
474 hls_split_discontinuity: Split HLS playlists to different formats at
475 discontinuities such as ad breaks (default: False)
476 extractor_args: A dictionary of arguments to be passed to the extractors.
477 See "EXTRACTOR ARGUMENTS" for details.
478 E.g. {'youtube': {'skip': ['dash', 'hls']}}
479 mark_watched: Mark videos watched (even with --simulate). Only for YouTube
481 The following options are deprecated and may be removed in the future:
483 force_generic_extractor: Force downloader to use the generic extractor
484 - Use allowed_extractors = ['generic', 'default']
485 playliststart: - Use playlist_items
486 Playlist item to start at.
487 playlistend: - Use playlist_items
488 Playlist item to end at.
489 playlistreverse: - Use playlist_items
490 Download playlist items in reverse order.
491 forceurl: - Use forceprint
492 Force printing final URL.
493 forcetitle: - Use forceprint
494 Force printing title.
495 forceid: - Use forceprint
497 forcethumbnail: - Use forceprint
498 Force printing thumbnail URL.
499 forcedescription: - Use forceprint
500 Force printing description.
501 forcefilename: - Use forceprint
502 Force printing final filename.
503 forceduration: - Use forceprint
504 Force printing duration.
505 allsubtitles: - Use subtitleslangs = ['all']
506 Downloads all the subtitles of the video
507 (requires writesubtitles or writeautomaticsub)
508 include_ads: - Doesn't work
510 call_home: - Not implemented
511 Boolean, true iff we are allowed to contact the
512 yt-dlp servers for debugging.
513 post_hooks: - Register a custom postprocessor
514 A list of functions that get called as the final step
515 for each video file, after all postprocessors have been
516 called. The filename will be passed as the only argument.
517 hls_prefer_native: - Use external_downloader = {'m3u8': 'native'} or {'m3u8': 'ffmpeg'}.
518 Use the native HLS downloader instead of ffmpeg/avconv
519 if True, otherwise use ffmpeg/avconv if False, otherwise
520 use downloader suggested by extractor if None.
521 prefer_ffmpeg: - avconv support is deprecated
522 If False, use avconv instead of ffmpeg if both are available,
523 otherwise prefer ffmpeg.
524 youtube_include_dash_manifest: - Use extractor_args
525 If True (default), DASH manifests and related
526 data will be downloaded and processed by extractor.
527 You can reduce network I/O by disabling it if you don't
528 care about DASH. (only for youtube)
529 youtube_include_hls_manifest: - Use extractor_args
530 If True (default), HLS manifests and related
531 data will be downloaded and processed by extractor.
532 You can reduce network I/O by disabling it if you don't
533 care about HLS. (only for youtube)
537 'width', 'height', 'asr', 'audio_channels', 'fps',
538 'tbr', 'abr', 'vbr', 'filesize', 'filesize_approx',
539 'timestamp', 'release_timestamp',
540 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
541 'average_rating', 'comment_count', 'age_limit',
542 'start_time', 'end_time',
543 'chapter_number', 'season_number', 'episode_number',
544 'track_number', 'disc_number', 'release_year',
548 # NB: Keep in sync with the docstring of extractor/common.py
549 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note',
550 'width', 'height', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels',
551 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx',
552 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
553 'preference', 'language', 'language_preference', 'quality', 'source_preference',
554 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'downloader_options',
555 'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
557 _format_selection_exts
= {
558 'audio': set(MEDIA_EXTENSIONS
.common_audio
),
559 'video': set(MEDIA_EXTENSIONS
.common_video
+ ('3gp', )),
560 'storyboards': set(MEDIA_EXTENSIONS
.storyboards
),
563 def __init__(self
, params
=None, auto_init
=True):
564 """Create a FileDownloader object with the given options.
565 @param auto_init Whether to load the default extractors and print header (if verbose).
566 Set to 'no_verbose_header' to not print the header
572 self
._ies
_instances
= {}
573 self
._pps
= {k: [] for k in POSTPROCESS_WHEN}
574 self
._printed
_messages
= set()
575 self
._first
_webpage
_request
= True
576 self
._post
_hooks
= []
577 self
._progress
_hooks
= []
578 self
._postprocessor
_hooks
= []
579 self
._download
_retcode
= 0
580 self
._num
_downloads
= 0
582 self
._playlist
_level
= 0
583 self
._playlist
_urls
= set()
584 self
.cache
= Cache(self
)
586 windows_enable_vt_mode()
587 stdout
= sys
.stderr
if self
.params
.get('logtostderr') else sys
.stdout
588 self
._out
_files
= Namespace(
591 screen
=sys
.stderr
if self
.params
.get('quiet') else stdout
,
592 console
=None if compat_os_name
== 'nt' else next(
593 filter(supports_terminal_sequences
, (sys
.stderr
, sys
.stdout
)), None)
595 self
._allow
_colors
= Namespace(**{
596 type_
: not self
.params
.get('no_color') and supports_terminal_sequences(stream
)
597 for type_
, stream
in self
._out
_files
.items_
if type_
!= 'console'
600 # The code is left like this to be reused for future deprecations
601 MIN_SUPPORTED
, MIN_RECOMMENDED
= (3, 7), (3, 7)
602 current_version
= sys
.version_info
[:2]
603 if current_version
< MIN_RECOMMENDED
:
604 msg
= ('Support for Python version %d.%d has been deprecated. '
605 'See https://github.com/yt-dlp/yt-dlp/issues/3764 for more details.'
606 '\n You will no longer receive updates on this version')
607 if current_version
< MIN_SUPPORTED
:
608 msg
= 'Python version %d.%d is no longer supported'
609 self
.deprecation_warning(
610 f
'{msg}! Please update to Python %d.%d or above' % (*current_version
, *MIN_RECOMMENDED
))
612 if self
.params
.get('allow_unplayable_formats'):
614 f
'You have asked for {self._format_err("UNPLAYABLE", self.Styles.EMPHASIS)} formats to be listed/downloaded. '
615 'This is a developer option intended for debugging. \n'
616 ' If you experience any issues while using this option, '
617 f
'{self._format_err("DO NOT", self.Styles.ERROR)} open a bug report')
619 def check_deprecated(param
, option
, suggestion
):
620 if self
.params
.get(param
) is not None:
621 self
.report_warning(f
'{option} is deprecated. Use {suggestion} instead')
625 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
626 if self
.params
.get('geo_verification_proxy') is None:
627 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
629 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
630 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
631 check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
633 for msg
in self
.params
.get('_warnings', []):
634 self
.report_warning(msg
)
635 for msg
in self
.params
.get('_deprecation_warnings', []):
636 self
.deprecated_feature(msg
)
638 self
.params
['compat_opts'] = set(self
.params
.get('compat_opts', ()))
639 if 'list-formats' in self
.params
['compat_opts']:
640 self
.params
['listformats_table'] = False
642 if 'overwrites' not in self
.params
and self
.params
.get('nooverwrites') is not None:
643 # nooverwrites was unnecessarily changed to overwrites
644 # in 0c3d0f51778b153f65c21906031c2e091fcfb641
645 # This ensures compatibility with both keys
646 self
.params
['overwrites'] = not self
.params
['nooverwrites']
647 elif self
.params
.get('overwrites') is None:
648 self
.params
.pop('overwrites', None)
650 self
.params
['nooverwrites'] = not self
.params
['overwrites']
652 self
.params
.setdefault('forceprint', {})
653 self
.params
.setdefault('print_to_file', {})
655 # Compatibility with older syntax
656 if not isinstance(params
['forceprint'], dict):
657 self
.params
['forceprint'] = {'video': params['forceprint']}
659 if self
.params
.get('bidi_workaround', False):
662 master
, slave
= pty
.openpty()
663 width
= shutil
.get_terminal_size().columns
664 width_args
= [] if width
is None else ['-w', str(width
)]
665 sp_kwargs
= {'stdin': subprocess.PIPE, 'stdout': slave, 'stderr': self._out_files.error}
667 self
._output
_process
= Popen(['bidiv'] + width_args
, **sp_kwargs
)
669 self
._output
_process
= Popen(['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
670 self
._output
_channel
= os
.fdopen(master
, 'rb')
671 except OSError as ose
:
672 if ose
.errno
== errno
.ENOENT
:
674 'Could not find fribidi executable, ignoring --bidi-workaround. '
675 'Make sure that fribidi is an executable file in one of the directories in your $PATH.')
680 if auto_init
!= 'no_verbose_header':
681 self
.print_debug_header()
682 self
.add_default_info_extractors()
684 if (sys
.platform
!= 'win32'
685 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
686 and not self
.params
.get('restrictfilenames', False)):
687 # Unicode filesystem API will throw errors (#1474, #13027)
689 'Assuming --restrict-filenames since file system encoding '
690 'cannot encode all characters. '
691 'Set the LC_ALL environment variable to fix this.')
692 self
.params
['restrictfilenames'] = True
694 self
._parse
_outtmpl
()
696 # Creating format selector here allows us to catch syntax errors before the extraction
697 self
.format_selector
= (
698 self
.params
.get('format') if self
.params
.get('format') in (None, '-')
699 else self
.params
['format'] if callable(self
.params
['format'])
700 else self
.build_format_selector(self
.params
['format']))
702 # Set http_headers defaults according to std_headers
703 self
.params
['http_headers'] = merge_headers(std_headers
, self
.params
.get('http_headers', {}))
706 'post_hooks': self
.add_post_hook
,
707 'progress_hooks': self
.add_progress_hook
,
708 'postprocessor_hooks': self
.add_postprocessor_hook
,
710 for opt
, fn
in hooks
.items():
711 for ph
in self
.params
.get(opt
, []):
714 for pp_def_raw
in self
.params
.get('postprocessors', []):
715 pp_def
= dict(pp_def_raw
)
716 when
= pp_def
.pop('when', 'post_process')
717 self
.add_post_processor(
718 get_postprocessor(pp_def
.pop('key'))(self
, **pp_def
),
722 register_socks_protocols()
724 def preload_download_archive(fn
):
725 """Preload the archive, if any is specified"""
729 elif not is_path_like(fn
):
732 self
.write_debug(f
'Loading archive file {fn!r}')
734 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
735 for line
in archive_file
:
736 archive
.add(line
.strip())
737 except OSError as ioe
:
738 if ioe
.errno
!= errno
.ENOENT
:
742 self
.archive
= preload_download_archive(self
.params
.get('download_archive'))
744 def warn_if_short_id(self
, argv
):
745 # short YouTube ID starting with dash?
747 i
for i
, a
in enumerate(argv
)
748 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
752 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
753 + ['--'] + [argv
[i
] for i
in idxs
]
756 'Long argument string detected. '
757 'Use -- to separate parameters and URLs, like this:\n%s' %
758 args_to_str(correct_argv
))
760 def add_info_extractor(self
, ie
):
761 """Add an InfoExtractor object to the end of the list."""
763 self
._ies
[ie_key
] = ie
764 if not isinstance(ie
, type):
765 self
._ies
_instances
[ie_key
] = ie
766 ie
.set_downloader(self
)
768 def get_info_extractor(self
, ie_key
):
770 Get an instance of an IE with name ie_key, it will try to get one from
771 the _ies list, if there's no instance it will create a new one and add
772 it to the extractor list.
774 ie
= self
._ies
_instances
.get(ie_key
)
776 ie
= get_info_extractor(ie_key
)()
777 self
.add_info_extractor(ie
)
780 def add_default_info_extractors(self
):
782 Add the InfoExtractors returned by gen_extractors to the end of the list
784 all_ies
= {ie.IE_NAME.lower(): ie for ie in gen_extractor_classes()}
785 all_ies
['end'] = UnsupportedURLIE()
787 ie_names
= orderedSet_from_options(
788 self
.params
.get('allowed_extractors', ['default']), {
789 'all': list(all_ies
),
790 'default': [name
for name
, ie
in all_ies
.items() if ie
._ENABLED
],
792 except re
.error
as e
:
793 raise ValueError(f
'Wrong regex for allowed_extractors: {e.pattern}')
794 for name
in ie_names
:
795 self
.add_info_extractor(all_ies
[name
])
796 self
.write_debug(f
'Loaded {len(ie_names)} extractors')
798 def add_post_processor(self
, pp
, when
='post_process'):
799 """Add a PostProcessor object to the end of the chain."""
800 assert when
in POSTPROCESS_WHEN
, f
'Invalid when={when}'
801 self
._pps
[when
].append(pp
)
802 pp
.set_downloader(self
)
804 def add_post_hook(self
, ph
):
805 """Add the post hook"""
806 self
._post
_hooks
.append(ph
)
808 def add_progress_hook(self
, ph
):
809 """Add the download progress hook"""
810 self
._progress
_hooks
.append(ph
)
812 def add_postprocessor_hook(self
, ph
):
813 """Add the postprocessing progress hook"""
814 self
._postprocessor
_hooks
.append(ph
)
815 for pps
in self
._pps
.values():
817 pp
.add_progress_hook(ph
)
819 def _bidi_workaround(self
, message
):
820 if not hasattr(self
, '_output_channel'):
823 assert hasattr(self
, '_output_process')
824 assert isinstance(message
, str)
825 line_count
= message
.count('\n') + 1
826 self
._output
_process
.stdin
.write((message
+ '\n').encode())
827 self
._output
_process
.stdin
.flush()
828 res
= ''.join(self
._output
_channel
.readline().decode()
829 for _
in range(line_count
))
830 return res
[:-len('\n')]
832 def _write_string(self
, message
, out
=None, only_once
=False):
834 if message
in self
._printed
_messages
:
836 self
._printed
_messages
.add(message
)
837 write_string(message
, out
=out
, encoding
=self
.params
.get('encoding'))
839 def to_stdout(self
, message
, skip_eol
=False, quiet
=None):
840 """Print message to stdout"""
841 if quiet
is not None:
842 self
.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. '
843 'Use "YoutubeDL.to_screen" instead')
844 if skip_eol
is not False:
845 self
.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument skip_eol. '
846 'Use "YoutubeDL.to_screen" instead')
847 self
._write
_string
(f
'{self._bidi_workaround(message)}\n', self
._out
_files
.out
)
849 def to_screen(self
, message
, skip_eol
=False, quiet
=None):
850 """Print message to screen if not in quiet mode"""
851 if self
.params
.get('logger'):
852 self
.params
['logger'].debug(message
)
854 if (self
.params
.get('quiet') if quiet
is None else quiet
) and not self
.params
.get('verbose'):
857 '%s%s' % (self
._bidi
_workaround
(message
), ('' if skip_eol
else '\n')),
858 self
._out
_files
.screen
)
860 def to_stderr(self
, message
, only_once
=False):
861 """Print message to stderr"""
862 assert isinstance(message
, str)
863 if self
.params
.get('logger'):
864 self
.params
['logger'].error(message
)
866 self
._write
_string
(f
'{self._bidi_workaround(message)}\n', self
._out
_files
.error
, only_once
=only_once
)
868 def _send_console_code(self
, code
):
869 if compat_os_name
== 'nt' or not self
._out
_files
.console
:
871 self
._write
_string
(code
, self
._out
_files
.console
)
873 def to_console_title(self
, message
):
874 if not self
.params
.get('consoletitle', False):
876 message
= remove_terminal_sequences(message
)
877 if compat_os_name
== 'nt':
878 if ctypes
.windll
.kernel32
.GetConsoleWindow():
879 # c_wchar_p() might not be necessary if `message` is
880 # already of type unicode()
881 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
883 self
._send
_console
_code
(f
'\033]0;{message}\007')
885 def save_console_title(self
):
886 if not self
.params
.get('consoletitle') or self
.params
.get('simulate'):
888 self
._send
_console
_code
('\033[22;0t') # Save the title on stack
890 def restore_console_title(self
):
891 if not self
.params
.get('consoletitle') or self
.params
.get('simulate'):
893 self
._send
_console
_code
('\033[23;0t') # Restore the title from stack
896 self
.save_console_title()
899 def __exit__(self
, *args
):
900 self
.restore_console_title()
902 if self
.params
.get('cookiefile') is not None:
903 self
.cookiejar
.save(ignore_discard
=True, ignore_expires
=True)
905 def trouble(self
, message
=None, tb
=None, is_error
=True):
906 """Determine action to take when a download problem appears.
908 Depending on if the downloader has been configured to ignore
909 download errors or not, this method may throw an exception or
910 not when errors are found, after printing the message.
912 @param tb If given, is additional traceback information
913 @param is_error Whether to raise error according to ignorerrors
915 if message
is not None:
916 self
.to_stderr(message
)
917 if self
.params
.get('verbose'):
919 if sys
.exc_info()[0]: # if .trouble has been called from an except block
921 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
922 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
923 tb
+= encode_compat_str(traceback
.format_exc())
925 tb_data
= traceback
.format_list(traceback
.extract_stack())
926 tb
= ''.join(tb_data
)
931 if not self
.params
.get('ignoreerrors'):
932 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
933 exc_info
= sys
.exc_info()[1].exc_info
935 exc_info
= sys
.exc_info()
936 raise DownloadError(message
, exc_info
)
937 self
._download
_retcode
= 1
941 EMPHASIS
='light blue',
947 SUPPRESS
='light black',
950 def _format_text(self
, handle
, allow_colors
, text
, f
, fallback
=None, *, test_encoding
=False):
954 # handle.encoding can be None. See https://github.com/yt-dlp/yt-dlp/issues/2711
955 encoding
= self
.params
.get('encoding') or getattr(handle
, 'encoding', None) or 'ascii'
956 text
= text
.encode(encoding
, 'ignore').decode(encoding
)
957 if fallback
is not None and text
!= original_text
:
959 return format_text(text
, f
) if allow_colors
else text
if fallback
is None else fallback
961 def _format_out(self
, *args
, **kwargs
):
962 return self
._format
_text
(self
._out
_files
.out
, self
._allow
_colors
.out
, *args
, **kwargs
)
964 def _format_screen(self
, *args
, **kwargs
):
965 return self
._format
_text
(self
._out
_files
.screen
, self
._allow
_colors
.screen
, *args
, **kwargs
)
967 def _format_err(self
, *args
, **kwargs
):
968 return self
._format
_text
(self
._out
_files
.error
, self
._allow
_colors
.error
, *args
, **kwargs
)
970 def report_warning(self
, message
, only_once
=False):
972 Print the message to stderr, it will be prefixed with 'WARNING:'
973 If stderr is a tty file the 'WARNING:' will be colored
975 if self
.params
.get('logger') is not None:
976 self
.params
['logger'].warning(message
)
978 if self
.params
.get('no_warnings'):
980 self
.to_stderr(f
'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once
)
982 def deprecation_warning(self
, message
, *, stacklevel
=0):
984 message
, stacklevel
=stacklevel
+ 1, printer
=self
.report_error
, is_error
=False)
986 def deprecated_feature(self
, message
):
987 if self
.params
.get('logger') is not None:
988 self
.params
['logger'].warning(f
'Deprecated Feature: {message}')
989 self
.to_stderr(f
'{self._format_err("Deprecated Feature:", self.Styles.ERROR)} {message}', True)
991 def report_error(self
, message
, *args
, **kwargs
):
993 Do the same as trouble, but prefixes the message with 'ERROR:', colored
994 in red if stderr is a tty file.
996 self
.trouble(f
'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args
, **kwargs
)
998 def write_debug(self
, message
, only_once
=False):
999 '''Log debug message or Print message to stderr'''
1000 if not self
.params
.get('verbose', False):
1002 message
= f
'[debug] {message}'
1003 if self
.params
.get('logger'):
1004 self
.params
['logger'].debug(message
)
1006 self
.to_stderr(message
, only_once
)
1008 def report_file_already_downloaded(self
, file_name
):
1009 """Report file has already been fully downloaded."""
1011 self
.to_screen('[download] %s has already been downloaded' % file_name
)
1012 except UnicodeEncodeError:
1013 self
.to_screen('[download] The file has already been downloaded')
1015 def report_file_delete(self
, file_name
):
1016 """Report that existing file will be deleted."""
1018 self
.to_screen('Deleting existing file %s' % file_name
)
1019 except UnicodeEncodeError:
1020 self
.to_screen('Deleting existing file')
1022 def raise_no_formats(self
, info
, forced
=False, *, msg
=None):
1023 has_drm
= info
.get('_has_drm')
1024 ignored
, expected
= self
.params
.get('ignore_no_formats_error'), bool(msg
)
1025 msg
= msg
or has_drm
and 'This video is DRM protected' or 'No video formats found!'
1026 if forced
or not ignored
:
1027 raise ExtractorError(msg
, video_id
=info
['id'], ie
=info
['extractor'],
1028 expected
=has_drm
or ignored
or expected
)
1030 self
.report_warning(msg
)
1032 def parse_outtmpl(self
):
1033 self
.deprecation_warning('"YoutubeDL.parse_outtmpl" is deprecated and may be removed in a future version')
1034 self
._parse
_outtmpl
()
1035 return self
.params
['outtmpl']
1037 def _parse_outtmpl(self
):
1039 if self
.params
.get('restrictfilenames'): # Remove spaces in the default template
1040 sanitize
= lambda x
: x
.replace(' - ', ' ').replace(' ', '-')
1042 outtmpl
= self
.params
.setdefault('outtmpl', {})
1043 if not isinstance(outtmpl
, dict):
1044 self
.params
['outtmpl'] = outtmpl
= {'default': outtmpl}
1045 outtmpl
.update({k: sanitize(v) for k, v in DEFAULT_OUTTMPL.items() if outtmpl.get(k) is None}
)
1047 def get_output_path(self
, dir_type
='', filename
=None):
1048 paths
= self
.params
.get('paths', {})
1049 assert isinstance(paths
, dict), '"paths" parameter must be a dictionary'
1050 path
= os
.path
.join(
1051 expand_path(paths
.get('home', '').strip()),
1052 expand_path(paths
.get(dir_type
, '').strip()) if dir_type
else '',
1054 return sanitize_path(path
, force
=self
.params
.get('windowsfilenames'))
1057 def _outtmpl_expandpath(outtmpl
):
1058 # expand_path translates '%%' into '%' and '$$' into '$'
1059 # correspondingly that is not what we want since we need to keep
1060 # '%%' intact for template dict substitution step. Working around
1061 # with boundary-alike separator hack.
1062 sep
= ''.join([random
.choice(ascii_letters
) for _
in range(32)])
1063 outtmpl
= outtmpl
.replace('%%', f
'%{sep}%').replace('$$', f
'${sep}$')
1065 # outtmpl should be expand_path'ed before template dict substitution
1066 # because meta fields may contain env variables we don't want to
1067 # be expanded. E.g. for outtmpl "%(title)s.%(ext)s" and
1068 # title "Hello $PATH", we don't want `$PATH` to be expanded.
1069 return expand_path(outtmpl
).replace(sep
, '')
1072 def escape_outtmpl(outtmpl
):
1073 ''' Escape any remaining strings like %s, %abc% etc. '''
1075 STR_FORMAT_RE_TMPL
.format('', '(?![%(\0])'),
1076 lambda mobj
: ('' if mobj
.group('has_key') else '%') + mobj
.group(0),
1080 def validate_outtmpl(cls
, outtmpl
):
1081 ''' @return None or Exception object '''
1083 STR_FORMAT_RE_TMPL
.format('[^)]*', '[ljhqBUDS]'),
1084 lambda mobj
: f
'{mobj.group(0)[:-1]}s',
1085 cls
._outtmpl
_expandpath
(outtmpl
))
1087 cls
.escape_outtmpl(outtmpl
) % collections
.defaultdict(int)
1089 except ValueError as err
:
1093 def _copy_infodict(info_dict
):
1094 info_dict
= dict(info_dict
)
1095 info_dict
.pop('__postprocessors', None)
1096 info_dict
.pop('__pending_error', None)
1099 def prepare_outtmpl(self
, outtmpl
, info_dict
, sanitize
=False):
1100 """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict
1101 @param sanitize Whether to sanitize the output as a filename.
1102 For backward compatibility, a function can also be passed
1105 info_dict
.setdefault('epoch', int(time
.time())) # keep epoch consistent once set
1107 info_dict
= self
._copy
_infodict
(info_dict
)
1108 info_dict
['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
1109 formatSeconds(info_dict
['duration'], '-' if sanitize
else ':')
1110 if info_dict
.get('duration', None) is not None
1112 info_dict
['autonumber'] = int(self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
)
1113 info_dict
['video_autonumber'] = self
._num
_videos
1114 if info_dict
.get('resolution') is None:
1115 info_dict
['resolution'] = self
.format_resolution(info_dict
, default
=None)
1117 # For fields playlist_index, playlist_autonumber and autonumber convert all occurrences
1118 # of %(field)s to %(field)0Nd for backward compatibility
1119 field_size_compat_map
= {
1120 'playlist_index': number_of_digits(info_dict
.get('__last_playlist_index') or 0),
1121 'playlist_autonumber': number_of_digits(info_dict
.get('n_entries') or 0),
1122 'autonumber': self
.params
.get('autonumber_size') or 5,
1126 EXTERNAL_FORMAT_RE
= re
.compile(STR_FORMAT_RE_TMPL
.format('[^)]*', f
'[{STR_FORMAT_TYPES}ljhqBUDS]'))
1131 # Field is of the form key1.key2...
1132 # where keys (except first) can be string, int, slice or "{field, ...}"
1133 FIELD_INNER_RE
= r
'(?:\w+|%(num)s|%(num)s?(?::%(num)s?){1,2})' % {'num': r'(?:-?\d+)'}
1134 FIELD_RE
= r
'\w*(?:\.(?:%(inner)s|{%(field)s(?:,%(field)s)*}))*' % {
1135 'inner': FIELD_INNER_RE
,
1136 'field': rf
'\w*(?:\.{FIELD_INNER_RE})*'
1138 MATH_FIELD_RE
= rf
'(?:{FIELD_RE}|-?{NUMBER_RE})'
1139 MATH_OPERATORS_RE
= r
'(?:%s)' % '|'.join(map(re
.escape
, MATH_FUNCTIONS
.keys()))
1140 INTERNAL_FORMAT_RE
= re
.compile(rf
'''(?x)
1142 (?P<fields>{FIELD_RE})
1143 (?P<maths>(?:{MATH_OPERATORS_RE}{MATH_FIELD_RE})*)
1144 (?:>(?P<strf_format>.+?))?
1146 (?P<alternate>(?<!\\),[^|&)]+)?
1147 (?:&(?P<replacement>.*?))?
1148 (?:\|(?P<default>.*?))?
1151 def _traverse_infodict(fields
):
1152 fields
= [f
for x
in re
.split(r
'\.({.+?})\.?', fields
)
1153 for f
in ([x
] if x
.startswith('{') else x
.split('.'))]
1155 if fields
and not fields
[i
]:
1158 for i
, f
in enumerate(fields
):
1159 if not f
.startswith('{'):
1161 assert f
.endswith('}'), f
'No closing brace for {f} in {fields}'
1162 fields
[i
] = {k: k.split('.') for k in f[1:-1].split(',')}
1164 return traverse_obj(info_dict
, fields
, is_user_input
=True, traverse_string
=True)
1166 def get_value(mdict
):
1168 value
= _traverse_infodict(mdict
['fields'])
1171 value
= float_or_none(value
)
1172 if value
is not None:
1175 offset_key
= mdict
['maths']
1177 value
= float_or_none(value
)
1181 MATH_FIELD_RE
if operator
else MATH_OPERATORS_RE
,
1182 offset_key
).group(0)
1183 offset_key
= offset_key
[len(item
):]
1184 if operator
is None:
1185 operator
= MATH_FUNCTIONS
[item
]
1187 item
, multiplier
= (item
[1:], -1) if item
[0] == '-' else (item
, 1)
1188 offset
= float_or_none(item
)
1190 offset
= float_or_none(_traverse_infodict(item
))
1192 value
= operator(value
, multiplier
* offset
)
1193 except (TypeError, ZeroDivisionError):
1196 # Datetime formatting
1197 if mdict
['strf_format']:
1198 value
= strftime_or_none(value
, mdict
['strf_format'].replace('\\,', ','))
1200 # XXX: Workaround for https://github.com/yt-dlp/yt-dlp/issues/4485
1201 if sanitize
and value
== '':
1205 na
= self
.params
.get('outtmpl_na_placeholder', 'NA')
1207 def filename_sanitizer(key
, value
, restricted
=self
.params
.get('restrictfilenames')):
1208 return sanitize_filename(str(value
), restricted
=restricted
, is_id
=(
1209 bool(re
.search(r
'(^|[_.])id(\.|$)', key
))
1210 if 'filename-sanitization' in self
.params
['compat_opts']
1213 sanitizer
= sanitize
if callable(sanitize
) else filename_sanitizer
1214 sanitize
= bool(sanitize
)
1216 def _dumpjson_default(obj
):
1217 if isinstance(obj
, (set, LazyList
)):
1221 def create_key(outer_mobj
):
1222 if not outer_mobj
.group('has_key'):
1223 return outer_mobj
.group(0)
1224 key
= outer_mobj
.group('key')
1225 mobj
= re
.match(INTERNAL_FORMAT_RE
, key
)
1226 initial_field
= mobj
.group('fields') if mobj
else ''
1227 value
, replacement
, default
= None, None, na
1229 mobj
= mobj
.groupdict()
1230 default
= mobj
['default'] if mobj
['default'] is not None else default
1231 value
= get_value(mobj
)
1232 replacement
= mobj
['replacement']
1233 if value
is None and mobj
['alternate']:
1234 mobj
= re
.match(INTERNAL_FORMAT_RE
, mobj
['remaining'][1:])
1238 fmt
= outer_mobj
.group('format')
1239 if fmt
== 's' and value
is not None and key
in field_size_compat_map
.keys():
1240 fmt
= f
'0{field_size_compat_map[key]:d}d'
1242 value
= default
if value
is None else value
if replacement
is None else replacement
1244 flags
= outer_mobj
.group('conversion') or ''
1245 str_fmt
= f
'{fmt[:-1]}s'
1246 if fmt
[-1] == 'l': # list
1247 delim
= '\n' if '#' in flags
else ', '
1248 value
, fmt
= delim
.join(map(str, variadic(value
, allowed_types
=(str, bytes)))), str_fmt
1249 elif fmt
[-1] == 'j': # json
1250 value
, fmt
= json
.dumps(value
, default
=_dumpjson_default
, indent
=4 if '#' in flags
else None), str_fmt
1251 elif fmt
[-1] == 'h': # html
1252 value
, fmt
= escapeHTML(value
), str_fmt
1253 elif fmt
[-1] == 'q': # quoted
1254 value
= map(str, variadic(value
) if '#' in flags
else [value
])
1255 value
, fmt
= ' '.join(map(compat_shlex_quote
, value
)), str_fmt
1256 elif fmt
[-1] == 'B': # bytes
1257 value
= f
'%{str_fmt}'.encode() % str(value
).encode()
1258 value
, fmt
= value
.decode('utf-8', 'ignore'), 's'
1259 elif fmt
[-1] == 'U': # unicode normalized
1260 value
, fmt
= unicodedata
.normalize(
1261 # "+" = compatibility equivalence, "#" = NFD
1262 'NF%s%s' % ('K' if '+' in flags
else '', 'D' if '#' in flags
else 'C'),
1264 elif fmt
[-1] == 'D': # decimal suffix
1265 num_fmt
, fmt
= fmt
[:-1].replace('#', ''), 's'
1266 value
= format_decimal_suffix(value
, f
'%{num_fmt}f%s' if num_fmt
else '%d%s',
1267 factor
=1024 if '#' in flags
else 1000)
1268 elif fmt
[-1] == 'S': # filename sanitization
1269 value
, fmt
= filename_sanitizer(initial_field
, value
, restricted
='#' in flags
), str_fmt
1270 elif fmt
[-1] == 'c':
1272 value
= str(value
)[0]
1275 elif fmt
[-1] not in 'rs': # numeric
1276 value
= float_or_none(value
)
1278 value
, fmt
= default
, 's'
1282 # If value is an object, sanitize might convert it to a string
1283 # So we convert it to repr first
1284 value
, fmt
= repr(value
), str_fmt
1285 if fmt
[-1] in 'csr':
1286 value
= sanitizer(initial_field
, value
)
1288 key
= '%s\0%s' % (key
.replace('%', '%\0'), outer_mobj
.group('format'))
1289 TMPL_DICT
[key
] = value
1290 return '{prefix}%({key}){fmt}'.format(key
=key
, fmt
=fmt
, prefix
=outer_mobj
.group('prefix'))
1292 return EXTERNAL_FORMAT_RE
.sub(create_key
, outtmpl
), TMPL_DICT
1294 def evaluate_outtmpl(self
, outtmpl
, info_dict
, *args
, **kwargs
):
1295 outtmpl
, info_dict
= self
.prepare_outtmpl(outtmpl
, info_dict
, *args
, **kwargs
)
1296 return self
.escape_outtmpl(outtmpl
) % info_dict
1298 def _prepare_filename(self
, info_dict
, *, outtmpl
=None, tmpl_type
=None):
1299 assert None in (outtmpl
, tmpl_type
), 'outtmpl and tmpl_type are mutually exclusive'
1301 outtmpl
= self
.params
['outtmpl'].get(tmpl_type
or 'default', self
.params
['outtmpl']['default'])
1303 outtmpl
= self
._outtmpl
_expandpath
(outtmpl
)
1304 filename
= self
.evaluate_outtmpl(outtmpl
, info_dict
, True)
1308 if tmpl_type
in ('', 'temp'):
1309 final_ext
, ext
= self
.params
.get('final_ext'), info_dict
.get('ext')
1310 if final_ext
and ext
and final_ext
!= ext
and filename
.endswith(f
'.{final_ext}'):
1311 filename
= replace_extension(filename
, ext
, final_ext
)
1313 force_ext
= OUTTMPL_TYPES
[tmpl_type
]
1315 filename
= replace_extension(filename
, force_ext
, info_dict
.get('ext'))
1317 # https://github.com/blackjack4494/youtube-dlc/issues/85
1318 trim_file_name
= self
.params
.get('trim_file_name', False)
1320 no_ext
, *ext
= filename
.rsplit('.', 2)
1321 filename
= join_nonempty(no_ext
[:trim_file_name
], *ext
, delim
='.')
1324 except ValueError as err
:
1325 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
1328 def prepare_filename(self
, info_dict
, dir_type
='', *, outtmpl
=None, warn
=False):
1329 """Generate the output filename"""
1331 assert not dir_type
, 'outtmpl and dir_type are mutually exclusive'
1333 filename
= self
._prepare
_filename
(info_dict
, tmpl_type
=dir_type
, outtmpl
=outtmpl
)
1334 if not filename
and dir_type
not in ('', 'temp'):
1338 if not self
.params
.get('paths'):
1340 elif filename
== '-':
1341 self
.report_warning('--paths is ignored when an outputting to stdout', only_once
=True)
1342 elif os
.path
.isabs(filename
):
1343 self
.report_warning('--paths is ignored since an absolute path is given in output template', only_once
=True)
1344 if filename
== '-' or not filename
:
1347 return self
.get_output_path(dir_type
, filename
)
1349 def _match_entry(self
, info_dict
, incomplete
=False, silent
=False):
1350 """ Returns None if the file should be downloaded """
1352 video_title
= info_dict
.get('title', info_dict
.get('id', 'entry'))
1355 if 'title' in info_dict
:
1356 # This can happen when we're just evaluating the playlist
1357 title
= info_dict
['title']
1358 matchtitle
= self
.params
.get('matchtitle', False)
1360 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
1361 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
1362 rejecttitle
= self
.params
.get('rejecttitle', False)
1364 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
1365 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
1366 date
= info_dict
.get('upload_date')
1367 if date
is not None:
1368 dateRange
= self
.params
.get('daterange', DateRange())
1369 if date
not in dateRange
:
1370 return f
'{date_from_str(date).isoformat()} upload date is not in range {dateRange}'
1371 view_count
= info_dict
.get('view_count')
1372 if view_count
is not None:
1373 min_views
= self
.params
.get('min_views')
1374 if min_views
is not None and view_count
< min_views
:
1375 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
1376 max_views
= self
.params
.get('max_views')
1377 if max_views
is not None and view_count
> max_views
:
1378 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
1379 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
1380 return 'Skipping "%s" because it is age restricted' % video_title
1382 match_filter
= self
.params
.get('match_filter')
1383 if match_filter
is not None:
1385 ret
= match_filter(info_dict
, incomplete
=incomplete
)
1387 # For backward compatibility
1388 ret
= None if incomplete
else match_filter(info_dict
)
1389 if ret
is NO_DEFAULT
:
1391 filename
= self
._format
_screen
(self
.prepare_filename(info_dict
), self
.Styles
.FILENAME
)
1392 reply
= input(self
._format
_screen
(
1393 f
'Download "{filename}"? (Y/n): ', self
.Styles
.EMPHASIS
)).lower().strip()
1394 if reply
in {'y', ''}
:
1397 return f
'Skipping {video_title}'
1398 elif ret
is not None:
1402 if self
.in_download_archive(info_dict
):
1403 reason
= '%s has already been recorded in the archive' % video_title
1404 break_opt
, break_err
= 'break_on_existing', ExistingVideoReached
1406 reason
= check_filter()
1407 break_opt
, break_err
= 'break_on_reject', RejectedVideoReached
1408 if reason
is not None:
1410 self
.to_screen('[download] ' + reason
)
1411 if self
.params
.get(break_opt
, False):
1416 def add_extra_info(info_dict
, extra_info
):
1417 '''Set the keys from extra_info in info dict if they are missing'''
1418 for key
, value
in extra_info
.items():
1419 info_dict
.setdefault(key
, value
)
1421 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
=None,
1422 process
=True, force_generic_extractor
=False):
1424 Extract and return the information dictionary of the URL
1427 @param url URL to extract
1430 @param download Whether to download videos
1431 @param process Whether to resolve all unresolved references (URLs, playlist items).
1432 Must be True for download to work
1433 @param ie_key Use only the extractor with this key
1435 @param extra_info Dictionary containing the extra values to add to the info (For internal use only)
1436 @force_generic_extractor Force using the generic extractor (Deprecated; use ie_key='Generic')
1439 if extra_info
is None:
1442 if not ie_key
and force_generic_extractor
:
1446 ies
= {ie_key: self._ies[ie_key]}
if ie_key
in self
._ies
else {}
1450 for key
, ie
in ies
.items():
1451 if not ie
.suitable(url
):
1454 if not ie
.working():
1455 self
.report_warning('The program functionality for this site has been marked as broken, '
1456 'and will probably not work.')
1458 temp_id
= ie
.get_temp_id(url
)
1459 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': key}
):
1460 self
.to_screen(f
'[{key}] {temp_id}: has already been recorded in the archive')
1461 if self
.params
.get('break_on_existing', False):
1462 raise ExistingVideoReached()
1464 return self
.__extract
_info
(url
, self
.get_info_extractor(key
), download
, extra_info
, process
)
1466 extractors_restricted
= self
.params
.get('allowed_extractors') not in (None, ['default'])
1467 self
.report_error(f
'No suitable extractor{format_field(ie_key, None, " (%s)")} found for URL {url}',
1468 tb
=False if extractors_restricted
else None)
1470 def _handle_extraction_exceptions(func
):
1471 @functools.wraps(func
)
1472 def wrapper(self
, *args
, **kwargs
):
1475 return func(self
, *args
, **kwargs
)
1476 except (DownloadCancelled
, LazyList
.IndexError, PagedList
.IndexError):
1478 except ReExtractInfo
as e
:
1480 self
.to_screen(f
'{e}; Re-extracting data')
1482 self
.to_stderr('\r')
1483 self
.report_warning(f
'{e}; Re-extracting data')
1485 except GeoRestrictedError
as e
:
1488 msg
+= '\nThis video is available in %s.' % ', '.join(
1489 map(ISO3166Utils
.short2full
, e
.countries
))
1490 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
1491 self
.report_error(msg
)
1492 except ExtractorError
as e
: # An error we somewhat expected
1493 self
.report_error(str(e
), e
.format_traceback())
1494 except Exception as e
:
1495 if self
.params
.get('ignoreerrors'):
1496 self
.report_error(str(e
), tb
=encode_compat_str(traceback
.format_exc()))
1502 def _wait_for_video(self
, ie_result
={}):
1503 if (not self
.params
.get('wait_for_video')
1504 or ie_result
.get('_type', 'video') != 'video'
1505 or ie_result
.get('formats') or ie_result
.get('url')):
1508 format_dur
= lambda dur
: '%02d:%02d:%02d' % timetuple_from_msec(dur
* 1000)[:-1]
1513 full_msg
= f
'{msg}\n'
1514 if not self
.params
.get('noprogress'):
1515 full_msg
= msg
+ ' ' * (len(last_msg
) - len(msg
)) + '\r'
1518 self
.to_screen(full_msg
, skip_eol
=True)
1521 min_wait
, max_wait
= self
.params
.get('wait_for_video')
1522 diff
= try_get(ie_result
, lambda x
: x
['release_timestamp'] - time
.time())
1523 if diff
is None and ie_result
.get('live_status') == 'is_upcoming':
1524 diff
= round(random
.uniform(min_wait
, max_wait
) if (max_wait
and min_wait
) else (max_wait
or min_wait
), 0)
1525 self
.report_warning('Release time of video is not known')
1526 elif ie_result
and (diff
or 0) <= 0:
1527 self
.report_warning('Video should already be available according to extracted info')
1528 diff
= min(max(diff
or 0, min_wait
or 0), max_wait
or float('inf'))
1529 self
.to_screen(f
'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now')
1531 wait_till
= time
.time() + diff
1534 diff
= wait_till
- time
.time()
1537 raise ReExtractInfo('[wait] Wait period ended', expected
=True)
1538 progress(f
'[wait] Remaining time until next attempt: {self._format_screen(format_dur(diff), self.Styles.EMPHASIS)}')
1540 except KeyboardInterrupt:
1542 raise ReExtractInfo('[wait] Interrupted by user', expected
=True)
1543 except BaseException
as e
:
1544 if not isinstance(e
, ReExtractInfo
):
1548 @_handle_extraction_exceptions
1549 def __extract_info(self
, url
, ie
, download
, extra_info
, process
):
1551 ie_result
= ie
.extract(url
)
1552 except UserNotLive
as e
:
1554 if self
.params
.get('wait_for_video'):
1555 self
.report_warning(e
)
1556 self
._wait
_for
_video
()
1558 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
1559 self
.report_warning(f
'Extractor {ie.IE_NAME} returned nothing{bug_reports_message()}')
1561 if isinstance(ie_result
, list):
1562 # Backwards compatibility: old IE result format
1564 '_type': 'compat_list',
1565 'entries': ie_result
,
1567 if extra_info
.get('original_url'):
1568 ie_result
.setdefault('original_url', extra_info
['original_url'])
1569 self
.add_default_extra_info(ie_result
, ie
, url
)
1571 self
._wait
_for
_video
(ie_result
)
1572 return self
.process_ie_result(ie_result
, download
, extra_info
)
1576 def add_default_extra_info(self
, ie_result
, ie
, url
):
1578 self
.add_extra_info(ie_result
, {
1580 'original_url': url
,
1582 webpage_url
= ie_result
.get('webpage_url')
1584 self
.add_extra_info(ie_result
, {
1585 'webpage_url_basename': url_basename(webpage_url
),
1586 'webpage_url_domain': get_domain(webpage_url
),
1589 self
.add_extra_info(ie_result
, {
1590 'extractor': ie
.IE_NAME
,
1591 'extractor_key': ie
.ie_key(),
1594 def process_ie_result(self
, ie_result
, download
=True, extra_info
=None):
1596 Take the result of the ie(may be modified) and resolve all unresolved
1597 references (URLs, playlist items).
1599 It will also download the videos if 'download'.
1600 Returns the resolved ie_result.
1602 if extra_info
is None:
1604 result_type
= ie_result
.get('_type', 'video')
1606 if result_type
in ('url', 'url_transparent'):
1607 ie_result
['url'] = sanitize_url(
1608 ie_result
['url'], scheme
='http' if self
.params
.get('prefer_insecure') else 'https')
1609 if ie_result
.get('original_url'):
1610 extra_info
.setdefault('original_url', ie_result
['original_url'])
1612 extract_flat
= self
.params
.get('extract_flat', False)
1613 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
1614 or extract_flat
is True):
1615 info_copy
= ie_result
.copy()
1616 ie
= try_get(ie_result
.get('ie_key'), self
.get_info_extractor
)
1617 if ie
and not ie_result
.get('id'):
1618 info_copy
['id'] = ie
.get_temp_id(ie_result
['url'])
1619 self
.add_default_extra_info(info_copy
, ie
, ie_result
['url'])
1620 self
.add_extra_info(info_copy
, extra_info
)
1621 info_copy
, _
= self
.pre_process(info_copy
)
1622 self
.__forced
_printings
(info_copy
, self
.prepare_filename(info_copy
), incomplete
=True)
1623 self
._raise
_pending
_errors
(info_copy
)
1624 if self
.params
.get('force_write_download_archive', False):
1625 self
.record_download_archive(info_copy
)
1628 if result_type
== 'video':
1629 self
.add_extra_info(ie_result
, extra_info
)
1630 ie_result
= self
.process_video_result(ie_result
, download
=download
)
1631 self
._raise
_pending
_errors
(ie_result
)
1632 additional_urls
= (ie_result
or {}).get('additional_urls')
1634 # TODO: Improve MetadataParserPP to allow setting a list
1635 if isinstance(additional_urls
, str):
1636 additional_urls
= [additional_urls
]
1638 '[info] %s: %d additional URL(s) requested' % (ie_result
['id'], len(additional_urls
)))
1639 self
.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls
))
1640 ie_result
['additional_entries'] = [
1642 url
, download
, extra_info
=extra_info
,
1643 force_generic_extractor
=self
.params
.get('force_generic_extractor'))
1644 for url
in additional_urls
1647 elif result_type
== 'url':
1648 # We have to add extra_info to the results because it may be
1649 # contained in a playlist
1650 return self
.extract_info(
1651 ie_result
['url'], download
,
1652 ie_key
=ie_result
.get('ie_key'),
1653 extra_info
=extra_info
)
1654 elif result_type
== 'url_transparent':
1655 # Use the information from the embedding page
1656 info
= self
.extract_info(
1657 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
1658 extra_info
=extra_info
, download
=False, process
=False)
1660 # extract_info may return None when ignoreerrors is enabled and
1661 # extraction failed with an error, don't crash and return early
1666 exempted_fields
= {'_type', 'url', 'ie_key'}
1667 if not ie_result
.get('section_end') and ie_result
.get('section_start') is None:
1668 # For video clips, the id etc of the clip extractor should be used
1669 exempted_fields |
= {'id', 'extractor', 'extractor_key'}
1671 new_result
= info
.copy()
1672 new_result
.update(filter_dict(ie_result
, lambda k
, v
: v
is not None and k
not in exempted_fields
))
1674 # Extracted info may not be a video result (i.e.
1675 # info.get('_type', 'video') != video) but rather an url or
1676 # url_transparent. In such cases outer metadata (from ie_result)
1677 # should be propagated to inner one (info). For this to happen
1678 # _type of info should be overridden with url_transparent. This
1679 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
1680 if new_result
.get('_type') == 'url':
1681 new_result
['_type'] = 'url_transparent'
1683 return self
.process_ie_result(
1684 new_result
, download
=download
, extra_info
=extra_info
)
1685 elif result_type
in ('playlist', 'multi_video'):
1686 # Protect from infinite recursion due to recursively nested playlists
1687 # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
1688 webpage_url
= ie_result
['webpage_url']
1689 if webpage_url
in self
._playlist
_urls
:
1691 '[download] Skipping already downloaded playlist: %s'
1692 % ie_result
.get('title') or ie_result
.get('id'))
1695 self
._playlist
_level
+= 1
1696 self
._playlist
_urls
.add(webpage_url
)
1697 self
._fill
_common
_fields
(ie_result
, False)
1698 self
._sanitize
_thumbnails
(ie_result
)
1700 return self
.__process
_playlist
(ie_result
, download
)
1702 self
._playlist
_level
-= 1
1703 if not self
._playlist
_level
:
1704 self
._playlist
_urls
.clear()
1705 elif result_type
== 'compat_list':
1706 self
.report_warning(
1707 'Extractor %s returned a compat_list result. '
1708 'It needs to be updated.' % ie_result
.get('extractor'))
1711 self
.add_extra_info(r
, {
1712 'extractor': ie_result
['extractor'],
1713 'webpage_url': ie_result
['webpage_url'],
1714 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1715 'webpage_url_domain': get_domain(ie_result
['webpage_url']),
1716 'extractor_key': ie_result
['extractor_key'],
1719 ie_result
['entries'] = [
1720 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1721 for r
in ie_result
['entries']
1725 raise Exception('Invalid result type: %s' % result_type
)
1727 def _ensure_dir_exists(self
, path
):
1728 return make_dir(path
, self
.report_error
)
1731 def _playlist_infodict(ie_result
, strict
=False, **kwargs
):
1733 'playlist_count': ie_result
.get('playlist_count'),
1734 'playlist': ie_result
.get('title') or ie_result
.get('id'),
1735 'playlist_id': ie_result
.get('id'),
1736 'playlist_title': ie_result
.get('title'),
1737 'playlist_uploader': ie_result
.get('uploader'),
1738 'playlist_uploader_id': ie_result
.get('uploader_id'),
1745 'playlist_index': 0,
1746 '__last_playlist_index': max(ie_result
['requested_entries'] or (0, 0)),
1747 'extractor': ie_result
['extractor'],
1748 'webpage_url': ie_result
['webpage_url'],
1749 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1750 'webpage_url_domain': get_domain(ie_result
['webpage_url']),
1751 'extractor_key': ie_result
['extractor_key'],
1754 def __process_playlist(self
, ie_result
, download
):
1755 """Process each entry in the playlist"""
1756 assert ie_result
['_type'] in ('playlist', 'multi_video')
1758 common_info
= self
._playlist
_infodict
(ie_result
, strict
=True)
1759 title
= common_info
.get('playlist') or '<Untitled>'
1760 if self
._match
_entry
(common_info
, incomplete
=True) is not None:
1762 self
.to_screen(f
'[download] Downloading {ie_result["_type"]}: {title}')
1764 all_entries
= PlaylistEntries(self
, ie_result
)
1765 entries
= orderedSet(all_entries
.get_requested_items(), lazy
=True)
1767 lazy
= self
.params
.get('lazy_playlist')
1769 resolved_entries
, n_entries
= [], 'N/A'
1770 ie_result
['requested_entries'], ie_result
['entries'] = None, None
1772 entries
= resolved_entries
= list(entries
)
1773 n_entries
= len(resolved_entries
)
1774 ie_result
['requested_entries'], ie_result
['entries'] = tuple(zip(*resolved_entries
)) or ([], [])
1775 if not ie_result
.get('playlist_count'):
1776 # Better to do this after potentially exhausting entries
1777 ie_result
['playlist_count'] = all_entries
.get_full_count()
1779 extra
= self
._playlist
_infodict
(ie_result
, n_entries
=int_or_none(n_entries
))
1780 ie_copy
= collections
.ChainMap(ie_result
, extra
)
1782 _infojson_written
= False
1783 write_playlist_files
= self
.params
.get('allow_playlist_files', True)
1784 if write_playlist_files
and self
.params
.get('list_thumbnails'):
1785 self
.list_thumbnails(ie_result
)
1786 if write_playlist_files
and not self
.params
.get('simulate'):
1787 _infojson_written
= self
._write
_info
_json
(
1788 'playlist', ie_result
, self
.prepare_filename(ie_copy
, 'pl_infojson'))
1789 if _infojson_written
is None:
1791 if self
._write
_description
('playlist', ie_result
,
1792 self
.prepare_filename(ie_copy
, 'pl_description')) is None:
1794 # TODO: This should be passed to ThumbnailsConvertor if necessary
1795 self
._write
_thumbnails
('playlist', ie_result
, self
.prepare_filename(ie_copy
, 'pl_thumbnail'))
1798 if self
.params
.get('playlistreverse') or self
.params
.get('playlistrandom'):
1799 self
.report_warning('playlistreverse and playlistrandom are not supported with lazy_playlist', only_once
=True)
1800 elif self
.params
.get('playlistreverse'):
1802 elif self
.params
.get('playlistrandom'):
1803 random
.shuffle(entries
)
1805 self
.to_screen(f
'[{ie_result["extractor"]}] Playlist {title}: Downloading {n_entries} videos'
1806 f
'{format_field(ie_result, "playlist_count", " of %s")}')
1808 keep_resolved_entries
= self
.params
.get('extract_flat') != 'discard'
1809 if self
.params
.get('extract_flat') == 'discard_in_playlist':
1810 keep_resolved_entries
= ie_result
['_type'] != 'playlist'
1811 if keep_resolved_entries
:
1812 self
.write_debug('The information of all playlist entries will be held in memory')
1815 max_failures
= self
.params
.get('skip_playlist_after_errors') or float('inf')
1816 for i
, (playlist_index
, entry
) in enumerate(entries
):
1818 resolved_entries
.append((playlist_index
, entry
))
1822 entry
['__x_forwarded_for_ip'] = ie_result
.get('__x_forwarded_for_ip')
1823 if not lazy
and 'playlist-index' in self
.params
.get('compat_opts', []):
1824 playlist_index
= ie_result
['requested_entries'][i
]
1826 entry_copy
= collections
.ChainMap(entry
, {
1828 'n_entries': int_or_none(n_entries
),
1829 'playlist_index': playlist_index
,
1830 'playlist_autonumber': i
+ 1,
1833 if self
._match
_entry
(entry_copy
, incomplete
=True) is not None:
1834 # For compatabilty with youtube-dl. See https://github.com/yt-dlp/yt-dlp/issues/4369
1835 resolved_entries
[i
] = (playlist_index
, NO_DEFAULT
)
1838 self
.to_screen('[download] Downloading video %s of %s' % (
1839 self
._format
_screen
(i
+ 1, self
.Styles
.ID
), self
._format
_screen
(n_entries
, self
.Styles
.EMPHASIS
)))
1842 'playlist_index': playlist_index
,
1843 'playlist_autonumber': i
+ 1,
1845 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, extra
)
1846 if not entry_result
:
1848 if failures
>= max_failures
:
1850 f
'Skipping the remaining entries in playlist "{title}" since {failures} items failed extraction')
1852 if keep_resolved_entries
:
1853 resolved_entries
[i
] = (playlist_index
, entry_result
)
1855 # Update with processed data
1856 ie_result
['requested_entries'] = [i
for i
, e
in resolved_entries
if e
is not NO_DEFAULT
]
1857 ie_result
['entries'] = [e
for _
, e
in resolved_entries
if e
is not NO_DEFAULT
]
1859 # Write the updated info to json
1860 if _infojson_written
is True and self
._write
_info
_json
(
1861 'updated playlist', ie_result
,
1862 self
.prepare_filename(ie_copy
, 'pl_infojson'), overwrite
=True) is None:
1865 ie_result
= self
.run_all_pps('playlist', ie_result
)
1866 self
.to_screen(f
'[download] Finished downloading playlist: {title}')
1869 @_handle_extraction_exceptions
1870 def __process_iterable_entry(self
, entry
, download
, extra_info
):
1871 return self
.process_ie_result(
1872 entry
, download
=download
, extra_info
=extra_info
)
1874 def _build_format_filter(self
, filter_spec
):
1875 " Returns a function to filter the formats according to the filter_spec "
1885 operator_rex
= re
.compile(r
'''(?x)\s*
1886 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)\s*
1887 (?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1888 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s*
1889 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
1890 m
= operator_rex
.fullmatch(filter_spec
)
1893 comparison_value
= int(m
.group('value'))
1895 comparison_value
= parse_filesize(m
.group('value'))
1896 if comparison_value
is None:
1897 comparison_value
= parse_filesize(m
.group('value') + 'B')
1898 if comparison_value
is None:
1900 'Invalid value %r in format specification %r' % (
1901 m
.group('value'), filter_spec
))
1902 op
= OPERATORS
[m
.group('op')]
1907 '^=': lambda attr
, value
: attr
.startswith(value
),
1908 '$=': lambda attr
, value
: attr
.endswith(value
),
1909 '*=': lambda attr
, value
: value
in attr
,
1910 '~=': lambda attr
, value
: value
.search(attr
) is not None
1912 str_operator_rex
= re
.compile(r
'''(?x)\s*
1913 (?P<key>[a-zA-Z0-9._-]+)\s*
1914 (?P<negation>!\s*)?(?P<op>%s)\s*(?P<none_inclusive>\?\s*)?
1916 (?P<value>(?(quote)(?:(?!(?P=quote))[^\\]|\\.)+|[\w.-]+))
1917 (?(quote)(?P=quote))\s*
1918 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
1919 m
= str_operator_rex
.fullmatch(filter_spec
)
1921 if m
.group('op') == '~=':
1922 comparison_value
= re
.compile(m
.group('value'))
1924 comparison_value
= re
.sub(r
'''\\([\\"'])''', r
'\1', m
.group('value'))
1925 str_op
= STR_OPERATORS
[m
.group('op')]
1926 if m
.group('negation'):
1927 op
= lambda attr
, value
: not str_op(attr
, value
)
1932 raise SyntaxError('Invalid filter specification %r' % filter_spec
)
1935 actual_value
= f
.get(m
.group('key'))
1936 if actual_value
is None:
1937 return m
.group('none_inclusive')
1938 return op(actual_value
, comparison_value
)
1941 def _check_formats(self
, formats
):
1943 self
.to_screen('[info] Testing format %s' % f
['format_id'])
1944 path
= self
.get_output_path('temp')
1945 if not self
._ensure
_dir
_exists
(f
'{path}/'):
1947 temp_file
= tempfile
.NamedTemporaryFile(suffix
='.tmp', delete
=False, dir=path
or None)
1950 success
, _
= self
.dl(temp_file
.name
, f
, test
=True)
1951 except (DownloadError
, OSError, ValueError) + network_exceptions
:
1954 if os
.path
.exists(temp_file
.name
):
1956 os
.remove(temp_file
.name
)
1958 self
.report_warning('Unable to delete temporary file "%s"' % temp_file
.name
)
1962 self
.to_screen('[info] Unable to download format %s. Skipping...' % f
['format_id'])
1964 def _default_format_spec(self
, info_dict
, download
=True):
1967 merger
= FFmpegMergerPP(self
)
1968 return merger
.available
and merger
.can_merge()
1971 not self
.params
.get('simulate')
1975 or info_dict
.get('is_live') and not self
.params
.get('live_from_start')
1976 or self
.params
['outtmpl']['default'] == '-'))
1979 or self
.params
.get('allow_multiple_audio_streams', False)
1980 or 'format-spec' in self
.params
['compat_opts'])
1983 'best/bestvideo+bestaudio' if prefer_best
1984 else 'bestvideo*+bestaudio/best' if not compat
1985 else 'bestvideo+bestaudio/best')
1987 def build_format_selector(self
, format_spec
):
1988 def syntax_error(note
, start
):
1990 'Invalid format specification: '
1991 '{}\n\t{}\n\t{}^'.format(note
, format_spec
, ' ' * start
[1]))
1992 return SyntaxError(message
)
1994 PICKFIRST
= 'PICKFIRST'
1998 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
2000 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
2001 'video': self
.params
.get('allow_multiple_video_streams', False)}
2003 check_formats
= self
.params
.get('check_formats') == 'selected'
2005 def _parse_filter(tokens
):
2007 for type, string
, start
, _
, _
in tokens
:
2008 if type == tokenize
.OP
and string
== ']':
2009 return ''.join(filter_parts
)
2011 filter_parts
.append(string
)
2013 def _remove_unused_ops(tokens
):
2014 # Remove operators that we don't use and join them with the surrounding strings.
2015 # E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
2016 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
2017 last_string
, last_start
, last_end
, last_line
= None, None, None, None
2018 for type, string
, start
, end
, line
in tokens
:
2019 if type == tokenize
.OP
and string
== '[':
2021 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
2023 yield type, string
, start
, end
, line
2024 # everything inside brackets will be handled by _parse_filter
2025 for type, string
, start
, end
, line
in tokens
:
2026 yield type, string
, start
, end
, line
2027 if type == tokenize
.OP
and string
== ']':
2029 elif type == tokenize
.OP
and string
in ALLOWED_OPS
:
2031 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
2033 yield type, string
, start
, end
, line
2034 elif type in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
2036 last_string
= string
2040 last_string
+= string
2042 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
2044 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
2046 current_selector
= None
2047 for type, string
, start
, _
, _
in tokens
:
2048 # ENCODING is only defined in python 3.x
2049 if type == getattr(tokenize
, 'ENCODING', None):
2051 elif type in [tokenize
.NAME
, tokenize
.NUMBER
]:
2052 current_selector
= FormatSelector(SINGLE
, string
, [])
2053 elif type == tokenize
.OP
:
2055 if not inside_group
:
2056 # ')' will be handled by the parentheses group
2057 tokens
.restore_last_token()
2059 elif inside_merge
and string
in ['/', ',']:
2060 tokens
.restore_last_token()
2062 elif inside_choice
and string
== ',':
2063 tokens
.restore_last_token()
2066 if not current_selector
:
2067 raise syntax_error('"," must follow a format selector', start
)
2068 selectors
.append(current_selector
)
2069 current_selector
= None
2071 if not current_selector
:
2072 raise syntax_error('"/" must follow a format selector', start
)
2073 first_choice
= current_selector
2074 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
2075 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
2077 if not current_selector
:
2078 current_selector
= FormatSelector(SINGLE
, 'best', [])
2079 format_filter
= _parse_filter(tokens
)
2080 current_selector
.filters
.append(format_filter
)
2082 if current_selector
:
2083 raise syntax_error('Unexpected "("', start
)
2084 group
= _parse_format_selection(tokens
, inside_group
=True)
2085 current_selector
= FormatSelector(GROUP
, group
, [])
2087 if not current_selector
:
2088 raise syntax_error('Unexpected "+"', start
)
2089 selector_1
= current_selector
2090 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
2092 raise syntax_error('Expected a selector', start
)
2093 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
2095 raise syntax_error(f
'Operator not recognized: "{string}"', start
)
2096 elif type == tokenize
.ENDMARKER
:
2098 if current_selector
:
2099 selectors
.append(current_selector
)
2102 def _merge(formats_pair
):
2103 format_1
, format_2
= formats_pair
2106 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
2107 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
2109 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
2110 get_no_more
= {'video': False, 'audio': False}
2111 for (i
, fmt_info
) in enumerate(formats_info
):
2112 if fmt_info
.get('acodec') == fmt_info
.get('vcodec') == 'none':
2115 for aud_vid
in ['audio', 'video']:
2116 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
2117 if get_no_more
[aud_vid
]:
2120 get_no_more
[aud_vid
] = True
2122 if len(formats_info
) == 1:
2123 return formats_info
[0]
2125 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
2126 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
2128 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
2129 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
2131 output_ext
= get_compatible_ext(
2132 vcodecs
=[f
.get('vcodec') for f
in video_fmts
],
2133 acodecs
=[f
.get('acodec') for f
in audio_fmts
],
2134 vexts
=[f
['ext'] for f
in video_fmts
],
2135 aexts
=[f
['ext'] for f
in audio_fmts
],
2136 preferences
=(try_call(lambda: self
.params
['merge_output_format'].split('/'))
2137 or self
.params
.get('prefer_free_formats') and ('webm', 'mkv')))
2139 filtered
= lambda *keys
: filter(None, (traverse_obj(fmt
, *keys
) for fmt
in formats_info
))
2142 'requested_formats': formats_info
,
2143 'format': '+'.join(filtered('format')),
2144 'format_id': '+'.join(filtered('format_id')),
2146 'protocol': '+'.join(map(determine_protocol
, formats_info
)),
2147 'language': '+'.join(orderedSet(filtered('language'))) or None,
2148 'format_note': '+'.join(orderedSet(filtered('format_note'))) or None,
2149 'filesize_approx': sum(filtered('filesize', 'filesize_approx')) or None,
2150 'tbr': sum(filtered('tbr', 'vbr', 'abr')),
2155 'width': the_only_video
.get('width'),
2156 'height': the_only_video
.get('height'),
2157 'resolution': the_only_video
.get('resolution') or self
.format_resolution(the_only_video
),
2158 'fps': the_only_video
.get('fps'),
2159 'dynamic_range': the_only_video
.get('dynamic_range'),
2160 'vcodec': the_only_video
.get('vcodec'),
2161 'vbr': the_only_video
.get('vbr'),
2162 'stretched_ratio': the_only_video
.get('stretched_ratio'),
2167 'acodec': the_only_audio
.get('acodec'),
2168 'abr': the_only_audio
.get('abr'),
2169 'asr': the_only_audio
.get('asr'),
2170 'audio_channels': the_only_audio
.get('audio_channels')
2175 def _check_formats(formats
):
2176 if not check_formats
:
2179 yield from self
._check
_formats
(formats
)
2181 def _build_selector_function(selector
):
2182 if isinstance(selector
, list): # ,
2183 fs
= [_build_selector_function(s
) for s
in selector
]
2185 def selector_function(ctx
):
2188 return selector_function
2190 elif selector
.type == GROUP
: # ()
2191 selector_function
= _build_selector_function(selector
.selector
)
2193 elif selector
.type == PICKFIRST
: # /
2194 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
2196 def selector_function(ctx
):
2198 picked_formats
= list(f(ctx
))
2200 return picked_formats
2203 elif selector
.type == MERGE
: # +
2204 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
2206 def selector_function(ctx
):
2207 for pair
in itertools
.product(selector_1(ctx
), selector_2(ctx
)):
2210 elif selector
.type == SINGLE
: # atom
2211 format_spec
= selector
.selector
or 'best'
2213 # TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
2214 if format_spec
== 'all':
2215 def selector_function(ctx
):
2216 yield from _check_formats(ctx
['formats'][::-1])
2217 elif format_spec
== 'mergeall':
2218 def selector_function(ctx
):
2219 formats
= list(_check_formats(
2220 f
for f
in ctx
['formats'] if f
.get('vcodec') != 'none' or f
.get('acodec') != 'none'))
2223 merged_format
= formats
[-1]
2224 for f
in formats
[-2::-1]:
2225 merged_format
= _merge((merged_format
, f
))
2229 format_fallback
, seperate_fallback
, format_reverse
, format_idx
= False, None, True, 1
2231 r
'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
2233 if mobj
is not None:
2234 format_idx
= int_or_none(mobj
.group('n'), default
=1)
2235 format_reverse
= mobj
.group('bw')[0] == 'b'
2236 format_type
= (mobj
.group('type') or [None])[0]
2237 not_format_type
= {'v': 'a', 'a': 'v'}
.get(format_type
)
2238 format_modified
= mobj
.group('mod') is not None
2240 format_fallback
= not format_type
and not format_modified
# for b, w
2242 (lambda f
: f
.get('%scodec' % format_type
) != 'none')
2243 if format_type
and format_modified
# bv*, ba*, wv*, wa*
2244 else (lambda f
: f
.get('%scodec' % not_format_type
) == 'none')
2245 if format_type
# bv, ba, wv, wa
2246 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
2247 if not format_modified
# b, w
2248 else lambda f
: True) # b*, w*
2249 filter_f
= lambda f
: _filter_f(f
) and (
2250 f
.get('vcodec') != 'none' or f
.get('acodec') != 'none')
2252 if format_spec
in self
._format
_selection
_exts
['audio']:
2253 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none'
2254 elif format_spec
in self
._format
_selection
_exts
['video']:
2255 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none' and f
.get('vcodec') != 'none'
2256 seperate_fallback
= lambda f
: f
.get('ext') == format_spec
and f
.get('vcodec') != 'none'
2257 elif format_spec
in self
._format
_selection
_exts
['storyboards']:
2258 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') == 'none' and f
.get('vcodec') == 'none'
2260 filter_f
= lambda f
: f
.get('format_id') == format_spec
# id
2262 def selector_function(ctx
):
2263 formats
= list(ctx
['formats'])
2264 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
2266 if format_fallback
and ctx
['incomplete_formats']:
2267 # for extractors with incomplete formats (audio only (soundcloud)
2268 # or video only (imgur)) best/worst will fallback to
2269 # best/worst {video,audio}-only format
2271 elif seperate_fallback
and not ctx
['has_merged_format']:
2272 # for compatibility with youtube-dl when there is no pre-merged format
2273 matches
= list(filter(seperate_fallback
, formats
))
2274 matches
= LazyList(_check_formats(matches
[::-1 if format_reverse
else 1]))
2276 yield matches
[format_idx
- 1]
2277 except LazyList
.IndexError:
2280 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
2282 def final_selector(ctx
):
2283 ctx_copy
= dict(ctx
)
2284 for _filter
in filters
:
2285 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
2286 return selector_function(ctx_copy
)
2287 return final_selector
2289 stream
= io
.BytesIO(format_spec
.encode())
2291 tokens
= list(_remove_unused_ops(tokenize
.tokenize(stream
.readline
)))
2292 except tokenize
.TokenError
:
2293 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
2295 class TokenIterator
:
2296 def __init__(self
, tokens
):
2297 self
.tokens
= tokens
2304 if self
.counter
>= len(self
.tokens
):
2305 raise StopIteration()
2306 value
= self
.tokens
[self
.counter
]
2312 def restore_last_token(self
):
2315 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
2316 return _build_selector_function(parsed_selector
)
2318 def _calc_headers(self
, info_dict
):
2319 res
= merge_headers(self
.params
['http_headers'], info_dict
.get('http_headers') or {})
2321 cookies
= self
._calc
_cookies
(info_dict
['url'])
2323 res
['Cookie'] = cookies
2325 if 'X-Forwarded-For' not in res
:
2326 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
2327 if x_forwarded_for_ip
:
2328 res
['X-Forwarded-For'] = x_forwarded_for_ip
2332 def _calc_cookies(self
, url
):
2333 pr
= sanitized_Request(url
)
2334 self
.cookiejar
.add_cookie_header(pr
)
2335 return pr
.get_header('Cookie')
2337 def _sort_thumbnails(self
, thumbnails
):
2338 thumbnails
.sort(key
=lambda t
: (
2339 t
.get('preference') if t
.get('preference') is not None else -1,
2340 t
.get('width') if t
.get('width') is not None else -1,
2341 t
.get('height') if t
.get('height') is not None else -1,
2342 t
.get('id') if t
.get('id') is not None else '',
2345 def _sanitize_thumbnails(self
, info_dict
):
2346 thumbnails
= info_dict
.get('thumbnails')
2347 if thumbnails
is None:
2348 thumbnail
= info_dict
.get('thumbnail')
2350 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
2354 def check_thumbnails(thumbnails
):
2355 for t
in thumbnails
:
2356 self
.to_screen(f
'[info] Testing thumbnail {t["id"]}')
2358 self
.urlopen(HEADRequest(t
['url']))
2359 except network_exceptions
as err
:
2360 self
.to_screen(f
'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...')
2364 self
._sort
_thumbnails
(thumbnails
)
2365 for i
, t
in enumerate(thumbnails
):
2366 if t
.get('id') is None:
2368 if t
.get('width') and t
.get('height'):
2369 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
2370 t
['url'] = sanitize_url(t
['url'])
2372 if self
.params
.get('check_formats') is True:
2373 info_dict
['thumbnails'] = LazyList(check_thumbnails(thumbnails
[::-1]), reverse
=True)
2375 info_dict
['thumbnails'] = thumbnails
2377 def _fill_common_fields(self
, info_dict
, is_video
=True):
2378 # TODO: move sanitization here
2380 # playlists are allowed to lack "title"
2381 title
= info_dict
.get('title', NO_DEFAULT
)
2382 if title
is NO_DEFAULT
:
2383 raise ExtractorError('Missing "title" field in extractor result',
2384 video_id
=info_dict
['id'], ie
=info_dict
['extractor'])
2385 info_dict
['fulltitle'] = title
2388 self
.write_debug('Extractor gave empty title. Creating a generic title')
2390 self
.report_warning('Extractor failed to obtain "title". Creating a generic title instead')
2391 info_dict
['title'] = f
'{info_dict["extractor"].replace(":", "-")} video #{info_dict["id"]}'
2393 if info_dict
.get('duration') is not None:
2394 info_dict
['duration_string'] = formatSeconds(info_dict
['duration'])
2396 for ts_key
, date_key
in (
2397 ('timestamp', 'upload_date'),
2398 ('release_timestamp', 'release_date'),
2399 ('modified_timestamp', 'modified_date'),
2401 if info_dict
.get(date_key
) is None and info_dict
.get(ts_key
) is not None:
2402 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
2403 # see http://bugs.python.org/issue1646728)
2404 with contextlib
.suppress(ValueError, OverflowError, OSError):
2405 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
[ts_key
])
2406 info_dict
[date_key
] = upload_date
.strftime('%Y%m%d')
2408 live_keys
= ('is_live', 'was_live')
2409 live_status
= info_dict
.get('live_status')
2410 if live_status
is None:
2411 for key
in live_keys
:
2412 if info_dict
.get(key
) is False:
2414 if info_dict
.get(key
):
2417 if all(info_dict
.get(key
) is False for key
in live_keys
):
2418 live_status
= 'not_live'
2420 info_dict
['live_status'] = live_status
2421 for key
in live_keys
:
2422 if info_dict
.get(key
) is None:
2423 info_dict
[key
] = (live_status
== key
)
2425 # Auto generate title fields corresponding to the *_number fields when missing
2426 # in order to always have clean titles. This is very common for TV series.
2427 for field
in ('chapter', 'season', 'episode'):
2428 if info_dict
.get('%s_number' % field
) is not None and not info_dict
.get(field
):
2429 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
['%s_number' % field
])
2431 def _raise_pending_errors(self
, info
):
2432 err
= info
.pop('__pending_error', None)
2434 self
.report_error(err
, tb
=False)
2436 def process_video_result(self
, info_dict
, download
=True):
2437 assert info_dict
.get('_type', 'video') == 'video'
2438 self
._num
_videos
+= 1
2440 if 'id' not in info_dict
:
2441 raise ExtractorError('Missing "id" field in extractor result', ie
=info_dict
['extractor'])
2442 elif not info_dict
.get('id'):
2443 raise ExtractorError('Extractor failed to obtain "id"', ie
=info_dict
['extractor'])
2445 def report_force_conversion(field
, field_not
, conversion
):
2446 self
.report_warning(
2447 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
2448 % (field
, field_not
, conversion
))
2450 def sanitize_string_field(info
, string_field
):
2451 field
= info
.get(string_field
)
2452 if field
is None or isinstance(field
, str):
2454 report_force_conversion(string_field
, 'a string', 'string')
2455 info
[string_field
] = str(field
)
2457 def sanitize_numeric_fields(info
):
2458 for numeric_field
in self
._NUMERIC
_FIELDS
:
2459 field
= info
.get(numeric_field
)
2460 if field
is None or isinstance(field
, (int, float)):
2462 report_force_conversion(numeric_field
, 'numeric', 'int')
2463 info
[numeric_field
] = int_or_none(field
)
2465 sanitize_string_field(info_dict
, 'id')
2466 sanitize_numeric_fields(info_dict
)
2467 if info_dict
.get('section_end') and info_dict
.get('section_start') is not None:
2468 info_dict
['duration'] = round(info_dict
['section_end'] - info_dict
['section_start'], 3)
2469 if (info_dict
.get('duration') or 0) <= 0 and info_dict
.pop('duration', None):
2470 self
.report_warning('"duration" field is negative, there is an error in extractor')
2472 chapters
= info_dict
.get('chapters') or []
2473 if chapters
and chapters
[0].get('start_time'):
2474 chapters
.insert(0, {'start_time': 0}
)
2476 dummy_chapter
= {'end_time': 0, 'start_time': info_dict.get('duration')}
2477 for idx
, (prev
, current
, next_
) in enumerate(zip(
2478 (dummy_chapter
, *chapters
), chapters
, (*chapters
[1:], dummy_chapter
)), 1):
2479 if current
.get('start_time') is None:
2480 current
['start_time'] = prev
.get('end_time')
2481 if not current
.get('end_time'):
2482 current
['end_time'] = next_
.get('start_time')
2483 if not current
.get('title'):
2484 current
['title'] = f
'<Untitled Chapter {idx}>'
2486 if 'playlist' not in info_dict
:
2487 # It isn't part of a playlist
2488 info_dict
['playlist'] = None
2489 info_dict
['playlist_index'] = None
2491 self
._sanitize
_thumbnails
(info_dict
)
2493 thumbnail
= info_dict
.get('thumbnail')
2494 thumbnails
= info_dict
.get('thumbnails')
2496 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
2498 info_dict
['thumbnail'] = thumbnails
[-1]['url']
2500 if info_dict
.get('display_id') is None and 'id' in info_dict
:
2501 info_dict
['display_id'] = info_dict
['id']
2503 self
._fill
_common
_fields
(info_dict
)
2505 for cc_kind
in ('subtitles', 'automatic_captions'):
2506 cc
= info_dict
.get(cc_kind
)
2508 for _
, subtitle
in cc
.items():
2509 for subtitle_format
in subtitle
:
2510 if subtitle_format
.get('url'):
2511 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
2512 if subtitle_format
.get('ext') is None:
2513 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
2515 automatic_captions
= info_dict
.get('automatic_captions')
2516 subtitles
= info_dict
.get('subtitles')
2518 info_dict
['requested_subtitles'] = self
.process_subtitles(
2519 info_dict
['id'], subtitles
, automatic_captions
)
2521 if info_dict
.get('formats') is None:
2522 # There's only one format available
2523 formats
= [info_dict
]
2525 formats
= info_dict
['formats']
2527 # or None ensures --clean-infojson removes it
2528 info_dict
['_has_drm'] = any(f
.get('has_drm') for f
in formats
) or None
2529 if not self
.params
.get('allow_unplayable_formats'):
2530 formats
= [f
for f
in formats
if not f
.get('has_drm')]
2532 if formats
and all(f
.get('acodec') == f
.get('vcodec') == 'none' for f
in formats
):
2533 self
.report_warning(
2534 f
'{"This video is DRM protected and " if info_dict["_has_drm"] else ""}'
2535 'only images are available for download. Use --list-formats to see them'.capitalize())
2537 get_from_start
= not info_dict
.get('is_live') or bool(self
.params
.get('live_from_start'))
2538 if not get_from_start
:
2539 info_dict
['title'] += ' ' + datetime
.datetime
.now().strftime('%Y-%m-%d %H:%M')
2540 if info_dict
.get('is_live') and formats
:
2541 formats
= [f
for f
in formats
if bool(f
.get('is_from_start')) == get_from_start
]
2542 if get_from_start
and not formats
:
2543 self
.raise_no_formats(info_dict
, msg
=(
2544 '--live-from-start is passed, but there are no formats that can be downloaded from the start. '
2545 'If you want to download from the current time, use --no-live-from-start'))
2547 def is_wellformed(f
):
2550 self
.report_warning(
2551 '"url" field is missing or empty - skipping format, '
2552 'there is an error in extractor')
2554 if isinstance(url
, bytes):
2555 sanitize_string_field(f
, 'url')
2558 # Filter out malformed formats for better extraction robustness
2559 formats
= list(filter(is_wellformed
, formats
or []))
2562 self
.raise_no_formats(info_dict
)
2566 # We check that all the formats have the format and format_id fields
2567 for i
, format
in enumerate(formats
):
2568 sanitize_string_field(format
, 'format_id')
2569 sanitize_numeric_fields(format
)
2570 format
['url'] = sanitize_url(format
['url'])
2571 if not format
.get('format_id'):
2572 format
['format_id'] = str(i
)
2574 # Sanitize format_id from characters used in format selector expression
2575 format
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', format
['format_id'])
2576 format_id
= format
['format_id']
2577 if format_id
not in formats_dict
:
2578 formats_dict
[format_id
] = []
2579 formats_dict
[format_id
].append(format
)
2581 # Make sure all formats have unique format_id
2582 common_exts
= set(itertools
.chain(*self
._format
_selection
_exts
.values()))
2583 for format_id
, ambiguous_formats
in formats_dict
.items():
2584 ambigious_id
= len(ambiguous_formats
) > 1
2585 for i
, format
in enumerate(ambiguous_formats
):
2587 format
['format_id'] = '%s-%d' % (format_id
, i
)
2588 if format
.get('ext') is None:
2589 format
['ext'] = determine_ext(format
['url']).lower()
2590 # Ensure there is no conflict between id and ext in format selection
2591 # See https://github.com/yt-dlp/yt-dlp/issues/1282
2592 if format
['format_id'] != format
['ext'] and format
['format_id'] in common_exts
:
2593 format
['format_id'] = 'f%s' % format
['format_id']
2595 for i
, format
in enumerate(formats
):
2596 if format
.get('format') is None:
2597 format
['format'] = '{id} - {res}{note}'.format(
2598 id=format
['format_id'],
2599 res
=self
.format_resolution(format
),
2600 note
=format_field(format
, 'format_note', ' (%s)'),
2602 if format
.get('protocol') is None:
2603 format
['protocol'] = determine_protocol(format
)
2604 if format
.get('resolution') is None:
2605 format
['resolution'] = self
.format_resolution(format
, default
=None)
2606 if format
.get('dynamic_range') is None and format
.get('vcodec') != 'none':
2607 format
['dynamic_range'] = 'SDR'
2608 if (info_dict
.get('duration') and format
.get('tbr')
2609 and not format
.get('filesize') and not format
.get('filesize_approx')):
2610 format
['filesize_approx'] = int(info_dict
['duration'] * format
['tbr'] * (1024 / 8))
2612 # Add HTTP headers, so that external programs can use them from the
2614 full_format_info
= info_dict
.copy()
2615 full_format_info
.update(format
)
2616 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
2617 # Remove private housekeeping stuff
2618 if '__x_forwarded_for_ip' in info_dict
:
2619 del info_dict
['__x_forwarded_for_ip']
2621 if self
.params
.get('check_formats') is True:
2622 formats
= LazyList(self
._check
_formats
(formats
[::-1]), reverse
=True)
2624 if not formats
or formats
[0] is not info_dict
:
2625 # only set the 'formats' fields if the original info_dict list them
2626 # otherwise we end up with a circular reference, the first (and unique)
2627 # element in the 'formats' field in info_dict is info_dict itself,
2628 # which can't be exported to json
2629 info_dict
['formats'] = formats
2631 info_dict
, _
= self
.pre_process(info_dict
)
2633 if self
._match
_entry
(info_dict
, incomplete
=self
._format
_fields
) is not None:
2636 self
.post_extract(info_dict
)
2637 info_dict
, _
= self
.pre_process(info_dict
, 'after_filter')
2639 # The pre-processors may have modified the formats
2640 formats
= info_dict
.get('formats', [info_dict
])
2642 list_only
= self
.params
.get('simulate') is None and (
2643 self
.params
.get('list_thumbnails') or self
.params
.get('listformats') or self
.params
.get('listsubtitles'))
2644 interactive_format_selection
= not list_only
and self
.format_selector
== '-'
2645 if self
.params
.get('list_thumbnails'):
2646 self
.list_thumbnails(info_dict
)
2647 if self
.params
.get('listsubtitles'):
2648 if 'automatic_captions' in info_dict
:
2649 self
.list_subtitles(
2650 info_dict
['id'], automatic_captions
, 'automatic captions')
2651 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
2652 if self
.params
.get('listformats') or interactive_format_selection
:
2653 self
.list_formats(info_dict
)
2655 # Without this printing, -F --print-json will not work
2656 self
.__forced
_printings
(info_dict
, self
.prepare_filename(info_dict
), incomplete
=True)
2659 format_selector
= self
.format_selector
2660 if format_selector
is None:
2661 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
2662 self
.write_debug('Default format spec: %s' % req_format
)
2663 format_selector
= self
.build_format_selector(req_format
)
2666 if interactive_format_selection
:
2668 self
._format
_screen
('\nEnter format selector: ', self
.Styles
.EMPHASIS
))
2670 format_selector
= self
.build_format_selector(req_format
)
2671 except SyntaxError as err
:
2672 self
.report_error(err
, tb
=False, is_error
=False)
2675 formats_to_download
= list(format_selector({
2677 'has_merged_format': any('none' not in (f
.get('acodec'), f
.get('vcodec')) for f
in formats
),
2678 'incomplete_formats': (
2679 # All formats are video-only or
2680 all(f
.get('vcodec') != 'none' and f
.get('acodec') == 'none' for f
in formats
)
2681 # all formats are audio-only
2682 or all(f
.get('vcodec') == 'none' and f
.get('acodec') != 'none' for f
in formats
)),
2684 if interactive_format_selection
and not formats_to_download
:
2685 self
.report_error('Requested format is not available', tb
=False, is_error
=False)
2689 if not formats_to_download
:
2690 if not self
.params
.get('ignore_no_formats_error'):
2691 raise ExtractorError(
2692 'Requested format is not available. Use --list-formats for a list of available formats',
2693 expected
=True, video_id
=info_dict
['id'], ie
=info_dict
['extractor'])
2694 self
.report_warning('Requested format is not available')
2695 # Process what we can, even without any available formats.
2696 formats_to_download
= [{}]
2698 requested_ranges
= self
.params
.get('download_ranges')
2699 if requested_ranges
:
2700 requested_ranges
= tuple(requested_ranges(info_dict
, self
))
2702 best_format
, downloaded_formats
= formats_to_download
[-1], []
2705 def to_screen(*msg
):
2706 self
.to_screen(f
'[info] {info_dict["id"]}: {" ".join(", ".join(variadic(m)) for m in msg)}')
2708 to_screen(f
'Downloading {len(formats_to_download)} format(s):',
2709 (f
['format_id'] for f
in formats_to_download
))
2710 if requested_ranges
:
2711 to_screen(f
'Downloading {len(requested_ranges)} time ranges:',
2712 (f
'{int(c["start_time"])}-{int(c["end_time"])}' for c
in requested_ranges
))
2713 max_downloads_reached
= False
2715 for fmt
, chapter
in itertools
.product(formats_to_download
, requested_ranges
or [{}]):
2716 new_info
= self
._copy
_infodict
(info_dict
)
2717 new_info
.update(fmt
)
2718 offset
, duration
= info_dict
.get('section_start') or 0, info_dict
.get('duration') or float('inf')
2719 if chapter
or offset
:
2721 'section_start': offset
+ chapter
.get('start_time', 0),
2722 'section_end': offset
+ min(chapter
.get('end_time', duration
), duration
),
2723 'section_title': chapter
.get('title'),
2724 'section_number': chapter
.get('index'),
2726 downloaded_formats
.append(new_info
)
2728 self
.process_info(new_info
)
2729 except MaxDownloadsReached
:
2730 max_downloads_reached
= True
2731 self
._raise
_pending
_errors
(new_info
)
2732 # Remove copied info
2733 for key
, val
in tuple(new_info
.items()):
2734 if info_dict
.get(key
) == val
:
2736 if max_downloads_reached
:
2739 write_archive
= {f.get('__write_download_archive', False) for f in downloaded_formats}
2740 assert write_archive
.issubset({True, False, 'ignore'}
)
2741 if True in write_archive
and False not in write_archive
:
2742 self
.record_download_archive(info_dict
)
2744 info_dict
['requested_downloads'] = downloaded_formats
2745 info_dict
= self
.run_all_pps('after_video', info_dict
)
2746 if max_downloads_reached
:
2747 raise MaxDownloadsReached()
2749 # We update the info dict with the selected best quality format (backwards compatibility)
2750 info_dict
.update(best_format
)
2753 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
2754 """Select the requested subtitles and their format"""
2755 available_subs
, normal_sub_langs
= {}, []
2756 if normal_subtitles
and self
.params
.get('writesubtitles'):
2757 available_subs
.update(normal_subtitles
)
2758 normal_sub_langs
= tuple(normal_subtitles
.keys())
2759 if automatic_captions
and self
.params
.get('writeautomaticsub'):
2760 for lang
, cap_info
in automatic_captions
.items():
2761 if lang
not in available_subs
:
2762 available_subs
[lang
] = cap_info
2764 if not available_subs
or (
2765 not self
.params
.get('writesubtitles')
2766 and not self
.params
.get('writeautomaticsub')):
2769 all_sub_langs
= tuple(available_subs
.keys())
2770 if self
.params
.get('allsubtitles', False):
2771 requested_langs
= all_sub_langs
2772 elif self
.params
.get('subtitleslangs', False):
2774 requested_langs
= orderedSet_from_options(
2775 self
.params
.get('subtitleslangs'), {'all': all_sub_langs}
, use_regex
=True)
2776 except re
.error
as e
:
2777 raise ValueError(f
'Wrong regex for subtitlelangs: {e.pattern}')
2778 elif normal_sub_langs
:
2779 requested_langs
= ['en'] if 'en' in normal_sub_langs
else normal_sub_langs
[:1]
2781 requested_langs
= ['en'] if 'en' in all_sub_langs
else all_sub_langs
[:1]
2783 self
.to_screen(f
'[info] {video_id}: Downloading subtitles: {", ".join(requested_langs)}')
2785 formats_query
= self
.params
.get('subtitlesformat', 'best')
2786 formats_preference
= formats_query
.split('/') if formats_query
else []
2788 for lang
in requested_langs
:
2789 formats
= available_subs
.get(lang
)
2791 self
.report_warning(f
'{lang} subtitles not available for {video_id}')
2793 for ext
in formats_preference
:
2797 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
2803 self
.report_warning(
2804 'No subtitle format found matching "%s" for language %s, '
2805 'using %s' % (formats_query
, lang
, f
['ext']))
2809 def _forceprint(self
, key
, info_dict
):
2810 if info_dict
is None:
2812 info_copy
= info_dict
.copy()
2813 info_copy
['formats_table'] = self
.render_formats_table(info_dict
)
2814 info_copy
['thumbnails_table'] = self
.render_thumbnails_table(info_dict
)
2815 info_copy
['subtitles_table'] = self
.render_subtitles_table(info_dict
.get('id'), info_dict
.get('subtitles'))
2816 info_copy
['automatic_captions_table'] = self
.render_subtitles_table(info_dict
.get('id'), info_dict
.get('automatic_captions'))
2818 def format_tmpl(tmpl
):
2819 mobj
= re
.fullmatch(r
'([\w.:,]|-\d|(?P<dict>{([\w.:,]|-\d)+}))+=?', tmpl
)
2824 if tmpl
.startswith('{'):
2826 if tmpl
.endswith('='):
2827 tmpl
, fmt
= tmpl
[:-1], '{0} = %({0})#j'
2828 return '\n'.join(map(fmt
.format
, [tmpl
] if mobj
.group('dict') else tmpl
.split(',')))
2830 for tmpl
in self
.params
['forceprint'].get(key
, []):
2831 self
.to_stdout(self
.evaluate_outtmpl(format_tmpl(tmpl
), info_copy
))
2833 for tmpl
, file_tmpl
in self
.params
['print_to_file'].get(key
, []):
2834 filename
= self
.prepare_filename(info_dict
, outtmpl
=file_tmpl
)
2835 tmpl
= format_tmpl(tmpl
)
2836 self
.to_screen(f
'[info] Writing {tmpl!r} to: {filename}')
2837 if self
._ensure
_dir
_exists
(filename
):
2838 with open(filename
, 'a', encoding
='utf-8') as f
:
2839 f
.write(self
.evaluate_outtmpl(tmpl
, info_copy
) + '\n')
2841 def __forced_printings(self
, info_dict
, filename
, incomplete
):
2842 def print_mandatory(field
, actual_field
=None):
2843 if actual_field
is None:
2844 actual_field
= field
2845 if (self
.params
.get('force%s' % field
, False)
2846 and (not incomplete
or info_dict
.get(actual_field
) is not None)):
2847 self
.to_stdout(info_dict
[actual_field
])
2849 def print_optional(field
):
2850 if (self
.params
.get('force%s' % field
, False)
2851 and info_dict
.get(field
) is not None):
2852 self
.to_stdout(info_dict
[field
])
2854 info_dict
= info_dict
.copy()
2855 if filename
is not None:
2856 info_dict
['filename'] = filename
2857 if info_dict
.get('requested_formats') is not None:
2858 # For RTMP URLs, also include the playpath
2859 info_dict
['urls'] = '\n'.join(f
['url'] + f
.get('play_path', '') for f
in info_dict
['requested_formats'])
2860 elif info_dict
.get('url'):
2861 info_dict
['urls'] = info_dict
['url'] + info_dict
.get('play_path', '')
2863 if (self
.params
.get('forcejson')
2864 or self
.params
['forceprint'].get('video')
2865 or self
.params
['print_to_file'].get('video')):
2866 self
.post_extract(info_dict
)
2867 self
._forceprint
('video', info_dict
)
2869 print_mandatory('title')
2870 print_mandatory('id')
2871 print_mandatory('url', 'urls')
2872 print_optional('thumbnail')
2873 print_optional('description')
2874 print_optional('filename')
2875 if self
.params
.get('forceduration') and info_dict
.get('duration') is not None:
2876 self
.to_stdout(formatSeconds(info_dict
['duration']))
2877 print_mandatory('format')
2879 if self
.params
.get('forcejson'):
2880 self
.to_stdout(json
.dumps(self
.sanitize_info(info_dict
)))
2882 def dl(self
, name
, info
, subtitle
=False, test
=False):
2883 if not info
.get('url'):
2884 self
.raise_no_formats(info
, True)
2887 verbose
= self
.params
.get('verbose')
2890 'quiet': self
.params
.get('quiet') or not verbose
,
2892 'noprogress': not verbose
,
2894 'skip_unavailable_fragments': False,
2895 'keep_fragments': False,
2897 '_no_ytdl_file': True,
2900 params
= self
.params
2901 fd
= get_suitable_downloader(info
, params
, to_stdout
=(name
== '-'))(self
, params
)
2903 for ph
in self
._progress
_hooks
:
2904 fd
.add_progress_hook(ph
)
2906 (f
['url'].split(',')[0] + ',<data>' if f
['url'].startswith('data:') else f
['url'])
2907 for f
in info
.get('requested_formats', []) or [info
])
2908 self
.write_debug(f
'Invoking {fd.FD_NAME} downloader on "{urls}"')
2910 # Note: Ideally info should be a deep-copied so that hooks cannot modify it.
2911 # But it may contain objects that are not deep-copyable
2912 new_info
= self
._copy
_infodict
(info
)
2913 if new_info
.get('http_headers') is None:
2914 new_info
['http_headers'] = self
._calc
_headers
(new_info
)
2915 return fd
.download(name
, new_info
, subtitle
)
2917 def existing_file(self
, filepaths
, *, default_overwrite
=True):
2918 existing_files
= list(filter(os
.path
.exists
, orderedSet(filepaths
)))
2919 if existing_files
and not self
.params
.get('overwrites', default_overwrite
):
2920 return existing_files
[0]
2922 for file in existing_files
:
2923 self
.report_file_delete(file)
2927 def process_info(self
, info_dict
):
2928 """Process a single resolved IE result. (Modifies it in-place)"""
2930 assert info_dict
.get('_type', 'video') == 'video'
2931 original_infodict
= info_dict
2933 if 'format' not in info_dict
and 'ext' in info_dict
:
2934 info_dict
['format'] = info_dict
['ext']
2936 # This is mostly just for backward compatibility of process_info
2937 # As a side-effect, this allows for format-specific filters
2938 if self
._match
_entry
(info_dict
) is not None:
2939 info_dict
['__write_download_archive'] = 'ignore'
2942 # Does nothing under normal operation - for backward compatibility of process_info
2943 self
.post_extract(info_dict
)
2944 self
._num
_downloads
+= 1
2946 # info_dict['_filename'] needs to be set for backward compatibility
2947 info_dict
['_filename'] = full_filename
= self
.prepare_filename(info_dict
, warn
=True)
2948 temp_filename
= self
.prepare_filename(info_dict
, 'temp')
2952 self
.__forced
_printings
(info_dict
, full_filename
, incomplete
=('format' not in info_dict
))
2954 def check_max_downloads():
2955 if self
._num
_downloads
>= float(self
.params
.get('max_downloads') or 'inf'):
2956 raise MaxDownloadsReached()
2958 if self
.params
.get('simulate'):
2959 info_dict
['__write_download_archive'] = self
.params
.get('force_write_download_archive')
2960 check_max_downloads()
2963 if full_filename
is None:
2965 if not self
._ensure
_dir
_exists
(encodeFilename(full_filename
)):
2967 if not self
._ensure
_dir
_exists
(encodeFilename(temp_filename
)):
2970 if self
._write
_description
('video', info_dict
,
2971 self
.prepare_filename(info_dict
, 'description')) is None:
2974 sub_files
= self
._write
_subtitles
(info_dict
, temp_filename
)
2975 if sub_files
is None:
2977 files_to_move
.update(dict(sub_files
))
2979 thumb_files
= self
._write
_thumbnails
(
2980 'video', info_dict
, temp_filename
, self
.prepare_filename(info_dict
, 'thumbnail'))
2981 if thumb_files
is None:
2983 files_to_move
.update(dict(thumb_files
))
2985 infofn
= self
.prepare_filename(info_dict
, 'infojson')
2986 _infojson_written
= self
._write
_info
_json
('video', info_dict
, infofn
)
2987 if _infojson_written
:
2988 info_dict
['infojson_filename'] = infofn
2989 # For backward compatibility, even though it was a private field
2990 info_dict
['__infojson_filename'] = infofn
2991 elif _infojson_written
is None:
2994 # Note: Annotations are deprecated
2996 if self
.params
.get('writeannotations', False):
2997 annofn
= self
.prepare_filename(info_dict
, 'annotation')
2999 if not self
._ensure
_dir
_exists
(encodeFilename(annofn
)):
3001 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(annofn
)):
3002 self
.to_screen('[info] Video annotations are already present')
3003 elif not info_dict
.get('annotations'):
3004 self
.report_warning('There are no annotations to write.')
3007 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
3008 with open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
3009 annofile
.write(info_dict
['annotations'])
3010 except (KeyError, TypeError):
3011 self
.report_warning('There are no annotations to write.')
3013 self
.report_error('Cannot write annotations file: ' + annofn
)
3016 # Write internet shortcut files
3017 def _write_link_file(link_type
):
3018 url
= try_get(info_dict
['webpage_url'], iri_to_uri
)
3020 self
.report_warning(
3021 f
'Cannot write internet shortcut file because the actual URL of "{info_dict["webpage_url"]}" is unknown')
3023 linkfn
= replace_extension(self
.prepare_filename(info_dict
, 'link'), link_type
, info_dict
.get('ext'))
3024 if not self
._ensure
_dir
_exists
(encodeFilename(linkfn
)):
3026 if self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(linkfn
)):
3027 self
.to_screen(f
'[info] Internet shortcut (.{link_type}) is already present')
3030 self
.to_screen(f
'[info] Writing internet shortcut (.{link_type}) to: {linkfn}')
3031 with open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8',
3032 newline
='\r\n' if link_type
== 'url' else '\n') as linkfile
:
3033 template_vars
= {'url': url}
3034 if link_type
== 'desktop':
3035 template_vars
['filename'] = linkfn
[:-(len(link_type
) + 1)]
3036 linkfile
.write(LINK_TEMPLATES
[link_type
] % template_vars
)
3038 self
.report_error(f
'Cannot write internet shortcut {linkfn}')
3043 'url': self
.params
.get('writeurllink'),
3044 'webloc': self
.params
.get('writewebloclink'),
3045 'desktop': self
.params
.get('writedesktoplink'),
3047 if self
.params
.get('writelink'):
3048 link_type
= ('webloc' if sys
.platform
== 'darwin'
3049 else 'desktop' if sys
.platform
.startswith('linux')
3051 write_links
[link_type
] = True
3053 if any(should_write
and not _write_link_file(link_type
)
3054 for link_type
, should_write
in write_links
.items()):
3057 def replace_info_dict(new_info
):
3059 if new_info
== info_dict
:
3062 info_dict
.update(new_info
)
3064 new_info
, files_to_move
= self
.pre_process(info_dict
, 'before_dl', files_to_move
)
3065 replace_info_dict(new_info
)
3067 if self
.params
.get('skip_download'):
3068 info_dict
['filepath'] = temp_filename
3069 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
3070 info_dict
['__files_to_move'] = files_to_move
3071 replace_info_dict(self
.run_pp(MoveFilesAfterDownloadPP(self
, False), info_dict
))
3072 info_dict
['__write_download_archive'] = self
.params
.get('force_write_download_archive')
3075 info_dict
.setdefault('__postprocessors', [])
3078 def existing_video_file(*filepaths
):
3079 ext
= info_dict
.get('ext')
3080 converted
= lambda file: replace_extension(file, self
.params
.get('final_ext') or ext
, ext
)
3081 file = self
.existing_file(itertools
.chain(*zip(map(converted
, filepaths
), filepaths
)),
3082 default_overwrite
=False)
3084 info_dict
['ext'] = os
.path
.splitext(file)[1][1:]
3087 fd
, success
= None, True
3088 if info_dict
.get('protocol') or info_dict
.get('url'):
3089 fd
= get_suitable_downloader(info_dict
, self
.params
, to_stdout
=temp_filename
== '-')
3090 if fd
is not FFmpegFD
and (
3091 info_dict
.get('section_start') or info_dict
.get('section_end')):
3092 msg
= ('This format cannot be partially downloaded' if FFmpegFD
.available()
3093 else 'You have requested downloading the video partially, but ffmpeg is not installed')
3094 self
.report_error(f
'{msg}. Aborting')
3097 if info_dict
.get('requested_formats') is not None:
3098 requested_formats
= info_dict
['requested_formats']
3099 old_ext
= info_dict
['ext']
3100 if self
.params
.get('merge_output_format') is None:
3101 if (info_dict
['ext'] == 'webm'
3102 and info_dict
.get('thumbnails')
3103 # check with type instead of pp_key, __name__, or isinstance
3104 # since we dont want any custom PPs to trigger this
3105 and any(type(pp
) == EmbedThumbnailPP
for pp
in self
._pps
['post_process'])): # noqa: E721
3106 info_dict
['ext'] = 'mkv'
3107 self
.report_warning(
3108 'webm doesn\'t support embedding a thumbnail, mkv will be used')
3109 new_ext
= info_dict
['ext']
3111 def correct_ext(filename
, ext
=new_ext
):
3114 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
3116 os
.path
.splitext(filename
)[0]
3117 if filename_real_ext
in (old_ext
, new_ext
)
3119 return f
'{filename_wo_ext}.{ext}'
3121 # Ensure filename always has a correct extension for successful merge
3122 full_filename
= correct_ext(full_filename
)
3123 temp_filename
= correct_ext(temp_filename
)
3124 dl_filename
= existing_video_file(full_filename
, temp_filename
)
3125 info_dict
['__real_download'] = False
3127 merger
= FFmpegMergerPP(self
)
3129 if dl_filename
is not None:
3130 self
.report_file_already_downloaded(dl_filename
)
3132 for f
in requested_formats
if fd
!= FFmpegFD
else []:
3133 f
['filepath'] = fname
= prepend_extension(
3134 correct_ext(temp_filename
, info_dict
['ext']),
3135 'f%s' % f
['format_id'], info_dict
['ext'])
3136 downloaded
.append(fname
)
3137 info_dict
['url'] = '\n'.join(f
['url'] for f
in requested_formats
)
3138 success
, real_download
= self
.dl(temp_filename
, info_dict
)
3139 info_dict
['__real_download'] = real_download
3141 if self
.params
.get('allow_unplayable_formats'):
3142 self
.report_warning(
3143 'You have requested merging of multiple formats '
3144 'while also allowing unplayable formats to be downloaded. '
3145 'The formats won\'t be merged to prevent data corruption.')
3146 elif not merger
.available
:
3147 msg
= 'You have requested merging of multiple formats but ffmpeg is not installed'
3148 if not self
.params
.get('ignoreerrors'):
3149 self
.report_error(f
'{msg}. Aborting due to --abort-on-error')
3151 self
.report_warning(f
'{msg}. The formats won\'t be merged')
3153 if temp_filename
== '-':
3154 reason
= ('using a downloader other than ffmpeg' if FFmpegFD
.can_merge_formats(info_dict
, self
.params
)
3155 else 'but the formats are incompatible for simultaneous download' if merger
.available
3156 else 'but ffmpeg is not installed')
3157 self
.report_warning(
3158 f
'You have requested downloading multiple formats to stdout {reason}. '
3159 'The formats will be streamed one after the other')
3160 fname
= temp_filename
3161 for f
in requested_formats
:
3162 new_info
= dict(info_dict
)
3163 del new_info
['requested_formats']
3165 if temp_filename
!= '-':
3166 fname
= prepend_extension(
3167 correct_ext(temp_filename
, new_info
['ext']),
3168 'f%s' % f
['format_id'], new_info
['ext'])
3169 if not self
._ensure
_dir
_exists
(fname
):
3171 f
['filepath'] = fname
3172 downloaded
.append(fname
)
3173 partial_success
, real_download
= self
.dl(fname
, new_info
)
3174 info_dict
['__real_download'] = info_dict
['__real_download'] or real_download
3175 success
= success
and partial_success
3177 if downloaded
and merger
.available
and not self
.params
.get('allow_unplayable_formats'):
3178 info_dict
['__postprocessors'].append(merger
)
3179 info_dict
['__files_to_merge'] = downloaded
3180 # Even if there were no downloads, it is being merged only now
3181 info_dict
['__real_download'] = True
3183 for file in downloaded
:
3184 files_to_move
[file] = None
3186 # Just a single file
3187 dl_filename
= existing_video_file(full_filename
, temp_filename
)
3188 if dl_filename
is None or dl_filename
== temp_filename
:
3189 # dl_filename == temp_filename could mean that the file was partially downloaded with --no-part.
3190 # So we should try to resume the download
3191 success
, real_download
= self
.dl(temp_filename
, info_dict
)
3192 info_dict
['__real_download'] = real_download
3194 self
.report_file_already_downloaded(dl_filename
)
3196 dl_filename
= dl_filename
or temp_filename
3197 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
3199 except network_exceptions
as err
:
3200 self
.report_error('unable to download video data: %s' % error_to_compat_str(err
))
3202 except OSError as err
:
3203 raise UnavailableVideoError(err
)
3204 except (ContentTooShortError
, ) as err
:
3205 self
.report_error(f
'content too short (expected {err.expected} bytes and served {err.downloaded})')
3208 self
._raise
_pending
_errors
(info_dict
)
3209 if success
and full_filename
!= '-':
3213 fixup_policy
= self
.params
.get('fixup')
3214 vid
= info_dict
['id']
3216 if fixup_policy
in ('ignore', 'never'):
3218 elif fixup_policy
== 'warn':
3220 elif fixup_policy
!= 'force':
3221 assert fixup_policy
in ('detect_or_warn', None)
3222 if not info_dict
.get('__real_download'):
3225 def ffmpeg_fixup(cndn
, msg
, cls
):
3226 if not (do_fixup
and cndn
):
3228 elif do_fixup
== 'warn':
3229 self
.report_warning(f
'{vid}: {msg}')
3233 info_dict
['__postprocessors'].append(pp
)
3235 self
.report_warning(f
'{vid}: {msg}. Install ffmpeg to fix this automatically')
3237 stretched_ratio
= info_dict
.get('stretched_ratio')
3238 ffmpeg_fixup(stretched_ratio
not in (1, None),
3239 f
'Non-uniform pixel ratio {stretched_ratio}',
3240 FFmpegFixupStretchedPP
)
3242 downloader
= get_suitable_downloader(info_dict
, self
.params
) if 'protocol' in info_dict
else None
3243 downloader
= downloader
.FD_NAME
if downloader
else None
3245 ext
= info_dict
.get('ext')
3246 postprocessed_by_ffmpeg
= info_dict
.get('requested_formats') or any((
3247 isinstance(pp
, FFmpegVideoConvertorPP
)
3248 and resolve_recode_mapping(ext
, pp
.mapping
)[0] not in (ext
, None)
3249 ) for pp
in self
._pps
['post_process'])
3251 if not postprocessed_by_ffmpeg
:
3252 ffmpeg_fixup(ext
== 'm4a' and info_dict
.get('container') == 'm4a_dash',
3253 'writing DASH m4a. Only some players support this container',
3255 ffmpeg_fixup(downloader
== 'hlsnative' and not self
.params
.get('hls_use_mpegts')
3256 or info_dict
.get('is_live') and self
.params
.get('hls_use_mpegts') is None,
3257 'Possible MPEG-TS in MP4 container or malformed AAC timestamps',
3259 ffmpeg_fixup(info_dict
.get('is_live') and downloader
== 'DashSegmentsFD',
3260 'Possible duplicate MOOV atoms', FFmpegFixupDuplicateMoovPP
)
3262 ffmpeg_fixup(downloader
== 'web_socket_fragment', 'Malformed timestamps detected', FFmpegFixupTimestampPP
)
3263 ffmpeg_fixup(downloader
== 'web_socket_fragment', 'Malformed duration detected', FFmpegFixupDurationPP
)
3267 replace_info_dict(self
.post_process(dl_filename
, info_dict
, files_to_move
))
3268 except PostProcessingError
as err
:
3269 self
.report_error('Postprocessing: %s' % str(err
))
3272 for ph
in self
._post
_hooks
:
3273 ph(info_dict
['filepath'])
3274 except Exception as err
:
3275 self
.report_error('post hooks: %s' % str(err
))
3277 info_dict
['__write_download_archive'] = True
3279 assert info_dict
is original_infodict
# Make sure the info_dict was modified in-place
3280 if self
.params
.get('force_write_download_archive'):
3281 info_dict
['__write_download_archive'] = True
3282 check_max_downloads()
3284 def __download_wrapper(self
, func
):
3285 @functools.wraps(func
)
3286 def wrapper(*args
, **kwargs
):
3288 res
= func(*args
, **kwargs
)
3289 except UnavailableVideoError
as e
:
3290 self
.report_error(e
)
3291 except DownloadCancelled
as e
:
3292 self
.to_screen(f
'[info] {e}')
3293 if not self
.params
.get('break_per_url'):
3295 self
._num
_downloads
= 0
3297 if self
.params
.get('dump_single_json', False):
3298 self
.post_extract(res
)
3299 self
.to_stdout(json
.dumps(self
.sanitize_info(res
)))
3302 def download(self
, url_list
):
3303 """Download a given list of URLs."""
3304 url_list
= variadic(url_list
) # Passing a single URL is a common mistake
3305 outtmpl
= self
.params
['outtmpl']['default']
3306 if (len(url_list
) > 1
3308 and '%' not in outtmpl
3309 and self
.params
.get('max_downloads') != 1):
3310 raise SameFileError(outtmpl
)
3312 for url
in url_list
:
3313 self
.__download
_wrapper
(self
.extract_info
)(
3314 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
3316 return self
._download
_retcode
3318 def download_with_info_file(self
, info_filename
):
3319 with contextlib
.closing(fileinput
.FileInput(
3320 [info_filename
], mode
='r',
3321 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
3322 # FileInput doesn't have a read method, we can't call json.load
3323 info
= self
.sanitize_info(json
.loads('\n'.join(f
)), self
.params
.get('clean_infojson', True))
3325 self
.__download
_wrapper
(self
.process_ie_result
)(info
, download
=True)
3326 except (DownloadError
, EntryNotInPlaylist
, ReExtractInfo
) as e
:
3327 if not isinstance(e
, EntryNotInPlaylist
):
3328 self
.to_stderr('\r')
3329 webpage_url
= info
.get('webpage_url')
3330 if webpage_url
is not None:
3331 self
.report_warning(f
'The info failed to download: {e}; trying with URL {webpage_url}')
3332 return self
.download([webpage_url
])
3335 return self
._download
_retcode
3338 def sanitize_info(info_dict
, remove_private_keys
=False):
3339 ''' Sanitize the infodict for converting to json '''
3340 if info_dict
is None:
3342 info_dict
.setdefault('epoch', int(time
.time()))
3343 info_dict
.setdefault('_type', 'video')
3344 info_dict
.setdefault('_version', {
3345 'version': __version__
,
3346 'current_git_head': current_git_head(),
3347 'release_git_head': RELEASE_GIT_HEAD
,
3348 'repository': REPOSITORY
,
3351 if remove_private_keys
:
3352 reject
= lambda k
, v
: v
is None or k
.startswith('__') or k
in {
3353 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
3354 'entries', 'filepath', '_filename', 'infojson_filename', 'original_url', 'playlist_autonumber',
3357 reject
= lambda k
, v
: False
3360 if isinstance(obj
, dict):
3361 return {k: filter_fn(v) for k, v in obj.items() if not reject(k, v)}
3362 elif isinstance(obj
, (list, tuple, set, LazyList
)):
3363 return list(map(filter_fn
, obj
))
3364 elif obj
is None or isinstance(obj
, (str, int, float, bool)):
3369 return filter_fn(info_dict
)
3372 def filter_requested_info(info_dict
, actually_filter
=True):
3373 ''' Alias of sanitize_info for backward compatibility '''
3374 return YoutubeDL
.sanitize_info(info_dict
, actually_filter
)
3376 def _delete_downloaded_files(self
, *files_to_delete
, info
={}, msg
=None):
3377 for filename
in set(filter(None, files_to_delete
)):
3379 self
.to_screen(msg
% filename
)
3383 self
.report_warning(f
'Unable to delete file {filename}')
3384 if filename
in info
.get('__files_to_move', []): # NB: Delete even if None
3385 del info
['__files_to_move'][filename
]
3388 def post_extract(info_dict
):
3389 def actual_post_extract(info_dict
):
3390 if info_dict
.get('_type') in ('playlist', 'multi_video'):
3391 for video_dict
in info_dict
.get('entries', {}):
3392 actual_post_extract(video_dict
or {})
3395 post_extractor
= info_dict
.pop('__post_extractor', None) or (lambda: {})
3396 info_dict
.update(post_extractor())
3398 actual_post_extract(info_dict
or {})
3400 def run_pp(self
, pp
, infodict
):
3401 files_to_delete
= []
3402 if '__files_to_move' not in infodict
:
3403 infodict
['__files_to_move'] = {}
3405 files_to_delete
, infodict
= pp
.run(infodict
)
3406 except PostProcessingError
as e
:
3407 # Must be True and not 'only_download'
3408 if self
.params
.get('ignoreerrors') is True:
3409 self
.report_error(e
)
3413 if not files_to_delete
:
3415 if self
.params
.get('keepvideo', False):
3416 for f
in files_to_delete
:
3417 infodict
['__files_to_move'].setdefault(f
, '')
3419 self
._delete
_downloaded
_files
(
3420 *files_to_delete
, info
=infodict
, msg
='Deleting original file %s (pass -k to keep)')
3423 def run_all_pps(self
, key
, info
, *, additional_pps
=None):
3424 self
._forceprint
(key
, info
)
3425 for pp
in (additional_pps
or []) + self
._pps
[key
]:
3426 info
= self
.run_pp(pp
, info
)
3429 def pre_process(self
, ie_info
, key
='pre_process', files_to_move
=None):
3430 info
= dict(ie_info
)
3431 info
['__files_to_move'] = files_to_move
or {}
3433 info
= self
.run_all_pps(key
, info
)
3434 except PostProcessingError
as err
:
3435 msg
= f
'Preprocessing: {err}'
3436 info
.setdefault('__pending_error', msg
)
3437 self
.report_error(msg
, is_error
=False)
3438 return info
, info
.pop('__files_to_move', None)
3440 def post_process(self
, filename
, info
, files_to_move
=None):
3441 """Run all the postprocessors on the given file."""
3442 info
['filepath'] = filename
3443 info
['__files_to_move'] = files_to_move
or {}
3444 info
= self
.run_all_pps('post_process', info
, additional_pps
=info
.get('__postprocessors'))
3445 info
= self
.run_pp(MoveFilesAfterDownloadPP(self
), info
)
3446 del info
['__files_to_move']
3447 return self
.run_all_pps('after_move', info
)
3449 def _make_archive_id(self
, info_dict
):
3450 video_id
= info_dict
.get('id')
3453 # Future-proof against any change in case
3454 # and backwards compatibility with prior versions
3455 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
3456 if extractor
is None:
3457 url
= str_or_none(info_dict
.get('url'))
3460 # Try to find matching extractor for the URL and take its ie_key
3461 for ie_key
, ie
in self
._ies
.items():
3462 if ie
.suitable(url
):
3467 return make_archive_id(extractor
, video_id
)
3469 def in_download_archive(self
, info_dict
):
3470 if not self
.archive
:
3473 vid_ids
= [self
._make
_archive
_id
(info_dict
)]
3474 vid_ids
.extend(info_dict
.get('_old_archive_ids') or [])
3475 return any(id_
in self
.archive
for id_
in vid_ids
)
3477 def record_download_archive(self
, info_dict
):
3478 fn
= self
.params
.get('download_archive')
3481 vid_id
= self
._make
_archive
_id
(info_dict
)
3484 self
.write_debug(f
'Adding to archive: {vid_id}')
3485 if isinstance(fn
, os
.PathLike
):
3486 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
3487 archive_file
.write(vid_id
+ '\n')
3488 self
.archive
.add(vid_id
)
3491 def format_resolution(format
, default
='unknown'):
3492 if format
.get('vcodec') == 'none' and format
.get('acodec') != 'none':
3494 if format
.get('resolution') is not None:
3495 return format
['resolution']
3496 if format
.get('width') and format
.get('height'):
3497 return '%dx%d' % (format
['width'], format
['height'])
3498 elif format
.get('height'):
3499 return '%sp' % format
['height']
3500 elif format
.get('width'):
3501 return '%dx?' % format
['width']
3504 def _list_format_headers(self
, *headers
):
3505 if self
.params
.get('listformats_table', True) is not False:
3506 return [self
._format
_out
(header
, self
.Styles
.HEADERS
) for header
in headers
]
3509 def _format_note(self
, fdict
):
3511 if fdict
.get('ext') in ['f4f', 'f4m']:
3512 res
+= '(unsupported)'
3513 if fdict
.get('language'):
3516 res
+= '[%s]' % fdict
['language']
3517 if fdict
.get('format_note') is not None:
3520 res
+= fdict
['format_note']
3521 if fdict
.get('tbr') is not None:
3524 res
+= '%4dk' % fdict
['tbr']
3525 if fdict
.get('container') is not None:
3528 res
+= '%s container' % fdict
['container']
3529 if (fdict
.get('vcodec') is not None
3530 and fdict
.get('vcodec') != 'none'):
3533 res
+= fdict
['vcodec']
3534 if fdict
.get('vbr') is not None:
3536 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
3538 if fdict
.get('vbr') is not None:
3539 res
+= '%4dk' % fdict
['vbr']
3540 if fdict
.get('fps') is not None:
3543 res
+= '%sfps' % fdict
['fps']
3544 if fdict
.get('acodec') is not None:
3547 if fdict
['acodec'] == 'none':
3550 res
+= '%-5s' % fdict
['acodec']
3551 elif fdict
.get('abr') is not None:
3555 if fdict
.get('abr') is not None:
3556 res
+= '@%3dk' % fdict
['abr']
3557 if fdict
.get('asr') is not None:
3558 res
+= ' (%5dHz)' % fdict
['asr']
3559 if fdict
.get('filesize') is not None:
3562 res
+= format_bytes(fdict
['filesize'])
3563 elif fdict
.get('filesize_approx') is not None:
3566 res
+= '~' + format_bytes(fdict
['filesize_approx'])
3569 def render_formats_table(self
, info_dict
):
3570 if not info_dict
.get('formats') and not info_dict
.get('url'):
3573 formats
= info_dict
.get('formats', [info_dict
])
3574 if not self
.params
.get('listformats_table', True) is not False:
3577 format_field(f
, 'format_id'),
3578 format_field(f
, 'ext'),
3579 self
.format_resolution(f
),
3580 self
._format
_note
(f
)
3581 ] for f
in formats
if f
.get('preference') is None or f
['preference'] >= -1000]
3582 return render_table(['format code', 'extension', 'resolution', 'note'], table
, extra_gap
=1)
3584 def simplified_codec(f
, field
):
3585 assert field
in ('acodec', 'vcodec')
3586 codec
= f
.get(field
, 'unknown')
3589 elif codec
!= 'none':
3590 return '.'.join(codec
.split('.')[:4])
3592 if field
== 'vcodec' and f
.get('acodec') == 'none':
3594 elif field
== 'acodec' and f
.get('vcodec') == 'none':
3596 return self
._format
_out
('audio only' if field
== 'vcodec' else 'video only',
3597 self
.Styles
.SUPPRESS
)
3599 delim
= self
._format
_out
('\u2502', self
.Styles
.DELIM
, '|', test_encoding
=True)
3602 self
._format
_out
(format_field(f
, 'format_id'), self
.Styles
.ID
),
3603 format_field(f
, 'ext'),
3604 format_field(f
, func
=self
.format_resolution
, ignore
=('audio only', 'images')),
3605 format_field(f
, 'fps', '\t%d', func
=round),
3606 format_field(f
, 'dynamic_range', '%s', ignore
=(None, 'SDR')).replace('HDR', ''),
3607 format_field(f
, 'audio_channels', '\t%s'),
3609 format_field(f
, 'filesize', ' \t%s', func
=format_bytes
) + format_field(f
, 'filesize_approx', '~\t%s', func
=format_bytes
),
3610 format_field(f
, 'tbr', '\t%dk', func
=round),
3611 shorten_protocol_name(f
.get('protocol', '')),
3613 simplified_codec(f
, 'vcodec'),
3614 format_field(f
, 'vbr', '\t%dk', func
=round),
3615 simplified_codec(f
, 'acodec'),
3616 format_field(f
, 'abr', '\t%dk', func
=round),
3617 format_field(f
, 'asr', '\t%s', func
=format_decimal_suffix
),
3619 self
._format
_out
('UNSUPPORTED', 'light red') if f
.get('ext') in ('f4f', 'f4m') else None,
3620 format_field(f
, 'language', '[%s]'),
3621 join_nonempty(format_field(f
, 'format_note'),
3622 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
3625 ] for f
in formats
if f
.get('preference') is None or f
['preference'] >= -1000]
3626 header_line
= self
._list
_format
_headers
(
3627 'ID', 'EXT', 'RESOLUTION', '\tFPS', 'HDR', 'CH', delim
, '\tFILESIZE', '\tTBR', 'PROTO',
3628 delim
, 'VCODEC', '\tVBR', 'ACODEC', '\tABR', '\tASR', 'MORE INFO')
3630 return render_table(
3631 header_line
, table
, hide_empty
=True,
3632 delim
=self
._format
_out
('\u2500', self
.Styles
.DELIM
, '-', test_encoding
=True))
3634 def render_thumbnails_table(self
, info_dict
):
3635 thumbnails
= list(info_dict
.get('thumbnails') or [])
3638 return render_table(
3639 self
._list
_format
_headers
('ID', 'Width', 'Height', 'URL'),
3640 [[t
.get('id'), t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
])
3642 def render_subtitles_table(self
, video_id
, subtitles
):
3643 def _row(lang
, formats
):
3644 exts
, names
= zip(*((f
['ext'], f
.get('name') or 'unknown') for f
in reversed(formats
)))
3645 if len(set(names
)) == 1:
3646 names
= [] if names
[0] == 'unknown' else names
[:1]
3647 return [lang
, ', '.join(names
), ', '.join(exts
)]
3651 return render_table(
3652 self
._list
_format
_headers
('Language', 'Name', 'Formats'),
3653 [_row(lang
, formats
) for lang
, formats
in subtitles
.items()],
3656 def __list_table(self
, video_id
, name
, func
, *args
):
3659 self
.to_screen(f
'{video_id} has no {name}')
3661 self
.to_screen(f
'[info] Available {name} for {video_id}:')
3662 self
.to_stdout(table
)
3664 def list_formats(self
, info_dict
):
3665 self
.__list
_table
(info_dict
['id'], 'formats', self
.render_formats_table
, info_dict
)
3667 def list_thumbnails(self
, info_dict
):
3668 self
.__list
_table
(info_dict
['id'], 'thumbnails', self
.render_thumbnails_table
, info_dict
)
3670 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
3671 self
.__list
_table
(video_id
, name
, self
.render_subtitles_table
, video_id
, subtitles
)
3673 def urlopen(self
, req
):
3674 """ Start an HTTP download """
3675 if isinstance(req
, str):
3676 req
= sanitized_Request(req
)
3677 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
3679 def print_debug_header(self
):
3680 if not self
.params
.get('verbose'):
3683 # These imports can be slow. So import them only as needed
3684 from .extractor
.extractors
import _LAZY_LOADER
3685 from .extractor
.extractors
import _PLUGIN_CLASSES
as plugin_extractors
3687 def get_encoding(stream
):
3688 ret
= str(getattr(stream
, 'encoding', 'missing (%s)' % type(stream
).__name
__))
3689 if not supports_terminal_sequences(stream
):
3690 from .utils
import WINDOWS_VT_MODE
# Must be imported locally
3691 ret
+= ' (No VT)' if WINDOWS_VT_MODE
is False else ' (No ANSI)'
3694 encoding_str
= 'Encodings: locale %s, fs %s, pref %s, %s' % (
3695 locale
.getpreferredencoding(),
3696 sys
.getfilesystemencoding(),
3697 self
.get_encoding(),
3699 f
'{key} {get_encoding(stream)}' for key
, stream
in self
._out
_files
.items_
3700 if stream
is not None and key
!= 'console')
3703 logger
= self
.params
.get('logger')
3705 write_debug
= lambda msg
: logger
.debug(f
'[debug] {msg}')
3706 write_debug(encoding_str
)
3708 write_string(f
'[debug] {encoding_str}\n', encoding
=None)
3709 write_debug
= lambda msg
: self
._write
_string
(f
'[debug] {msg}\n')
3711 source
= detect_variant()
3712 if VARIANT
not in (None, 'pip'):
3714 write_debug(join_nonempty(
3715 f
'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
3717 f
'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD
else '',
3718 '' if source
== 'unknown' else f
'({source})',
3720 if not _LAZY_LOADER
:
3721 if os
.environ
.get('YTDLP_NO_LAZY_EXTRACTORS'):
3722 write_debug('Lazy loading extractors is forcibly disabled')
3724 write_debug('Lazy loading extractors is disabled')
3725 if plugin_extractors
or plugin_postprocessors
:
3726 write_debug('Plugins: %s' % [
3727 '%s%s' % (klass
.__name
__, '' if klass
.__name
__ == name
else f
' as {name}')
3728 for name
, klass
in itertools
.chain(plugin_extractors
.items(), plugin_postprocessors
.items())])
3729 if self
.params
['compat_opts']:
3730 write_debug('Compatibility options: %s' % ', '.join(self
.params
['compat_opts']))
3732 if current_git_head():
3733 write_debug(f
'Git HEAD: {current_git_head()}')
3734 write_debug(system_identifier())
3736 exe_versions
, ffmpeg_features
= FFmpegPostProcessor
.get_versions_and_features(self
)
3737 ffmpeg_features
= {key for key, val in ffmpeg_features.items() if val}
3739 exe_versions
['ffmpeg'] += ' (%s)' % ','.join(sorted(ffmpeg_features
))
3741 exe_versions
['rtmpdump'] = rtmpdump_version()
3742 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
3743 exe_str
= ', '.join(
3744 f
'{exe} {v}' for exe
, v
in sorted(exe_versions
.items()) if v
3746 write_debug('exe versions: %s' % exe_str
)
3748 from .compat
.compat_utils
import get_package_info
3749 from .dependencies
import available_dependencies
3751 write_debug('Optional libraries: %s' % (', '.join(sorted({
3752 join_nonempty(*get_package_info(m
)) for m
in available_dependencies
.values()
3755 self
._setup
_opener
()
3757 for handler
in self
._opener
.handlers
:
3758 if hasattr(handler
, 'proxies'):
3759 proxy_map
.update(handler
.proxies
)
3760 write_debug(f
'Proxy map: {proxy_map}')
3763 if False and self
.params
.get('call_home'):
3764 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode()
3765 write_debug('Public IP address: %s' % ipaddr
)
3766 latest_version
= self
.urlopen(
3767 'https://yt-dl.org/latest/version').read().decode()
3768 if version_tuple(latest_version
) > version_tuple(__version__
):
3769 self
.report_warning(
3770 'You are using an outdated version (newest version: %s)! '
3771 'See https://yt-dl.org/update if you need help updating.' %
3774 def _setup_opener(self
):
3775 if hasattr(self
, '_opener'):
3777 timeout_val
= self
.params
.get('socket_timeout')
3778 self
._socket
_timeout
= 20 if timeout_val
is None else float(timeout_val
)
3780 opts_cookiesfrombrowser
= self
.params
.get('cookiesfrombrowser')
3781 opts_cookiefile
= self
.params
.get('cookiefile')
3782 opts_proxy
= self
.params
.get('proxy')
3784 self
.cookiejar
= load_cookies(opts_cookiefile
, opts_cookiesfrombrowser
, self
)
3786 cookie_processor
= YoutubeDLCookieProcessor(self
.cookiejar
)
3787 if opts_proxy
is not None:
3788 if opts_proxy
== '':
3791 proxies
= {'http': opts_proxy, 'https': opts_proxy}
3793 proxies
= urllib
.request
.getproxies()
3794 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
3795 if 'http' in proxies
and 'https' not in proxies
:
3796 proxies
['https'] = proxies
['http']
3797 proxy_handler
= PerRequestProxyHandler(proxies
)
3799 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
3800 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
3801 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
3802 redirect_handler
= YoutubeDLRedirectHandler()
3803 data_handler
= urllib
.request
.DataHandler()
3805 # When passing our own FileHandler instance, build_opener won't add the
3806 # default FileHandler and allows us to disable the file protocol, which
3807 # can be used for malicious purposes (see
3808 # https://github.com/ytdl-org/youtube-dl/issues/8227)
3809 file_handler
= urllib
.request
.FileHandler()
3811 def file_open(*args
, **kwargs
):
3812 raise urllib
.error
.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
3813 file_handler
.file_open
= file_open
3815 opener
= urllib
.request
.build_opener(
3816 proxy_handler
, https_handler
, cookie_processor
, ydlh
, redirect_handler
, data_handler
, file_handler
)
3818 # Delete the default user-agent header, which would otherwise apply in
3819 # cases where our custom HTTP handler doesn't come into play
3820 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
3821 opener
.addheaders
= []
3822 self
._opener
= opener
3824 def encode(self
, s
):
3825 if isinstance(s
, bytes):
3826 return s
# Already encoded
3829 return s
.encode(self
.get_encoding())
3830 except UnicodeEncodeError as err
:
3831 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
3834 def get_encoding(self
):
3835 encoding
= self
.params
.get('encoding')
3836 if encoding
is None:
3837 encoding
= preferredencoding()
3840 def _write_info_json(self
, label
, ie_result
, infofn
, overwrite
=None):
3841 ''' Write infojson and returns True = written, 'exists' = Already exists, False = skip, None = error '''
3842 if overwrite
is None:
3843 overwrite
= self
.params
.get('overwrites', True)
3844 if not self
.params
.get('writeinfojson'):
3847 self
.write_debug(f
'Skipping writing {label} infojson')
3849 elif not self
._ensure
_dir
_exists
(infofn
):
3851 elif not overwrite
and os
.path
.exists(infofn
):
3852 self
.to_screen(f
'[info] {label.title()} metadata is already present')
3855 self
.to_screen(f
'[info] Writing {label} metadata as JSON to: {infofn}')
3857 write_json_file(self
.sanitize_info(ie_result
, self
.params
.get('clean_infojson', True)), infofn
)
3860 self
.report_error(f
'Cannot write {label} metadata to JSON file {infofn}')
3863 def _write_description(self
, label
, ie_result
, descfn
):
3864 ''' Write description and returns True = written, False = skip, None = error '''
3865 if not self
.params
.get('writedescription'):
3868 self
.write_debug(f
'Skipping writing {label} description')
3870 elif not self
._ensure
_dir
_exists
(descfn
):
3872 elif not self
.params
.get('overwrites', True) and os
.path
.exists(descfn
):
3873 self
.to_screen(f
'[info] {label.title()} description is already present')
3874 elif ie_result
.get('description') is None:
3875 self
.report_warning(f
'There\'s no {label} description to write')
3879 self
.to_screen(f
'[info] Writing {label} description to: {descfn}')
3880 with open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
3881 descfile
.write(ie_result
['description'])
3883 self
.report_error(f
'Cannot write {label} description file {descfn}')
3887 def _write_subtitles(self
, info_dict
, filename
):
3888 ''' Write subtitles to file and return list of (sub_filename, final_sub_filename); or None if error'''
3890 subtitles
= info_dict
.get('requested_subtitles')
3891 if not subtitles
or not (self
.params
.get('writesubtitles') or self
.params
.get('writeautomaticsub')):
3892 # subtitles download errors are already managed as troubles in relevant IE
3893 # that way it will silently go on when used with unsupporting IE
3896 sub_filename_base
= self
.prepare_filename(info_dict
, 'subtitle')
3897 if not sub_filename_base
:
3898 self
.to_screen('[info] Skipping writing video subtitles')
3900 for sub_lang
, sub_info
in subtitles
.items():
3901 sub_format
= sub_info
['ext']
3902 sub_filename
= subtitles_filename(filename
, sub_lang
, sub_format
, info_dict
.get('ext'))
3903 sub_filename_final
= subtitles_filename(sub_filename_base
, sub_lang
, sub_format
, info_dict
.get('ext'))
3904 existing_sub
= self
.existing_file((sub_filename_final
, sub_filename
))
3906 self
.to_screen(f
'[info] Video subtitle {sub_lang}.{sub_format} is already present')
3907 sub_info
['filepath'] = existing_sub
3908 ret
.append((existing_sub
, sub_filename_final
))
3911 self
.to_screen(f
'[info] Writing video subtitles to: {sub_filename}')
3912 if sub_info
.get('data') is not None:
3914 # Use newline='' to prevent conversion of newline characters
3915 # See https://github.com/ytdl-org/youtube-dl/issues/10268
3916 with open(sub_filename
, 'w', encoding
='utf-8', newline
='') as subfile
:
3917 subfile
.write(sub_info
['data'])
3918 sub_info
['filepath'] = sub_filename
3919 ret
.append((sub_filename
, sub_filename_final
))
3922 self
.report_error(f
'Cannot write video subtitles file {sub_filename}')
3926 sub_copy
= sub_info
.copy()
3927 sub_copy
.setdefault('http_headers', info_dict
.get('http_headers'))
3928 self
.dl(sub_filename
, sub_copy
, subtitle
=True)
3929 sub_info
['filepath'] = sub_filename
3930 ret
.append((sub_filename
, sub_filename_final
))
3931 except (DownloadError
, ExtractorError
, IOError, OSError, ValueError) + network_exceptions
as err
:
3932 msg
= f
'Unable to download video subtitles for {sub_lang!r}: {err}'
3933 if self
.params
.get('ignoreerrors') is not True: # False or 'only_download'
3934 if not self
.params
.get('ignoreerrors'):
3935 self
.report_error(msg
)
3936 raise DownloadError(msg
)
3937 self
.report_warning(msg
)
3940 def _write_thumbnails(self
, label
, info_dict
, filename
, thumb_filename_base
=None):
3941 ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename) '''
3942 write_all
= self
.params
.get('write_all_thumbnails', False)
3943 thumbnails
, ret
= [], []
3944 if write_all
or self
.params
.get('writethumbnail', False):
3945 thumbnails
= info_dict
.get('thumbnails') or []
3946 multiple
= write_all
and len(thumbnails
) > 1
3948 if thumb_filename_base
is None:
3949 thumb_filename_base
= filename
3950 if thumbnails
and not thumb_filename_base
:
3951 self
.write_debug(f
'Skipping writing {label} thumbnail')
3954 for idx
, t
in list(enumerate(thumbnails
))[::-1]:
3955 thumb_ext
= (f
'{t["id"]}.' if multiple
else '') + determine_ext(t
['url'], 'jpg')
3956 thumb_display_id
= f
'{label} thumbnail {t["id"]}'
3957 thumb_filename
= replace_extension(filename
, thumb_ext
, info_dict
.get('ext'))
3958 thumb_filename_final
= replace_extension(thumb_filename_base
, thumb_ext
, info_dict
.get('ext'))
3960 existing_thumb
= self
.existing_file((thumb_filename_final
, thumb_filename
))
3962 self
.to_screen('[info] %s is already present' % (
3963 thumb_display_id
if multiple
else f
'{label} thumbnail').capitalize())
3964 t
['filepath'] = existing_thumb
3965 ret
.append((existing_thumb
, thumb_filename_final
))
3967 self
.to_screen(f
'[info] Downloading {thumb_display_id} ...')
3969 uf
= self
.urlopen(sanitized_Request(t
['url'], headers
=t
.get('http_headers', {})))
3970 self
.to_screen(f
'[info] Writing {thumb_display_id} to: {thumb_filename}')
3971 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
3972 shutil
.copyfileobj(uf
, thumbf
)
3973 ret
.append((thumb_filename
, thumb_filename_final
))
3974 t
['filepath'] = thumb_filename
3975 except network_exceptions
as err
:
3977 self
.report_warning(f
'Unable to download {thumb_display_id}: {err}')
3978 if ret
and not write_all
: