26 from string
import ascii_letters
28 from .cache
import Cache
30 compat_get_terminal_size
,
35 compat_urllib_request
,
36 windows_enable_vt_mode
,
38 from .cookies
import load_cookies
39 from .downloader
import FFmpegFD
, get_suitable_downloader
, shorten_protocol_name
40 from .downloader
.rtmp
import rtmpdump_version
41 from .extractor
import _LAZY_LOADER
42 from .extractor
import _PLUGIN_CLASSES
as plugin_extractors
43 from .extractor
import gen_extractor_classes
, get_info_extractor
44 from .extractor
.openload
import PhantomJSwrapper
45 from .minicurses
import format_text
46 from .postprocessor
import _PLUGIN_CLASSES
as plugin_postprocessors
47 from .postprocessor
import (
49 FFmpegFixupDuplicateMoovPP
,
50 FFmpegFixupDurationPP
,
53 FFmpegFixupStretchedPP
,
54 FFmpegFixupTimestampPP
,
57 MoveFilesAfterDownloadPP
,
60 from .update
import detect_variant
84 PerRequestProxyHandler
,
90 UnavailableVideoError
,
91 YoutubeDLCookieProcessor
,
93 YoutubeDLRedirectHandler
,
106 format_decimal_suffix
,
124 register_socks_protocols
,
125 remove_terminal_sequences
,
136 supports_terminal_sequences
,
147 from .version
import RELEASE_GIT_HEAD
, __version__
149 if compat_os_name
== 'nt':
156 YoutubeDL objects are the ones responsible of downloading the
157 actual video file and writing it to disk if the user has requested
158 it, among some other tasks. In most cases there should be one per
159 program. As, given a video URL, the downloader doesn't know how to
160 extract all the needed information, task that InfoExtractors do, it
161 has to pass the URL to one of them.
163 For this, YoutubeDL objects have a method that allows
164 InfoExtractors to be registered in a given order. When it is passed
165 a URL, the YoutubeDL object handles it to the first InfoExtractor it
166 finds that reports being able to handle it. The InfoExtractor extracts
167 all the information about the video or videos the URL refers to, and
168 YoutubeDL process the extracted information, possibly using a File
169 Downloader to download the video.
171 YoutubeDL objects accept a lot of parameters. In order not to saturate
172 the object constructor with arguments, it receives a dictionary of
173 options instead. These options are available through the params
174 attribute for the InfoExtractors to use. The YoutubeDL also
175 registers itself as the downloader in charge for the InfoExtractors
176 that are added to it, so this is a "mutual registration".
180 username: Username for authentication purposes.
181 password: Password for authentication purposes.
182 videopassword: Password for accessing a video.
183 ap_mso: Adobe Pass multiple-system operator identifier.
184 ap_username: Multiple-system operator account username.
185 ap_password: Multiple-system operator account password.
186 usenetrc: Use netrc for authentication instead.
187 verbose: Print additional info to stdout.
188 quiet: Do not print messages to stdout.
189 no_warnings: Do not print out anything for warnings.
190 forceprint: A dict with keys WHEN mapped to a list of templates to
191 print to stdout. The allowed keys are video or any of the
192 items in utils.POSTPROCESS_WHEN.
193 For compatibility, a single list is also accepted
194 print_to_file: A dict with keys WHEN (same as forceprint) mapped to
195 a list of tuples with (template, filename)
196 forceurl: Force printing final URL. (Deprecated)
197 forcetitle: Force printing title. (Deprecated)
198 forceid: Force printing ID. (Deprecated)
199 forcethumbnail: Force printing thumbnail URL. (Deprecated)
200 forcedescription: Force printing description. (Deprecated)
201 forcefilename: Force printing final filename. (Deprecated)
202 forceduration: Force printing duration. (Deprecated)
203 forcejson: Force printing info_dict as JSON.
204 dump_single_json: Force printing the info_dict of the whole playlist
205 (or video) as a single JSON line.
206 force_write_download_archive: Force writing download archive regardless
207 of 'skip_download' or 'simulate'.
208 simulate: Do not download the video files. If unset (or None),
209 simulate only if listsubtitles, listformats or list_thumbnails is used
210 format: Video format code. see "FORMAT SELECTION" for more details.
211 You can also pass a function. The function takes 'ctx' as
212 argument and returns the formats to download.
213 See "build_format_selector" for an implementation
214 allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
215 ignore_no_formats_error: Ignore "No video formats" error. Usefull for
216 extracting metadata even if the video is not actually
217 available for download (experimental)
218 format_sort: A list of fields by which to sort the video formats.
219 See "Sorting Formats" for more details.
220 format_sort_force: Force the given format_sort. see "Sorting Formats"
222 prefer_free_formats: Whether to prefer video formats with free containers
223 over non-free ones of same quality.
224 allow_multiple_video_streams: Allow multiple video streams to be merged
226 allow_multiple_audio_streams: Allow multiple audio streams to be merged
228 check_formats Whether to test if the formats are downloadable.
229 Can be True (check all), False (check none),
230 'selected' (check selected formats),
231 or None (check only if requested by extractor)
232 paths: Dictionary of output paths. The allowed keys are 'home'
233 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
234 outtmpl: Dictionary of templates for output names. Allowed keys
235 are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
236 For compatibility with youtube-dl, a single string can also be used
237 outtmpl_na_placeholder: Placeholder for unavailable meta fields.
238 restrictfilenames: Do not allow "&" and spaces in file names
239 trim_file_name: Limit length of filename (extension excluded)
240 windowsfilenames: Force the filenames to be windows compatible
241 ignoreerrors: Do not stop on download/postprocessing errors.
242 Can be 'only_download' to ignore only download errors.
243 Default is 'only_download' for CLI, but False for API
244 skip_playlist_after_errors: Number of allowed failures until the rest of
245 the playlist is skipped
246 force_generic_extractor: Force downloader to use the generic extractor
247 overwrites: Overwrite all video and metadata files if True,
248 overwrite only non-video files if None
249 and don't overwrite any file if False
250 For compatibility with youtube-dl,
251 "nooverwrites" may also be used instead
252 playliststart: Playlist item to start at.
253 playlistend: Playlist item to end at.
254 playlist_items: Specific indices of playlist to download.
255 playlistreverse: Download playlist items in reverse order.
256 playlistrandom: Download playlist items in random order.
257 matchtitle: Download only matching titles.
258 rejecttitle: Reject downloads for matching titles.
259 logger: Log messages to a logging.Logger instance.
260 logtostderr: Log messages to stderr instead of stdout.
261 consoletitle: Display progress in console window's titlebar.
262 writedescription: Write the video description to a .description file
263 writeinfojson: Write the video description to a .info.json file
264 clean_infojson: Remove private fields from the infojson
265 getcomments: Extract video comments. This will not be written to disk
266 unless writeinfojson is also given
267 writeannotations: Write the video annotations to a .annotations.xml file
268 writethumbnail: Write the thumbnail image to a file
269 allow_playlist_files: Whether to write playlists' description, infojson etc
270 also to disk when using the 'write*' options
271 write_all_thumbnails: Write all thumbnail formats to files
272 writelink: Write an internet shortcut file, depending on the
273 current platform (.url/.webloc/.desktop)
274 writeurllink: Write a Windows internet shortcut file (.url)
275 writewebloclink: Write a macOS internet shortcut file (.webloc)
276 writedesktoplink: Write a Linux internet shortcut file (.desktop)
277 writesubtitles: Write the video subtitles to a file
278 writeautomaticsub: Write the automatically generated subtitles to a file
279 allsubtitles: Deprecated - Use subtitleslangs = ['all']
280 Downloads all the subtitles of the video
281 (requires writesubtitles or writeautomaticsub)
282 listsubtitles: Lists all available subtitles for the video
283 subtitlesformat: The format code for subtitles
284 subtitleslangs: List of languages of the subtitles to download (can be regex).
285 The list may contain "all" to refer to all the available
286 subtitles. The language can be prefixed with a "-" to
287 exclude it from the requested languages. Eg: ['all', '-live_chat']
288 keepvideo: Keep the video file after post-processing
289 daterange: A DateRange object, download only if the upload_date is in the range.
290 skip_download: Skip the actual download of the video file
291 cachedir: Location of the cache files in the filesystem.
292 False to disable filesystem cache.
293 noplaylist: Download single video instead of a playlist if in doubt.
294 age_limit: An integer representing the user's age in years.
295 Unsuitable videos for the given age are skipped.
296 min_views: An integer representing the minimum view count the video
297 must have in order to not be skipped.
298 Videos without view count information are always
299 downloaded. None for no limit.
300 max_views: An integer representing the maximum view count.
301 Videos that are more popular than that are not
303 Videos without view count information are always
304 downloaded. None for no limit.
305 download_archive: File name of a file where all downloads are recorded.
306 Videos already present in the file are not downloaded
308 break_on_existing: Stop the download process after attempting to download a
309 file that is in the archive.
310 break_on_reject: Stop the download process when encountering a video that
311 has been filtered out.
312 break_per_url: Whether break_on_reject and break_on_existing
313 should act on each input URL as opposed to for the entire queue
314 cookiefile: File name where cookies should be read from and dumped to
315 cookiesfrombrowser: A tuple containing the name of the browser, the profile
316 name/pathfrom where cookies are loaded, and the name of the
317 keyring. Eg: ('chrome', ) or ('vivaldi', 'default', 'BASICTEXT')
318 legacyserverconnect: Explicitly allow HTTPS connection to servers that do not
319 support RFC 5746 secure renegotiation
320 nocheckcertificate: Do not verify SSL certificates
321 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
322 At the moment, this is only supported by YouTube.
323 http_headers: A dictionary of custom headers to be used for all requests
324 proxy: URL of the proxy server to use
325 geo_verification_proxy: URL of the proxy to use for IP address verification
326 on geo-restricted sites.
327 socket_timeout: Time to wait for unresponsive hosts, in seconds
328 bidi_workaround: Work around buggy terminals without bidirectional text
329 support, using fridibi
330 debug_printtraffic:Print out sent and received HTTP traffic
331 include_ads: Download ads as well (deprecated)
332 default_search: Prepend this string if an input url is not valid.
333 'auto' for elaborate guessing
334 encoding: Use this encoding instead of the system-specified.
335 extract_flat: Do not resolve URLs, return the immediate result.
336 Pass in 'in_playlist' to only show this behavior for
338 wait_for_video: If given, wait for scheduled streams to become available.
339 The value should be a tuple containing the range
340 (min_secs, max_secs) to wait between retries
341 postprocessors: A list of dictionaries, each with an entry
342 * key: The name of the postprocessor. See
343 yt_dlp/postprocessor/__init__.py for a list.
344 * when: When to run the postprocessor. Allowed values are
345 the entries of utils.POSTPROCESS_WHEN
346 Assumed to be 'post_process' if not given
347 post_hooks: Deprecated - Register a custom postprocessor instead
348 A list of functions that get called as the final step
349 for each video file, after all postprocessors have been
350 called. The filename will be passed as the only argument.
351 progress_hooks: A list of functions that get called on download
352 progress, with a dictionary with the entries
353 * status: One of "downloading", "error", or "finished".
354 Check this first and ignore unknown values.
355 * info_dict: The extracted info_dict
357 If status is one of "downloading", or "finished", the
358 following properties may also be present:
359 * filename: The final filename (always present)
360 * tmpfilename: The filename we're currently writing to
361 * downloaded_bytes: Bytes on disk
362 * total_bytes: Size of the whole file, None if unknown
363 * total_bytes_estimate: Guess of the eventual file size,
365 * elapsed: The number of seconds since download started.
366 * eta: The estimated time in seconds, None if unknown
367 * speed: The download speed in bytes/second, None if
369 * fragment_index: The counter of the currently
370 downloaded video fragment.
371 * fragment_count: The number of fragments (= individual
372 files that will be merged)
374 Progress hooks are guaranteed to be called at least once
375 (with status "finished") if the download is successful.
376 postprocessor_hooks: A list of functions that get called on postprocessing
377 progress, with a dictionary with the entries
378 * status: One of "started", "processing", or "finished".
379 Check this first and ignore unknown values.
380 * postprocessor: Name of the postprocessor
381 * info_dict: The extracted info_dict
383 Progress hooks are guaranteed to be called at least twice
384 (with status "started" and "finished") if the processing is successful.
385 merge_output_format: Extension to use when merging formats.
386 final_ext: Expected final extension; used to detect when the file was
387 already downloaded and converted
388 fixup: Automatically correct known faults of the file.
390 - "never": do nothing
391 - "warn": only emit a warning
392 - "detect_or_warn": check whether we can do anything
393 about it, warn otherwise (default)
394 source_address: Client-side IP address to bind to.
395 call_home: Boolean, true iff we are allowed to contact the
396 yt-dlp servers for debugging. (BROKEN)
397 sleep_interval_requests: Number of seconds to sleep between requests
399 sleep_interval: Number of seconds to sleep before each download when
400 used alone or a lower bound of a range for randomized
401 sleep before each download (minimum possible number
402 of seconds to sleep) when used along with
404 max_sleep_interval:Upper bound of a range for randomized sleep before each
405 download (maximum possible number of seconds to sleep).
406 Must only be used along with sleep_interval.
407 Actual sleep time will be a random float from range
408 [sleep_interval; max_sleep_interval].
409 sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
410 listformats: Print an overview of available video formats and exit.
411 list_thumbnails: Print a table of all thumbnails and exit.
412 match_filter: A function that gets called with the info_dict of
414 If it returns a message, the video is ignored.
415 If it returns None, the video is downloaded.
416 match_filter_func in utils.py is one example for this.
417 no_color: Do not emit color codes in output.
418 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
421 Two-letter ISO 3166-2 country code that will be used for
422 explicit geographic restriction bypassing via faking
423 X-Forwarded-For HTTP header
425 IP range in CIDR notation that will be used similarly to
428 The following options determine which downloader is picked:
429 external_downloader: A dictionary of protocol keys and the executable of the
430 external downloader to use for it. The allowed protocols
431 are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
432 Set the value to 'native' to use the native downloader
433 hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
434 or {'m3u8': 'ffmpeg'} instead.
435 Use the native HLS downloader instead of ffmpeg/avconv
436 if True, otherwise use ffmpeg/avconv if False, otherwise
437 use downloader suggested by extractor if None.
438 compat_opts: Compatibility options. See "Differences in default behavior".
439 The following options do not work when used through the API:
440 filename, abort-on-error, multistreams, no-live-chat, format-sort
441 no-clean-infojson, no-playlist-metafiles, no-keep-subs, no-attach-info-json.
442 Refer __init__.py for their implementation
443 progress_template: Dictionary of templates for progress outputs.
444 Allowed keys are 'download', 'postprocess',
445 'download-title' (console title) and 'postprocess-title'.
446 The template is mapped on a dictionary with keys 'progress' and 'info'
448 The following parameters are not used by YoutubeDL itself, they are used by
449 the downloader (see yt_dlp/downloader/common.py):
450 nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
451 max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries,
452 continuedl, noprogress, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
453 external_downloader_args, concurrent_fragment_downloads.
455 The following options are used by the post processors:
456 prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
457 otherwise prefer ffmpeg. (avconv support is deprecated)
458 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
459 to the binary or its containing directory.
460 postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
461 and a list of additional command-line arguments for the
462 postprocessor/executable. The dict can also have "PP+EXE" keys
463 which are used when the given exe is used by the given PP.
464 Use 'default' as the name for arguments to passed to all PP
465 For compatibility with youtube-dl, a single list of args
468 The following options are used by the extractors:
469 extractor_retries: Number of times to retry for known errors
470 dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
471 hls_split_discontinuity: Split HLS playlists to different formats at
472 discontinuities such as ad breaks (default: False)
473 extractor_args: A dictionary of arguments to be passed to the extractors.
474 See "EXTRACTOR ARGUMENTS" for details.
475 Eg: {'youtube': {'skip': ['dash', 'hls']}}
476 mark_watched: Mark videos watched (even with --simulate). Only for YouTube
477 youtube_include_dash_manifest: Deprecated - Use extractor_args instead.
478 If True (default), DASH manifests and related
479 data will be downloaded and processed by extractor.
480 You can reduce network I/O by disabling it if you don't
481 care about DASH. (only for youtube)
482 youtube_include_hls_manifest: Deprecated - Use extractor_args instead.
483 If True (default), HLS manifests and related
484 data will be downloaded and processed by extractor.
485 You can reduce network I/O by disabling it if you don't
486 care about HLS. (only for youtube)
490 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
491 'timestamp', 'release_timestamp',
492 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
493 'average_rating', 'comment_count', 'age_limit',
494 'start_time', 'end_time',
495 'chapter_number', 'season_number', 'episode_number',
496 'track_number', 'disc_number', 'release_year',
500 # NB: Keep in sync with the docstring of extractor/common.py
501 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note',
502 'width', 'height', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr',
503 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx',
504 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start',
505 'preference', 'language', 'language_preference', 'quality', 'source_preference',
506 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'downloader_options',
507 'page_url', 'app', 'play_path', 'tc_url', 'flash_version', 'rtmp_live', 'rtmp_conn', 'rtmp_protocol', 'rtmp_real_time'
509 _format_selection_exts
= {
510 'audio': {'m4a', 'mp3', 'ogg', 'aac'}
,
511 'video': {'mp4', 'flv', 'webm', '3gp'}
,
512 'storyboards': {'mhtml'}
,
515 def __init__(self
, params
=None, auto_init
=True):
516 """Create a FileDownloader object with the given options.
517 @param auto_init Whether to load the default extractors and print header (if verbose).
518 Set to 'no_verbose_header' to not print the header
524 self
._ies
_instances
= {}
525 self
._pps
= {k: [] for k in POSTPROCESS_WHEN}
526 self
._printed
_messages
= set()
527 self
._first
_webpage
_request
= True
528 self
._post
_hooks
= []
529 self
._progress
_hooks
= []
530 self
._postprocessor
_hooks
= []
531 self
._download
_retcode
= 0
532 self
._num
_downloads
= 0
534 self
._playlist
_level
= 0
535 self
._playlist
_urls
= set()
536 self
.cache
= Cache(self
)
538 windows_enable_vt_mode()
541 'print': sys
.stderr
if self
.params
.get('logtostderr') else sys
.stdout
,
542 'console': None if compat_os_name
== 'nt' else next(
543 filter(supports_terminal_sequences
, (sys
.stderr
, sys
.stdout
)), None)
545 self
._out
_files
['screen'] = sys
.stderr
if self
.params
.get('quiet') else self
._out
_files
['print']
546 self
._allow
_colors
= {
547 type_
: not self
.params
.get('no_color') and supports_terminal_sequences(self
._out
_files
[type_
])
548 for type_
in ('screen', 'error')
551 if sys
.version_info
< (3, 6):
553 'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys
.version_info
[:2])
555 if self
.params
.get('allow_unplayable_formats'):
557 f
'You have asked for {self._format_err("UNPLAYABLE", self.Styles.EMPHASIS)} formats to be listed/downloaded. '
558 'This is a developer option intended for debugging. \n'
559 ' If you experience any issues while using this option, '
560 f
'{self._format_err("DO NOT", self.Styles.ERROR)} open a bug report')
562 def check_deprecated(param
, option
, suggestion
):
563 if self
.params
.get(param
) is not None:
564 self
.report_warning(f
'{option} is deprecated. Use {suggestion} instead')
568 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
569 if self
.params
.get('geo_verification_proxy') is None:
570 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
572 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
573 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
574 check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
576 for msg
in self
.params
.get('_warnings', []):
577 self
.report_warning(msg
)
578 for msg
in self
.params
.get('_deprecation_warnings', []):
579 self
.deprecation_warning(msg
)
581 if 'list-formats' in self
.params
.get('compat_opts', []):
582 self
.params
['listformats_table'] = False
584 if 'overwrites' not in self
.params
and self
.params
.get('nooverwrites') is not None:
585 # nooverwrites was unnecessarily changed to overwrites
586 # in 0c3d0f51778b153f65c21906031c2e091fcfb641
587 # This ensures compatibility with both keys
588 self
.params
['overwrites'] = not self
.params
['nooverwrites']
589 elif self
.params
.get('overwrites') is None:
590 self
.params
.pop('overwrites', None)
592 self
.params
['nooverwrites'] = not self
.params
['overwrites']
594 self
.params
.setdefault('forceprint', {})
595 self
.params
.setdefault('print_to_file', {})
597 # Compatibility with older syntax
598 if not isinstance(params
['forceprint'], dict):
599 self
.params
['forceprint'] = {'video': params['forceprint']}
601 if self
.params
.get('bidi_workaround', False):
604 master
, slave
= pty
.openpty()
605 width
= compat_get_terminal_size().columns
609 width_args
= ['-w', str(width
)]
611 stdin
=subprocess
.PIPE
,
613 stderr
=self
._out
_files
['error'])
615 self
._output
_process
= Popen(['bidiv'] + width_args
, **sp_kwargs
)
617 self
._output
_process
= Popen(['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
618 self
._output
_channel
= os
.fdopen(master
, 'rb')
619 except OSError as ose
:
620 if ose
.errno
== errno
.ENOENT
:
622 'Could not find fribidi executable, ignoring --bidi-workaround. '
623 'Make sure that fribidi is an executable file in one of the directories in your $PATH.')
628 if auto_init
!= 'no_verbose_header':
629 self
.print_debug_header()
630 self
.add_default_info_extractors()
632 if (sys
.platform
!= 'win32'
633 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
634 and not self
.params
.get('restrictfilenames', False)):
635 # Unicode filesystem API will throw errors (#1474, #13027)
637 'Assuming --restrict-filenames since file system encoding '
638 'cannot encode all characters. '
639 'Set the LC_ALL environment variable to fix this.')
640 self
.params
['restrictfilenames'] = True
642 self
.outtmpl_dict
= self
.parse_outtmpl()
644 # Creating format selector here allows us to catch syntax errors before the extraction
645 self
.format_selector
= (
646 self
.params
.get('format') if self
.params
.get('format') in (None, '-')
647 else self
.params
['format'] if callable(self
.params
['format'])
648 else self
.build_format_selector(self
.params
['format']))
650 # Set http_headers defaults according to std_headers
651 self
.params
['http_headers'] = merge_headers(std_headers
, self
.params
.get('http_headers', {}))
654 'post_hooks': self
.add_post_hook
,
655 'progress_hooks': self
.add_progress_hook
,
656 'postprocessor_hooks': self
.add_postprocessor_hook
,
658 for opt
, fn
in hooks
.items():
659 for ph
in self
.params
.get(opt
, []):
662 for pp_def_raw
in self
.params
.get('postprocessors', []):
663 pp_def
= dict(pp_def_raw
)
664 when
= pp_def
.pop('when', 'post_process')
665 self
.add_post_processor(
666 get_postprocessor(pp_def
.pop('key'))(self
, **pp_def
),
670 register_socks_protocols()
672 def preload_download_archive(fn
):
673 """Preload the archive, if any is specified"""
676 self
.write_debug(f
'Loading archive file {fn!r}')
678 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
679 for line
in archive_file
:
680 self
.archive
.add(line
.strip())
681 except OSError as ioe
:
682 if ioe
.errno
!= errno
.ENOENT
:
688 preload_download_archive(self
.params
.get('download_archive'))
690 def warn_if_short_id(self
, argv
):
691 # short YouTube ID starting with dash?
693 i
for i
, a
in enumerate(argv
)
694 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
698 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
699 + ['--'] + [argv
[i
] for i
in idxs
]
702 'Long argument string detected. '
703 'Use -- to separate parameters and URLs, like this:\n%s' %
704 args_to_str(correct_argv
))
706 def add_info_extractor(self
, ie
):
707 """Add an InfoExtractor object to the end of the list."""
709 self
._ies
[ie_key
] = ie
710 if not isinstance(ie
, type):
711 self
._ies
_instances
[ie_key
] = ie
712 ie
.set_downloader(self
)
714 def _get_info_extractor_class(self
, ie_key
):
715 ie
= self
._ies
.get(ie_key
)
717 ie
= get_info_extractor(ie_key
)
718 self
.add_info_extractor(ie
)
721 def get_info_extractor(self
, ie_key
):
723 Get an instance of an IE with name ie_key, it will try to get one from
724 the _ies list, if there's no instance it will create a new one and add
725 it to the extractor list.
727 ie
= self
._ies
_instances
.get(ie_key
)
729 ie
= get_info_extractor(ie_key
)()
730 self
.add_info_extractor(ie
)
733 def add_default_info_extractors(self
):
735 Add the InfoExtractors returned by gen_extractors to the end of the list
737 for ie
in gen_extractor_classes():
738 self
.add_info_extractor(ie
)
740 def add_post_processor(self
, pp
, when
='post_process'):
741 """Add a PostProcessor object to the end of the chain."""
742 self
._pps
[when
].append(pp
)
743 pp
.set_downloader(self
)
745 def add_post_hook(self
, ph
):
746 """Add the post hook"""
747 self
._post
_hooks
.append(ph
)
749 def add_progress_hook(self
, ph
):
750 """Add the download progress hook"""
751 self
._progress
_hooks
.append(ph
)
753 def add_postprocessor_hook(self
, ph
):
754 """Add the postprocessing progress hook"""
755 self
._postprocessor
_hooks
.append(ph
)
756 for pps
in self
._pps
.values():
758 pp
.add_progress_hook(ph
)
760 def _bidi_workaround(self
, message
):
761 if not hasattr(self
, '_output_channel'):
764 assert hasattr(self
, '_output_process')
765 assert isinstance(message
, compat_str
)
766 line_count
= message
.count('\n') + 1
767 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
768 self
._output
_process
.stdin
.flush()
769 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
770 for _
in range(line_count
))
771 return res
[:-len('\n')]
773 def _write_string(self
, message
, out
=None, only_once
=False):
775 if message
in self
._printed
_messages
:
777 self
._printed
_messages
.add(message
)
778 write_string(message
, out
=out
, encoding
=self
.params
.get('encoding'))
780 def to_stdout(self
, message
, skip_eol
=False, quiet
=None):
781 """Print message to stdout"""
782 if quiet
is not None:
783 self
.deprecation_warning('"YoutubeDL.to_stdout" no longer accepts the argument quiet. Use "YoutubeDL.to_screen" instead')
785 '%s%s' % (self
._bidi
_workaround
(message
), ('' if skip_eol
else '\n')),
786 self
._out
_files
['print'])
788 def to_screen(self
, message
, skip_eol
=False, quiet
=None):
789 """Print message to screen if not in quiet mode"""
790 if self
.params
.get('logger'):
791 self
.params
['logger'].debug(message
)
793 if (self
.params
.get('quiet') if quiet
is None else quiet
) and not self
.params
.get('verbose'):
796 '%s%s' % (self
._bidi
_workaround
(message
), ('' if skip_eol
else '\n')),
797 self
._out
_files
['screen'])
799 def to_stderr(self
, message
, only_once
=False):
800 """Print message to stderr"""
801 assert isinstance(message
, compat_str
)
802 if self
.params
.get('logger'):
803 self
.params
['logger'].error(message
)
805 self
._write
_string
('%s\n' % self
._bidi
_workaround
(message
), self
._out
_files
['error'], only_once
=only_once
)
807 def _send_console_code(self
, code
):
808 if compat_os_name
== 'nt' or not self
._out
_files
['console']:
810 self
._write
_string
(code
, self
._out
_files
['console'])
812 def to_console_title(self
, message
):
813 if not self
.params
.get('consoletitle', False):
815 message
= remove_terminal_sequences(message
)
816 if compat_os_name
== 'nt':
817 if ctypes
.windll
.kernel32
.GetConsoleWindow():
818 # c_wchar_p() might not be necessary if `message` is
819 # already of type unicode()
820 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
822 self
._send
_console
_code
(f
'\033]0;{message}\007')
824 def save_console_title(self
):
825 if not self
.params
.get('consoletitle') or self
.params
.get('simulate'):
827 self
._send
_console
_code
('\033[22;0t') # Save the title on stack
829 def restore_console_title(self
):
830 if not self
.params
.get('consoletitle') or self
.params
.get('simulate'):
832 self
._send
_console
_code
('\033[23;0t') # Restore the title from stack
835 self
.save_console_title()
838 def __exit__(self
, *args
):
839 self
.restore_console_title()
841 if self
.params
.get('cookiefile') is not None:
842 self
.cookiejar
.save(ignore_discard
=True, ignore_expires
=True)
844 def trouble(self
, message
=None, tb
=None, is_error
=True):
845 """Determine action to take when a download problem appears.
847 Depending on if the downloader has been configured to ignore
848 download errors or not, this method may throw an exception or
849 not when errors are found, after printing the message.
851 @param tb If given, is additional traceback information
852 @param is_error Whether to raise error according to ignorerrors
854 if message
is not None:
855 self
.to_stderr(message
)
856 if self
.params
.get('verbose'):
858 if sys
.exc_info()[0]: # if .trouble has been called from an except block
860 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
861 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
862 tb
+= encode_compat_str(traceback
.format_exc())
864 tb_data
= traceback
.format_list(traceback
.extract_stack())
865 tb
= ''.join(tb_data
)
870 if not self
.params
.get('ignoreerrors'):
871 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
872 exc_info
= sys
.exc_info()[1].exc_info
874 exc_info
= sys
.exc_info()
875 raise DownloadError(message
, exc_info
)
876 self
._download
_retcode
= 1
880 EMPHASIS
='light blue',
885 SUPPRESS
='light black',
888 def _format_text(self
, handle
, allow_colors
, text
, f
, fallback
=None, *, test_encoding
=False):
892 # handle.encoding can be None. See https://github.com/yt-dlp/yt-dlp/issues/2711
893 encoding
= self
.params
.get('encoding') or getattr(handle
, 'encoding', None) or 'ascii'
894 text
= text
.encode(encoding
, 'ignore').decode(encoding
)
895 if fallback
is not None and text
!= original_text
:
897 return format_text(text
, f
) if allow_colors
else text
if fallback
is None else fallback
899 def _format_screen(self
, *args
, **kwargs
):
900 return self
._format
_text
(
901 self
._out
_files
['screen'], self
._allow
_colors
['screen'], *args
, **kwargs
)
903 def _format_err(self
, *args
, **kwargs
):
904 return self
._format
_text
(
905 self
._out
_files
['error'], self
._allow
_colors
['error'], *args
, **kwargs
)
907 def report_warning(self
, message
, only_once
=False):
909 Print the message to stderr, it will be prefixed with 'WARNING:'
910 If stderr is a tty file the 'WARNING:' will be colored
912 if self
.params
.get('logger') is not None:
913 self
.params
['logger'].warning(message
)
915 if self
.params
.get('no_warnings'):
917 self
.to_stderr(f
'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once
)
919 def deprecation_warning(self
, message
):
920 if self
.params
.get('logger') is not None:
921 self
.params
['logger'].warning(f
'DeprecationWarning: {message}')
923 self
.to_stderr(f
'{self._format_err("DeprecationWarning:", self.Styles.ERROR)} {message}', True)
925 def report_error(self
, message
, *args
, **kwargs
):
927 Do the same as trouble, but prefixes the message with 'ERROR:', colored
928 in red if stderr is a tty file.
930 self
.trouble(f
'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args
, **kwargs
)
932 def write_debug(self
, message
, only_once
=False):
933 '''Log debug message or Print message to stderr'''
934 if not self
.params
.get('verbose', False):
936 message
= '[debug] %s' % message
937 if self
.params
.get('logger'):
938 self
.params
['logger'].debug(message
)
940 self
.to_stderr(message
, only_once
)
942 def report_file_already_downloaded(self
, file_name
):
943 """Report file has already been fully downloaded."""
945 self
.to_screen('[download] %s has already been downloaded' % file_name
)
946 except UnicodeEncodeError:
947 self
.to_screen('[download] The file has already been downloaded')
949 def report_file_delete(self
, file_name
):
950 """Report that existing file will be deleted."""
952 self
.to_screen('Deleting existing file %s' % file_name
)
953 except UnicodeEncodeError:
954 self
.to_screen('Deleting existing file')
956 def raise_no_formats(self
, info
, forced
=False, *, msg
=None):
957 has_drm
= info
.get('__has_drm')
958 ignored
, expected
= self
.params
.get('ignore_no_formats_error'), bool(msg
)
959 msg
= msg
or has_drm
and 'This video is DRM protected' or 'No video formats found!'
960 if forced
or not ignored
:
961 raise ExtractorError(msg
, video_id
=info
['id'], ie
=info
['extractor'],
962 expected
=has_drm
or ignored
or expected
)
964 self
.report_warning(msg
)
966 def parse_outtmpl(self
):
967 outtmpl_dict
= self
.params
.get('outtmpl', {})
968 if not isinstance(outtmpl_dict
, dict):
969 outtmpl_dict
= {'default': outtmpl_dict}
970 # Remove spaces in the default template
971 if self
.params
.get('restrictfilenames'):
972 sanitize
= lambda x
: x
.replace(' - ', ' ').replace(' ', '-')
974 sanitize
= lambda x
: x
975 outtmpl_dict
.update({
976 k
: sanitize(v
) for k
, v
in DEFAULT_OUTTMPL
.items()
977 if outtmpl_dict
.get(k
) is None})
978 for _
, val
in outtmpl_dict
.items():
979 if isinstance(val
, bytes):
980 self
.report_warning('Parameter outtmpl is bytes, but should be a unicode string')
983 def get_output_path(self
, dir_type
='', filename
=None):
984 paths
= self
.params
.get('paths', {})
985 assert isinstance(paths
, dict)
987 expand_path(paths
.get('home', '').strip()),
988 expand_path(paths
.get(dir_type
, '').strip()) if dir_type
else '',
990 return sanitize_path(path
, force
=self
.params
.get('windowsfilenames'))
993 def _outtmpl_expandpath(outtmpl
):
994 # expand_path translates '%%' into '%' and '$$' into '$'
995 # correspondingly that is not what we want since we need to keep
996 # '%%' intact for template dict substitution step. Working around
997 # with boundary-alike separator hack.
998 sep
= ''.join([random
.choice(ascii_letters
) for _
in range(32)])
999 outtmpl
= outtmpl
.replace('%%', f
'%{sep}%').replace('$$', f
'${sep}$')
1001 # outtmpl should be expand_path'ed before template dict substitution
1002 # because meta fields may contain env variables we don't want to
1003 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
1004 # title "Hello $PATH", we don't want `$PATH` to be expanded.
1005 return expand_path(outtmpl
).replace(sep
, '')
1008 def escape_outtmpl(outtmpl
):
1009 ''' Escape any remaining strings like %s, %abc% etc. '''
1011 STR_FORMAT_RE_TMPL
.format('', '(?![%(\0])'),
1012 lambda mobj
: ('' if mobj
.group('has_key') else '%') + mobj
.group(0),
1016 def validate_outtmpl(cls
, outtmpl
):
1017 ''' @return None or Exception object '''
1019 STR_FORMAT_RE_TMPL
.format('[^)]*', '[ljqBUDS]'),
1020 lambda mobj
: f
'{mobj.group(0)[:-1]}s',
1021 cls
._outtmpl
_expandpath
(outtmpl
))
1023 cls
.escape_outtmpl(outtmpl
) % collections
.defaultdict(int)
1025 except ValueError as err
:
1029 def _copy_infodict(info_dict
):
1030 info_dict
= dict(info_dict
)
1031 info_dict
.pop('__postprocessors', None)
1034 def prepare_outtmpl(self
, outtmpl
, info_dict
, sanitize
=False):
1035 """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict
1036 @param sanitize Whether to sanitize the output as a filename.
1037 For backward compatibility, a function can also be passed
1040 info_dict
.setdefault('epoch', int(time
.time())) # keep epoch consistent once set
1042 info_dict
= self
._copy
_infodict
(info_dict
)
1043 info_dict
['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
1044 formatSeconds(info_dict
['duration'], '-' if sanitize
else ':')
1045 if info_dict
.get('duration', None) is not None
1047 info_dict
['autonumber'] = self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
1048 info_dict
['video_autonumber'] = self
._num
_videos
1049 if info_dict
.get('resolution') is None:
1050 info_dict
['resolution'] = self
.format_resolution(info_dict
, default
=None)
1052 # For fields playlist_index, playlist_autonumber and autonumber convert all occurrences
1053 # of %(field)s to %(field)0Nd for backward compatibility
1054 field_size_compat_map
= {
1055 'playlist_index': number_of_digits(info_dict
.get('_last_playlist_index') or 0),
1056 'playlist_autonumber': number_of_digits(info_dict
.get('n_entries') or 0),
1057 'autonumber': self
.params
.get('autonumber_size') or 5,
1061 EXTERNAL_FORMAT_RE
= re
.compile(STR_FORMAT_RE_TMPL
.format('[^)]*', f
'[{STR_FORMAT_TYPES}ljqBUDS]'))
1066 # Field is of the form key1.key2...
1067 # where keys (except first) can be string, int or slice
1068 FIELD_RE
= r
'\w*(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num
=r
'(?:-?\d+)')
1069 MATH_FIELD_RE
= r
'''(?:{field}|{num})'''.format(field
=FIELD_RE
, num
=r
'-?\d+(?:.\d+)?')
1070 MATH_OPERATORS_RE
= r
'(?:%s)' % '|'.join(map(re
.escape
, MATH_FUNCTIONS
.keys()))
1071 INTERNAL_FORMAT_RE
= re
.compile(r
'''(?x)
1074 (?P<maths>(?:{math_op}{math_field})*)
1075 (?:>(?P<strf_format>.+?))?
1077 (?P<alternate>(?<!\\),[^|&)]+)?
1078 (?:&(?P<replacement>.*?))?
1079 (?:\|(?P<default>.*?))?
1080 )$'''.format(field
=FIELD_RE
, math_op
=MATH_OPERATORS_RE
, math_field
=MATH_FIELD_RE
))
1082 def _traverse_infodict(k
):
1086 return traverse_obj(info_dict
, k
, is_user_input
=True, traverse_string
=True)
1088 def get_value(mdict
):
1090 value
= _traverse_infodict(mdict
['fields'])
1093 value
= float_or_none(value
)
1094 if value
is not None:
1097 offset_key
= mdict
['maths']
1099 value
= float_or_none(value
)
1103 MATH_FIELD_RE
if operator
else MATH_OPERATORS_RE
,
1104 offset_key
).group(0)
1105 offset_key
= offset_key
[len(item
):]
1106 if operator
is None:
1107 operator
= MATH_FUNCTIONS
[item
]
1109 item
, multiplier
= (item
[1:], -1) if item
[0] == '-' else (item
, 1)
1110 offset
= float_or_none(item
)
1112 offset
= float_or_none(_traverse_infodict(item
))
1114 value
= operator(value
, multiplier
* offset
)
1115 except (TypeError, ZeroDivisionError):
1118 # Datetime formatting
1119 if mdict
['strf_format']:
1120 value
= strftime_or_none(value
, mdict
['strf_format'].replace('\\,', ','))
1124 na
= self
.params
.get('outtmpl_na_placeholder', 'NA')
1126 def filename_sanitizer(key
, value
, restricted
=self
.params
.get('restrictfilenames')):
1127 return sanitize_filename(str(value
), restricted
=restricted
, is_id
=(
1128 bool(re
.search(r
'(^|[_.])id(\.|$)', key
))
1129 if 'filename-sanitization' in self
.params
.get('compat_opts', [])
1132 sanitizer
= sanitize
if callable(sanitize
) else filename_sanitizer
1133 sanitize
= bool(sanitize
)
1135 def _dumpjson_default(obj
):
1136 if isinstance(obj
, (set, LazyList
)):
1140 def create_key(outer_mobj
):
1141 if not outer_mobj
.group('has_key'):
1142 return outer_mobj
.group(0)
1143 key
= outer_mobj
.group('key')
1144 mobj
= re
.match(INTERNAL_FORMAT_RE
, key
)
1145 initial_field
= mobj
.group('fields') if mobj
else ''
1146 value
, replacement
, default
= None, None, na
1148 mobj
= mobj
.groupdict()
1149 default
= mobj
['default'] if mobj
['default'] is not None else default
1150 value
= get_value(mobj
)
1151 replacement
= mobj
['replacement']
1152 if value
is None and mobj
['alternate']:
1153 mobj
= re
.match(INTERNAL_FORMAT_RE
, mobj
['remaining'][1:])
1157 fmt
= outer_mobj
.group('format')
1158 if fmt
== 's' and value
is not None and key
in field_size_compat_map
.keys():
1159 fmt
= f
'0{field_size_compat_map[key]:d}d'
1161 value
= default
if value
is None else value
if replacement
is None else replacement
1163 flags
= outer_mobj
.group('conversion') or ''
1164 str_fmt
= f
'{fmt[:-1]}s'
1165 if fmt
[-1] == 'l': # list
1166 delim
= '\n' if '#' in flags
else ', '
1167 value
, fmt
= delim
.join(map(str, variadic(value
, allowed_types
=(str, bytes)))), str_fmt
1168 elif fmt
[-1] == 'j': # json
1169 value
, fmt
= json
.dumps(value
, default
=_dumpjson_default
, indent
=4 if '#' in flags
else None), str_fmt
1170 elif fmt
[-1] == 'q': # quoted
1171 value
= map(str, variadic(value
) if '#' in flags
else [value
])
1172 value
, fmt
= ' '.join(map(compat_shlex_quote
, value
)), str_fmt
1173 elif fmt
[-1] == 'B': # bytes
1174 value
= f
'%{str_fmt}'.encode() % str(value
).encode('utf-8')
1175 value
, fmt
= value
.decode('utf-8', 'ignore'), 's'
1176 elif fmt
[-1] == 'U': # unicode normalized
1177 value
, fmt
= unicodedata
.normalize(
1178 # "+" = compatibility equivalence, "#" = NFD
1179 'NF%s%s' % ('K' if '+' in flags
else '', 'D' if '#' in flags
else 'C'),
1181 elif fmt
[-1] == 'D': # decimal suffix
1182 num_fmt
, fmt
= fmt
[:-1].replace('#', ''), 's'
1183 value
= format_decimal_suffix(value
, f
'%{num_fmt}f%s' if num_fmt
else '%d%s',
1184 factor
=1024 if '#' in flags
else 1000)
1185 elif fmt
[-1] == 'S': # filename sanitization
1186 value
, fmt
= filename_sanitizer(initial_field
, value
, restricted
='#' in flags
), str_fmt
1187 elif fmt
[-1] == 'c':
1189 value
= str(value
)[0]
1192 elif fmt
[-1] not in 'rs': # numeric
1193 value
= float_or_none(value
)
1195 value
, fmt
= default
, 's'
1199 # If value is an object, sanitize might convert it to a string
1200 # So we convert it to repr first
1201 value
, fmt
= repr(value
), str_fmt
1202 if fmt
[-1] in 'csr':
1203 value
= sanitizer(initial_field
, value
)
1205 key
= '%s\0%s' % (key
.replace('%', '%\0'), outer_mobj
.group('format'))
1206 TMPL_DICT
[key
] = value
1207 return '{prefix}%({key}){fmt}'.format(key
=key
, fmt
=fmt
, prefix
=outer_mobj
.group('prefix'))
1209 return EXTERNAL_FORMAT_RE
.sub(create_key
, outtmpl
), TMPL_DICT
1211 def evaluate_outtmpl(self
, outtmpl
, info_dict
, *args
, **kwargs
):
1212 outtmpl
, info_dict
= self
.prepare_outtmpl(outtmpl
, info_dict
, *args
, **kwargs
)
1213 return self
.escape_outtmpl(outtmpl
) % info_dict
1215 def _prepare_filename(self
, info_dict
, *, outtmpl
=None, tmpl_type
=None):
1216 assert None in (outtmpl
, tmpl_type
), 'outtmpl and tmpl_type are mutually exclusive'
1218 outtmpl
= self
.outtmpl_dict
.get(tmpl_type
or 'default', self
.outtmpl_dict
['default'])
1220 outtmpl
= self
._outtmpl
_expandpath
(outtmpl
)
1221 filename
= self
.evaluate_outtmpl(outtmpl
, info_dict
, True)
1225 if tmpl_type
in ('', 'temp'):
1226 final_ext
, ext
= self
.params
.get('final_ext'), info_dict
.get('ext')
1227 if final_ext
and ext
and final_ext
!= ext
and filename
.endswith(f
'.{final_ext}'):
1228 filename
= replace_extension(filename
, ext
, final_ext
)
1230 force_ext
= OUTTMPL_TYPES
[tmpl_type
]
1232 filename
= replace_extension(filename
, force_ext
, info_dict
.get('ext'))
1234 # https://github.com/blackjack4494/youtube-dlc/issues/85
1235 trim_file_name
= self
.params
.get('trim_file_name', False)
1237 no_ext
, *ext
= filename
.rsplit('.', 2)
1238 filename
= join_nonempty(no_ext
[:trim_file_name
], *ext
, delim
='.')
1241 except ValueError as err
:
1242 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
1245 def prepare_filename(self
, info_dict
, dir_type
='', *, outtmpl
=None, warn
=False):
1246 """Generate the output filename"""
1248 assert not dir_type
, 'outtmpl and dir_type are mutually exclusive'
1250 filename
= self
._prepare
_filename
(info_dict
, tmpl_type
=dir_type
, outtmpl
=outtmpl
)
1251 if not filename
and dir_type
not in ('', 'temp'):
1255 if not self
.params
.get('paths'):
1257 elif filename
== '-':
1258 self
.report_warning('--paths is ignored when an outputting to stdout', only_once
=True)
1259 elif os
.path
.isabs(filename
):
1260 self
.report_warning('--paths is ignored since an absolute path is given in output template', only_once
=True)
1261 if filename
== '-' or not filename
:
1264 return self
.get_output_path(dir_type
, filename
)
1266 def _match_entry(self
, info_dict
, incomplete
=False, silent
=False):
1267 """ Returns None if the file should be downloaded """
1269 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
1272 if 'title' in info_dict
:
1273 # This can happen when we're just evaluating the playlist
1274 title
= info_dict
['title']
1275 matchtitle
= self
.params
.get('matchtitle', False)
1277 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
1278 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
1279 rejecttitle
= self
.params
.get('rejecttitle', False)
1281 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
1282 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
1283 date
= info_dict
.get('upload_date')
1284 if date
is not None:
1285 dateRange
= self
.params
.get('daterange', DateRange())
1286 if date
not in dateRange
:
1287 return f
'{date_from_str(date).isoformat()} upload date is not in range {dateRange}'
1288 view_count
= info_dict
.get('view_count')
1289 if view_count
is not None:
1290 min_views
= self
.params
.get('min_views')
1291 if min_views
is not None and view_count
< min_views
:
1292 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
1293 max_views
= self
.params
.get('max_views')
1294 if max_views
is not None and view_count
> max_views
:
1295 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
1296 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
1297 return 'Skipping "%s" because it is age restricted' % video_title
1299 match_filter
= self
.params
.get('match_filter')
1300 if match_filter
is not None:
1302 ret
= match_filter(info_dict
, incomplete
=incomplete
)
1304 # For backward compatibility
1305 ret
= None if incomplete
else match_filter(info_dict
)
1310 if self
.in_download_archive(info_dict
):
1311 reason
= '%s has already been recorded in the archive' % video_title
1312 break_opt
, break_err
= 'break_on_existing', ExistingVideoReached
1314 reason
= check_filter()
1315 break_opt
, break_err
= 'break_on_reject', RejectedVideoReached
1316 if reason
is not None:
1318 self
.to_screen('[download] ' + reason
)
1319 if self
.params
.get(break_opt
, False):
1324 def add_extra_info(info_dict
, extra_info
):
1325 '''Set the keys from extra_info in info dict if they are missing'''
1326 for key
, value
in extra_info
.items():
1327 info_dict
.setdefault(key
, value
)
1329 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
=None,
1330 process
=True, force_generic_extractor
=False):
1332 Return a list with a dictionary for each video extracted.
1335 url -- URL to extract
1338 download -- whether to download videos during extraction
1339 ie_key -- extractor key hint
1340 extra_info -- dictionary containing the extra values to add to each result
1341 process -- whether to resolve all unresolved references (URLs, playlist items),
1342 must be True for download to work.
1343 force_generic_extractor -- force using the generic extractor
1346 if extra_info
is None:
1349 if not ie_key
and force_generic_extractor
:
1353 ies
= {ie_key: self._get_info_extractor_class(ie_key)}
1357 for ie_key
, ie
in ies
.items():
1358 if not ie
.suitable(url
):
1361 if not ie
.working():
1362 self
.report_warning('The program functionality for this site has been marked as broken, '
1363 'and will probably not work.')
1365 temp_id
= ie
.get_temp_id(url
)
1366 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': ie_key}
):
1367 self
.to_screen(f
'[{ie_key}] {temp_id}: has already been recorded in the archive')
1368 if self
.params
.get('break_on_existing', False):
1369 raise ExistingVideoReached()
1371 return self
.__extract
_info
(url
, self
.get_info_extractor(ie_key
), download
, extra_info
, process
)
1373 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
1375 def __handle_extraction_exceptions(func
):
1376 @functools.wraps(func
)
1377 def wrapper(self
, *args
, **kwargs
):
1380 return func(self
, *args
, **kwargs
)
1381 except (DownloadCancelled
, LazyList
.IndexError, PagedList
.IndexError):
1383 except ReExtractInfo
as e
:
1385 self
.to_screen(f
'{e}; Re-extracting data')
1387 self
.to_stderr('\r')
1388 self
.report_warning(f
'{e}; Re-extracting data')
1390 except GeoRestrictedError
as e
:
1393 msg
+= '\nThis video is available in %s.' % ', '.join(
1394 map(ISO3166Utils
.short2full
, e
.countries
))
1395 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
1396 self
.report_error(msg
)
1397 except ExtractorError
as e
: # An error we somewhat expected
1398 self
.report_error(str(e
), e
.format_traceback())
1399 except Exception as e
:
1400 if self
.params
.get('ignoreerrors'):
1401 self
.report_error(str(e
), tb
=encode_compat_str(traceback
.format_exc()))
1407 def _wait_for_video(self
, ie_result
):
1408 if (not self
.params
.get('wait_for_video')
1409 or ie_result
.get('_type', 'video') != 'video'
1410 or ie_result
.get('formats') or ie_result
.get('url')):
1413 format_dur
= lambda dur
: '%02d:%02d:%02d' % timetuple_from_msec(dur
* 1000)[:-1]
1418 self
.to_screen(msg
+ ' ' * (len(last_msg
) - len(msg
)) + '\r', skip_eol
=True)
1421 min_wait
, max_wait
= self
.params
.get('wait_for_video')
1422 diff
= try_get(ie_result
, lambda x
: x
['release_timestamp'] - time
.time())
1423 if diff
is None and ie_result
.get('live_status') == 'is_upcoming':
1424 diff
= round(random
.uniform(min_wait
, max_wait
) if (max_wait
and min_wait
) else (max_wait
or min_wait
), 0)
1425 self
.report_warning('Release time of video is not known')
1426 elif (diff
or 0) <= 0:
1427 self
.report_warning('Video should already be available according to extracted info')
1428 diff
= min(max(diff
or 0, min_wait
or 0), max_wait
or float('inf'))
1429 self
.to_screen(f
'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now')
1431 wait_till
= time
.time() + diff
1434 diff
= wait_till
- time
.time()
1437 raise ReExtractInfo('[wait] Wait period ended', expected
=True)
1438 progress(f
'[wait] Remaining time until next attempt: {self._format_screen(format_dur(diff), self.Styles.EMPHASIS)}')
1440 except KeyboardInterrupt:
1442 raise ReExtractInfo('[wait] Interrupted by user', expected
=True)
1443 except BaseException
as e
:
1444 if not isinstance(e
, ReExtractInfo
):
1448 @__handle_extraction_exceptions
1449 def __extract_info(self
, url
, ie
, download
, extra_info
, process
):
1450 ie_result
= ie
.extract(url
)
1451 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
1453 if isinstance(ie_result
, list):
1454 # Backwards compatibility: old IE result format
1456 '_type': 'compat_list',
1457 'entries': ie_result
,
1459 if extra_info
.get('original_url'):
1460 ie_result
.setdefault('original_url', extra_info
['original_url'])
1461 self
.add_default_extra_info(ie_result
, ie
, url
)
1463 self
._wait
_for
_video
(ie_result
)
1464 return self
.process_ie_result(ie_result
, download
, extra_info
)
1468 def add_default_extra_info(self
, ie_result
, ie
, url
):
1470 self
.add_extra_info(ie_result
, {
1472 'original_url': url
,
1474 webpage_url
= ie_result
.get('webpage_url')
1476 self
.add_extra_info(ie_result
, {
1477 'webpage_url_basename': url_basename(webpage_url
),
1478 'webpage_url_domain': get_domain(webpage_url
),
1481 self
.add_extra_info(ie_result
, {
1482 'extractor': ie
.IE_NAME
,
1483 'extractor_key': ie
.ie_key(),
1486 def process_ie_result(self
, ie_result
, download
=True, extra_info
=None):
1488 Take the result of the ie(may be modified) and resolve all unresolved
1489 references (URLs, playlist items).
1491 It will also download the videos if 'download'.
1492 Returns the resolved ie_result.
1494 if extra_info
is None:
1496 result_type
= ie_result
.get('_type', 'video')
1498 if result_type
in ('url', 'url_transparent'):
1499 ie_result
['url'] = sanitize_url(ie_result
['url'])
1500 if ie_result
.get('original_url'):
1501 extra_info
.setdefault('original_url', ie_result
['original_url'])
1503 extract_flat
= self
.params
.get('extract_flat', False)
1504 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
1505 or extract_flat
is True):
1506 info_copy
= ie_result
.copy()
1507 ie
= try_get(ie_result
.get('ie_key'), self
.get_info_extractor
)
1508 if ie
and not ie_result
.get('id'):
1509 info_copy
['id'] = ie
.get_temp_id(ie_result
['url'])
1510 self
.add_default_extra_info(info_copy
, ie
, ie_result
['url'])
1511 self
.add_extra_info(info_copy
, extra_info
)
1512 info_copy
, _
= self
.pre_process(info_copy
)
1513 self
.__forced
_printings
(info_copy
, self
.prepare_filename(info_copy
), incomplete
=True)
1514 if self
.params
.get('force_write_download_archive', False):
1515 self
.record_download_archive(info_copy
)
1518 if result_type
== 'video':
1519 self
.add_extra_info(ie_result
, extra_info
)
1520 ie_result
= self
.process_video_result(ie_result
, download
=download
)
1521 additional_urls
= (ie_result
or {}).get('additional_urls')
1523 # TODO: Improve MetadataParserPP to allow setting a list
1524 if isinstance(additional_urls
, compat_str
):
1525 additional_urls
= [additional_urls
]
1527 '[info] %s: %d additional URL(s) requested' % (ie_result
['id'], len(additional_urls
)))
1528 self
.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls
))
1529 ie_result
['additional_entries'] = [
1531 url
, download
, extra_info
=extra_info
,
1532 force_generic_extractor
=self
.params
.get('force_generic_extractor'))
1533 for url
in additional_urls
1536 elif result_type
== 'url':
1537 # We have to add extra_info to the results because it may be
1538 # contained in a playlist
1539 return self
.extract_info(
1540 ie_result
['url'], download
,
1541 ie_key
=ie_result
.get('ie_key'),
1542 extra_info
=extra_info
)
1543 elif result_type
== 'url_transparent':
1544 # Use the information from the embedding page
1545 info
= self
.extract_info(
1546 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
1547 extra_info
=extra_info
, download
=False, process
=False)
1549 # extract_info may return None when ignoreerrors is enabled and
1550 # extraction failed with an error, don't crash and return early
1555 new_result
= info
.copy()
1556 new_result
.update(filter_dict(ie_result
, lambda k
, v
: (
1557 v
is not None and k
not in {'_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'}
)))
1559 # Extracted info may not be a video result (i.e.
1560 # info.get('_type', 'video') != video) but rather an url or
1561 # url_transparent. In such cases outer metadata (from ie_result)
1562 # should be propagated to inner one (info). For this to happen
1563 # _type of info should be overridden with url_transparent. This
1564 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
1565 if new_result
.get('_type') == 'url':
1566 new_result
['_type'] = 'url_transparent'
1568 return self
.process_ie_result(
1569 new_result
, download
=download
, extra_info
=extra_info
)
1570 elif result_type
in ('playlist', 'multi_video'):
1571 # Protect from infinite recursion due to recursively nested playlists
1572 # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
1573 webpage_url
= ie_result
['webpage_url']
1574 if webpage_url
in self
._playlist
_urls
:
1576 '[download] Skipping already downloaded playlist: %s'
1577 % ie_result
.get('title') or ie_result
.get('id'))
1580 self
._playlist
_level
+= 1
1581 self
._playlist
_urls
.add(webpage_url
)
1582 self
._fill
_common
_fields
(ie_result
, False)
1583 self
._sanitize
_thumbnails
(ie_result
)
1585 return self
.__process
_playlist
(ie_result
, download
)
1587 self
._playlist
_level
-= 1
1588 if not self
._playlist
_level
:
1589 self
._playlist
_urls
.clear()
1590 elif result_type
== 'compat_list':
1591 self
.report_warning(
1592 'Extractor %s returned a compat_list result. '
1593 'It needs to be updated.' % ie_result
.get('extractor'))
1596 self
.add_extra_info(r
, {
1597 'extractor': ie_result
['extractor'],
1598 'webpage_url': ie_result
['webpage_url'],
1599 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1600 'webpage_url_domain': get_domain(ie_result
['webpage_url']),
1601 'extractor_key': ie_result
['extractor_key'],
1604 ie_result
['entries'] = [
1605 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1606 for r
in ie_result
['entries']
1610 raise Exception('Invalid result type: %s' % result_type
)
1612 def _ensure_dir_exists(self
, path
):
1613 return make_dir(path
, self
.report_error
)
1616 def _playlist_infodict(ie_result
, **kwargs
):
1619 'playlist': ie_result
.get('title') or ie_result
.get('id'),
1620 'playlist_id': ie_result
.get('id'),
1621 'playlist_title': ie_result
.get('title'),
1622 'playlist_uploader': ie_result
.get('uploader'),
1623 'playlist_uploader_id': ie_result
.get('uploader_id'),
1624 'playlist_index': 0,
1628 def __process_playlist(self
, ie_result
, download
):
1629 # We process each entry in the playlist
1630 playlist
= ie_result
.get('title') or ie_result
.get('id')
1631 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
1633 if 'entries' not in ie_result
:
1634 raise EntryNotInPlaylist('There are no entries')
1636 MissingEntry
= object()
1637 incomplete_entries
= bool(ie_result
.get('requested_entries'))
1638 if incomplete_entries
:
1639 def fill_missing_entries(entries
, indices
):
1640 ret
= [MissingEntry
] * max(indices
)
1641 for i
, entry
in zip(indices
, entries
):
1644 ie_result
['entries'] = fill_missing_entries(ie_result
['entries'], ie_result
['requested_entries'])
1646 playlist_results
= []
1648 playliststart
= self
.params
.get('playliststart', 1)
1649 playlistend
= self
.params
.get('playlistend')
1650 # For backwards compatibility, interpret -1 as whole list
1651 if playlistend
== -1:
1654 playlistitems_str
= self
.params
.get('playlist_items')
1655 playlistitems
= None
1656 if playlistitems_str
is not None:
1657 def iter_playlistitems(format
):
1658 for string_segment
in format
.split(','):
1659 if '-' in string_segment
:
1660 start
, end
= string_segment
.split('-')
1661 for item
in range(int(start
), int(end
) + 1):
1664 yield int(string_segment
)
1665 playlistitems
= orderedSet(iter_playlistitems(playlistitems_str
))
1667 ie_entries
= ie_result
['entries']
1668 if isinstance(ie_entries
, list):
1669 playlist_count
= len(ie_entries
)
1670 msg
= f
'Collected {playlist_count} videos; downloading %d of them'
1671 ie_result
['playlist_count'] = ie_result
.get('playlist_count') or playlist_count
1674 return ie_entries
[i
- 1]
1676 msg
= 'Downloading %d videos'
1677 if not isinstance(ie_entries
, (PagedList
, LazyList
)):
1678 ie_entries
= LazyList(ie_entries
)
1679 elif isinstance(ie_entries
, InAdvancePagedList
):
1680 if ie_entries
._pagesize
== 1:
1681 playlist_count
= ie_entries
._pagecount
1684 return YoutubeDL
.__handle
_extraction
_exceptions
(
1685 lambda self
, i
: ie_entries
[i
- 1]
1688 entries
, broken
= [], False
1689 items
= playlistitems
if playlistitems
is not None else itertools
.count(playliststart
)
1693 if playlistitems
is None and playlistend
is not None and playlistend
< i
:
1697 entry
= get_entry(i
)
1698 if entry
is MissingEntry
:
1699 raise EntryNotInPlaylist()
1700 except (IndexError, EntryNotInPlaylist
):
1701 if incomplete_entries
:
1702 raise EntryNotInPlaylist(f
'Entry {i} cannot be found')
1703 elif not playlistitems
:
1705 entries
.append(entry
)
1707 if entry
is not None:
1708 # TODO: Add auto-generated fields
1709 self
._match
_entry
(entry
, incomplete
=True, silent
=True)
1710 except (ExistingVideoReached
, RejectedVideoReached
):
1713 ie_result
['entries'] = entries
1715 # Save playlist_index before re-ordering
1717 ((playlistitems
[i
- 1] if playlistitems
else i
+ playliststart
- 1), entry
)
1718 for i
, entry
in enumerate(entries
, 1)
1719 if entry
is not None]
1720 n_entries
= len(entries
)
1722 if not (ie_result
.get('playlist_count') or broken
or playlistitems
or playlistend
):
1723 ie_result
['playlist_count'] = n_entries
1725 if not playlistitems
and (playliststart
!= 1 or playlistend
):
1726 playlistitems
= list(range(playliststart
, playliststart
+ n_entries
))
1727 ie_result
['requested_entries'] = playlistitems
1729 _infojson_written
= False
1730 write_playlist_files
= self
.params
.get('allow_playlist_files', True)
1731 if write_playlist_files
and self
.params
.get('list_thumbnails'):
1732 self
.list_thumbnails(ie_result
)
1733 if write_playlist_files
and not self
.params
.get('simulate'):
1734 ie_copy
= self
._playlist
_infodict
(ie_result
, n_entries
=n_entries
)
1735 _infojson_written
= self
._write
_info
_json
(
1736 'playlist', ie_result
, self
.prepare_filename(ie_copy
, 'pl_infojson'))
1737 if _infojson_written
is None:
1739 if self
._write
_description
('playlist', ie_result
,
1740 self
.prepare_filename(ie_copy
, 'pl_description')) is None:
1742 # TODO: This should be passed to ThumbnailsConvertor if necessary
1743 self
._write
_thumbnails
('playlist', ie_copy
, self
.prepare_filename(ie_copy
, 'pl_thumbnail'))
1745 if self
.params
.get('playlistreverse', False):
1746 entries
= entries
[::-1]
1747 if self
.params
.get('playlistrandom', False):
1748 random
.shuffle(entries
)
1750 x_forwarded_for
= ie_result
.get('__x_forwarded_for_ip')
1752 self
.to_screen(f
'[{ie_result["extractor"]}] playlist {playlist}: {msg % n_entries}')
1754 max_failures
= self
.params
.get('skip_playlist_after_errors') or float('inf')
1755 for i
, entry_tuple
in enumerate(entries
, 1):
1756 playlist_index
, entry
= entry_tuple
1757 if 'playlist-index' in self
.params
.get('compat_opts', []):
1758 playlist_index
= playlistitems
[i
- 1] if playlistitems
else i
+ playliststart
- 1
1759 self
.to_screen('[download] Downloading video %s of %s' % (
1760 self
._format
_screen
(i
, self
.Styles
.ID
), self
._format
_screen
(n_entries
, self
.Styles
.EMPHASIS
)))
1761 # This __x_forwarded_for_ip thing is a bit ugly but requires
1764 entry
['__x_forwarded_for_ip'] = x_forwarded_for
1766 'n_entries': n_entries
,
1767 '_last_playlist_index': max(playlistitems
) if playlistitems
else (playlistend
or n_entries
),
1768 'playlist_count': ie_result
.get('playlist_count'),
1769 'playlist_index': playlist_index
,
1770 'playlist_autonumber': i
,
1771 'playlist': playlist
,
1772 'playlist_id': ie_result
.get('id'),
1773 'playlist_title': ie_result
.get('title'),
1774 'playlist_uploader': ie_result
.get('uploader'),
1775 'playlist_uploader_id': ie_result
.get('uploader_id'),
1776 'extractor': ie_result
['extractor'],
1777 'webpage_url': ie_result
['webpage_url'],
1778 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1779 'webpage_url_domain': get_domain(ie_result
['webpage_url']),
1780 'extractor_key': ie_result
['extractor_key'],
1783 if self
._match
_entry
(entry
, incomplete
=True) is not None:
1786 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, extra
)
1787 if not entry_result
:
1789 if failures
>= max_failures
:
1791 'Skipping the remaining entries in playlist "%s" since %d items failed extraction' % (playlist
, failures
))
1793 playlist_results
.append(entry_result
)
1794 ie_result
['entries'] = playlist_results
1796 # Write the updated info to json
1797 if _infojson_written
is True and self
._write
_info
_json
(
1798 'updated playlist', ie_result
,
1799 self
.prepare_filename(ie_copy
, 'pl_infojson'), overwrite
=True) is None:
1802 ie_result
= self
.run_all_pps('playlist', ie_result
)
1803 self
.to_screen(f
'[download] Finished downloading playlist: {playlist}')
1806 @__handle_extraction_exceptions
1807 def __process_iterable_entry(self
, entry
, download
, extra_info
):
1808 return self
.process_ie_result(
1809 entry
, download
=download
, extra_info
=extra_info
)
1811 def _build_format_filter(self
, filter_spec
):
1812 " Returns a function to filter the formats according to the filter_spec "
1822 operator_rex
= re
.compile(r
'''(?x)\s*
1823 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)\s*
1824 (?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1825 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s*
1826 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
1827 m
= operator_rex
.fullmatch(filter_spec
)
1830 comparison_value
= int(m
.group('value'))
1832 comparison_value
= parse_filesize(m
.group('value'))
1833 if comparison_value
is None:
1834 comparison_value
= parse_filesize(m
.group('value') + 'B')
1835 if comparison_value
is None:
1837 'Invalid value %r in format specification %r' % (
1838 m
.group('value'), filter_spec
))
1839 op
= OPERATORS
[m
.group('op')]
1844 '^=': lambda attr
, value
: attr
.startswith(value
),
1845 '$=': lambda attr
, value
: attr
.endswith(value
),
1846 '*=': lambda attr
, value
: value
in attr
,
1847 '~=': lambda attr
, value
: value
.search(attr
) is not None
1849 str_operator_rex
= re
.compile(r
'''(?x)\s*
1850 (?P<key>[a-zA-Z0-9._-]+)\s*
1851 (?P<negation>!\s*)?(?P<op>%s)\s*(?P<none_inclusive>\?\s*)?
1853 (?P<value>(?(quote)(?:(?!(?P=quote))[^\\]|\\.)+|[\w.-]+))
1854 (?(quote)(?P=quote))\s*
1855 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
1856 m
= str_operator_rex
.fullmatch(filter_spec
)
1858 if m
.group('op') == '~=':
1859 comparison_value
= re
.compile(m
.group('value'))
1861 comparison_value
= re
.sub(r
'''\\([\\"'])''', r
'\1', m
.group('value'))
1862 str_op
= STR_OPERATORS
[m
.group('op')]
1863 if m
.group('negation'):
1864 op
= lambda attr
, value
: not str_op(attr
, value
)
1869 raise SyntaxError('Invalid filter specification %r' % filter_spec
)
1872 actual_value
= f
.get(m
.group('key'))
1873 if actual_value
is None:
1874 return m
.group('none_inclusive')
1875 return op(actual_value
, comparison_value
)
1878 def _check_formats(self
, formats
):
1880 self
.to_screen('[info] Testing format %s' % f
['format_id'])
1881 path
= self
.get_output_path('temp')
1882 if not self
._ensure
_dir
_exists
(f
'{path}/'):
1884 temp_file
= tempfile
.NamedTemporaryFile(suffix
='.tmp', delete
=False, dir=path
or None)
1887 success
, _
= self
.dl(temp_file
.name
, f
, test
=True)
1888 except (DownloadError
, IOError, OSError, ValueError) + network_exceptions
:
1891 if os
.path
.exists(temp_file
.name
):
1893 os
.remove(temp_file
.name
)
1895 self
.report_warning('Unable to delete temporary file "%s"' % temp_file
.name
)
1899 self
.to_screen('[info] Unable to download format %s. Skipping...' % f
['format_id'])
1901 def _default_format_spec(self
, info_dict
, download
=True):
1904 merger
= FFmpegMergerPP(self
)
1905 return merger
.available
and merger
.can_merge()
1908 not self
.params
.get('simulate')
1912 or info_dict
.get('is_live', False)
1913 or self
.outtmpl_dict
['default'] == '-'))
1916 or self
.params
.get('allow_multiple_audio_streams', False)
1917 or 'format-spec' in self
.params
.get('compat_opts', []))
1920 'best/bestvideo+bestaudio' if prefer_best
1921 else 'bestvideo*+bestaudio/best' if not compat
1922 else 'bestvideo+bestaudio/best')
1924 def build_format_selector(self
, format_spec
):
1925 def syntax_error(note
, start
):
1927 'Invalid format specification: '
1928 '{}\n\t{}\n\t{}^'.format(note
, format_spec
, ' ' * start
[1]))
1929 return SyntaxError(message
)
1931 PICKFIRST
= 'PICKFIRST'
1935 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1937 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
1938 'video': self
.params
.get('allow_multiple_video_streams', False)}
1940 check_formats
= self
.params
.get('check_formats') == 'selected'
1942 def _parse_filter(tokens
):
1944 for type, string
, start
, _
, _
in tokens
:
1945 if type == tokenize
.OP
and string
== ']':
1946 return ''.join(filter_parts
)
1948 filter_parts
.append(string
)
1950 def _remove_unused_ops(tokens
):
1951 # Remove operators that we don't use and join them with the surrounding strings
1952 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1953 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
1954 last_string
, last_start
, last_end
, last_line
= None, None, None, None
1955 for type, string
, start
, end
, line
in tokens
:
1956 if type == tokenize
.OP
and string
== '[':
1958 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1960 yield type, string
, start
, end
, line
1961 # everything inside brackets will be handled by _parse_filter
1962 for type, string
, start
, end
, line
in tokens
:
1963 yield type, string
, start
, end
, line
1964 if type == tokenize
.OP
and string
== ']':
1966 elif type == tokenize
.OP
and string
in ALLOWED_OPS
:
1968 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1970 yield type, string
, start
, end
, line
1971 elif type in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
1973 last_string
= string
1977 last_string
+= string
1979 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1981 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
1983 current_selector
= None
1984 for type, string
, start
, _
, _
in tokens
:
1985 # ENCODING is only defined in python 3.x
1986 if type == getattr(tokenize
, 'ENCODING', None):
1988 elif type in [tokenize
.NAME
, tokenize
.NUMBER
]:
1989 current_selector
= FormatSelector(SINGLE
, string
, [])
1990 elif type == tokenize
.OP
:
1992 if not inside_group
:
1993 # ')' will be handled by the parentheses group
1994 tokens
.restore_last_token()
1996 elif inside_merge
and string
in ['/', ',']:
1997 tokens
.restore_last_token()
1999 elif inside_choice
and string
== ',':
2000 tokens
.restore_last_token()
2003 if not current_selector
:
2004 raise syntax_error('"," must follow a format selector', start
)
2005 selectors
.append(current_selector
)
2006 current_selector
= None
2008 if not current_selector
:
2009 raise syntax_error('"/" must follow a format selector', start
)
2010 first_choice
= current_selector
2011 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
2012 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
2014 if not current_selector
:
2015 current_selector
= FormatSelector(SINGLE
, 'best', [])
2016 format_filter
= _parse_filter(tokens
)
2017 current_selector
.filters
.append(format_filter
)
2019 if current_selector
:
2020 raise syntax_error('Unexpected "("', start
)
2021 group
= _parse_format_selection(tokens
, inside_group
=True)
2022 current_selector
= FormatSelector(GROUP
, group
, [])
2024 if not current_selector
:
2025 raise syntax_error('Unexpected "+"', start
)
2026 selector_1
= current_selector
2027 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
2029 raise syntax_error('Expected a selector', start
)
2030 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
2032 raise syntax_error(f
'Operator not recognized: "{string}"', start
)
2033 elif type == tokenize
.ENDMARKER
:
2035 if current_selector
:
2036 selectors
.append(current_selector
)
2039 def _merge(formats_pair
):
2040 format_1
, format_2
= formats_pair
2043 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
2044 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
2046 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
2047 get_no_more
= {'video': False, 'audio': False}
2048 for (i
, fmt_info
) in enumerate(formats_info
):
2049 if fmt_info
.get('acodec') == fmt_info
.get('vcodec') == 'none':
2052 for aud_vid
in ['audio', 'video']:
2053 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
2054 if get_no_more
[aud_vid
]:
2057 get_no_more
[aud_vid
] = True
2059 if len(formats_info
) == 1:
2060 return formats_info
[0]
2062 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
2063 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
2065 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
2066 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
2068 output_ext
= self
.params
.get('merge_output_format')
2071 output_ext
= the_only_video
['ext']
2072 elif the_only_audio
and not video_fmts
:
2073 output_ext
= the_only_audio
['ext']
2077 filtered
= lambda *keys
: filter(None, (traverse_obj(fmt
, *keys
) for fmt
in formats_info
))
2080 'requested_formats': formats_info
,
2081 'format': '+'.join(filtered('format')),
2082 'format_id': '+'.join(filtered('format_id')),
2084 'protocol': '+'.join(map(determine_protocol
, formats_info
)),
2085 'language': '+'.join(orderedSet(filtered('language'))) or None,
2086 'format_note': '+'.join(orderedSet(filtered('format_note'))) or None,
2087 'filesize_approx': sum(filtered('filesize', 'filesize_approx')) or None,
2088 'tbr': sum(filtered('tbr', 'vbr', 'abr')),
2093 'width': the_only_video
.get('width'),
2094 'height': the_only_video
.get('height'),
2095 'resolution': the_only_video
.get('resolution') or self
.format_resolution(the_only_video
),
2096 'fps': the_only_video
.get('fps'),
2097 'dynamic_range': the_only_video
.get('dynamic_range'),
2098 'vcodec': the_only_video
.get('vcodec'),
2099 'vbr': the_only_video
.get('vbr'),
2100 'stretched_ratio': the_only_video
.get('stretched_ratio'),
2105 'acodec': the_only_audio
.get('acodec'),
2106 'abr': the_only_audio
.get('abr'),
2107 'asr': the_only_audio
.get('asr'),
2112 def _check_formats(formats
):
2113 if not check_formats
:
2116 yield from self
._check
_formats
(formats
)
2118 def _build_selector_function(selector
):
2119 if isinstance(selector
, list): # ,
2120 fs
= [_build_selector_function(s
) for s
in selector
]
2122 def selector_function(ctx
):
2125 return selector_function
2127 elif selector
.type == GROUP
: # ()
2128 selector_function
= _build_selector_function(selector
.selector
)
2130 elif selector
.type == PICKFIRST
: # /
2131 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
2133 def selector_function(ctx
):
2135 picked_formats
= list(f(ctx
))
2137 return picked_formats
2140 elif selector
.type == MERGE
: # +
2141 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
2143 def selector_function(ctx
):
2144 for pair
in itertools
.product(selector_1(ctx
), selector_2(ctx
)):
2147 elif selector
.type == SINGLE
: # atom
2148 format_spec
= selector
.selector
or 'best'
2150 # TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
2151 if format_spec
== 'all':
2152 def selector_function(ctx
):
2153 yield from _check_formats(ctx
['formats'][::-1])
2154 elif format_spec
== 'mergeall':
2155 def selector_function(ctx
):
2156 formats
= list(_check_formats(
2157 f
for f
in ctx
['formats'] if f
.get('vcodec') != 'none' or f
.get('acodec') != 'none'))
2160 merged_format
= formats
[-1]
2161 for f
in formats
[-2::-1]:
2162 merged_format
= _merge((merged_format
, f
))
2166 format_fallback
, seperate_fallback
, format_reverse
, format_idx
= False, None, True, 1
2168 r
'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
2170 if mobj
is not None:
2171 format_idx
= int_or_none(mobj
.group('n'), default
=1)
2172 format_reverse
= mobj
.group('bw')[0] == 'b'
2173 format_type
= (mobj
.group('type') or [None])[0]
2174 not_format_type
= {'v': 'a', 'a': 'v'}
.get(format_type
)
2175 format_modified
= mobj
.group('mod') is not None
2177 format_fallback
= not format_type
and not format_modified
# for b, w
2179 (lambda f
: f
.get('%scodec' % format_type
) != 'none')
2180 if format_type
and format_modified
# bv*, ba*, wv*, wa*
2181 else (lambda f
: f
.get('%scodec' % not_format_type
) == 'none')
2182 if format_type
# bv, ba, wv, wa
2183 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
2184 if not format_modified
# b, w
2185 else lambda f
: True) # b*, w*
2186 filter_f
= lambda f
: _filter_f(f
) and (
2187 f
.get('vcodec') != 'none' or f
.get('acodec') != 'none')
2189 if format_spec
in self
._format
_selection
_exts
['audio']:
2190 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none'
2191 elif format_spec
in self
._format
_selection
_exts
['video']:
2192 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') != 'none' and f
.get('vcodec') != 'none'
2193 seperate_fallback
= lambda f
: f
.get('ext') == format_spec
and f
.get('vcodec') != 'none'
2194 elif format_spec
in self
._format
_selection
_exts
['storyboards']:
2195 filter_f
= lambda f
: f
.get('ext') == format_spec
and f
.get('acodec') == 'none' and f
.get('vcodec') == 'none'
2197 filter_f
= lambda f
: f
.get('format_id') == format_spec
# id
2199 def selector_function(ctx
):
2200 formats
= list(ctx
['formats'])
2201 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
2203 if format_fallback
and ctx
['incomplete_formats']:
2204 # for extractors with incomplete formats (audio only (soundcloud)
2205 # or video only (imgur)) best/worst will fallback to
2206 # best/worst {video,audio}-only format
2208 elif seperate_fallback
and not ctx
['has_merged_format']:
2209 # for compatibility with youtube-dl when there is no pre-merged format
2210 matches
= list(filter(seperate_fallback
, formats
))
2211 matches
= LazyList(_check_formats(matches
[::-1 if format_reverse
else 1]))
2213 yield matches
[format_idx
- 1]
2214 except LazyList
.IndexError:
2217 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
2219 def final_selector(ctx
):
2220 ctx_copy
= dict(ctx
)
2221 for _filter
in filters
:
2222 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
2223 return selector_function(ctx_copy
)
2224 return final_selector
2226 stream
= io
.BytesIO(format_spec
.encode('utf-8'))
2228 tokens
= list(_remove_unused_ops(tokenize
.tokenize(stream
.readline
)))
2229 except tokenize
.TokenError
:
2230 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
2232 class TokenIterator
:
2233 def __init__(self
, tokens
):
2234 self
.tokens
= tokens
2241 if self
.counter
>= len(self
.tokens
):
2242 raise StopIteration()
2243 value
= self
.tokens
[self
.counter
]
2249 def restore_last_token(self
):
2252 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
2253 return _build_selector_function(parsed_selector
)
2255 def _calc_headers(self
, info_dict
):
2256 res
= merge_headers(self
.params
['http_headers'], info_dict
.get('http_headers') or {})
2258 cookies
= self
._calc
_cookies
(info_dict
)
2260 res
['Cookie'] = cookies
2262 if 'X-Forwarded-For' not in res
:
2263 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
2264 if x_forwarded_for_ip
:
2265 res
['X-Forwarded-For'] = x_forwarded_for_ip
2269 def _calc_cookies(self
, info_dict
):
2270 pr
= sanitized_Request(info_dict
['url'])
2271 self
.cookiejar
.add_cookie_header(pr
)
2272 return pr
.get_header('Cookie')
2274 def _sort_thumbnails(self
, thumbnails
):
2275 thumbnails
.sort(key
=lambda t
: (
2276 t
.get('preference') if t
.get('preference') is not None else -1,
2277 t
.get('width') if t
.get('width') is not None else -1,
2278 t
.get('height') if t
.get('height') is not None else -1,
2279 t
.get('id') if t
.get('id') is not None else '',
2282 def _sanitize_thumbnails(self
, info_dict
):
2283 thumbnails
= info_dict
.get('thumbnails')
2284 if thumbnails
is None:
2285 thumbnail
= info_dict
.get('thumbnail')
2287 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
2291 def check_thumbnails(thumbnails
):
2292 for t
in thumbnails
:
2293 self
.to_screen(f
'[info] Testing thumbnail {t["id"]}')
2295 self
.urlopen(HEADRequest(t
['url']))
2296 except network_exceptions
as err
:
2297 self
.to_screen(f
'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...')
2301 self
._sort
_thumbnails
(thumbnails
)
2302 for i
, t
in enumerate(thumbnails
):
2303 if t
.get('id') is None:
2305 if t
.get('width') and t
.get('height'):
2306 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
2307 t
['url'] = sanitize_url(t
['url'])
2309 if self
.params
.get('check_formats') is True:
2310 info_dict
['thumbnails'] = LazyList(check_thumbnails(thumbnails
[::-1]), reverse
=True)
2312 info_dict
['thumbnails'] = thumbnails
2314 def _fill_common_fields(self
, info_dict
, is_video
=True):
2315 # TODO: move sanitization here
2317 # playlists are allowed to lack "title"
2318 info_dict
['fulltitle'] = info_dict
.get('title')
2319 if 'title' not in info_dict
:
2320 raise ExtractorError('Missing "title" field in extractor result',
2321 video_id
=info_dict
['id'], ie
=info_dict
['extractor'])
2322 elif not info_dict
.get('title'):
2323 self
.report_warning('Extractor failed to obtain "title". Creating a generic title instead')
2324 info_dict
['title'] = f
'{info_dict["extractor"]} video #{info_dict["id"]}'
2326 if info_dict
.get('duration') is not None:
2327 info_dict
['duration_string'] = formatSeconds(info_dict
['duration'])
2329 for ts_key
, date_key
in (
2330 ('timestamp', 'upload_date'),
2331 ('release_timestamp', 'release_date'),
2332 ('modified_timestamp', 'modified_date'),
2334 if info_dict
.get(date_key
) is None and info_dict
.get(ts_key
) is not None:
2335 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
2336 # see http://bugs.python.org/issue1646728)
2337 with contextlib
.suppress(ValueError, OverflowError, OSError):
2338 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
[ts_key
])
2339 info_dict
[date_key
] = upload_date
.strftime('%Y%m%d')
2341 live_keys
= ('is_live', 'was_live')
2342 live_status
= info_dict
.get('live_status')
2343 if live_status
is None:
2344 for key
in live_keys
:
2345 if info_dict
.get(key
) is False:
2347 if info_dict
.get(key
):
2350 if all(info_dict
.get(key
) is False for key
in live_keys
):
2351 live_status
= 'not_live'
2353 info_dict
['live_status'] = live_status
2354 for key
in live_keys
:
2355 if info_dict
.get(key
) is None:
2356 info_dict
[key
] = (live_status
== key
)
2358 # Auto generate title fields corresponding to the *_number fields when missing
2359 # in order to always have clean titles. This is very common for TV series.
2360 for field
in ('chapter', 'season', 'episode'):
2361 if info_dict
.get('%s_number' % field
) is not None and not info_dict
.get(field
):
2362 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
['%s_number' % field
])
2364 def process_video_result(self
, info_dict
, download
=True):
2365 assert info_dict
.get('_type', 'video') == 'video'
2366 self
._num
_videos
+= 1
2368 if 'id' not in info_dict
:
2369 raise ExtractorError('Missing "id" field in extractor result', ie
=info_dict
['extractor'])
2370 elif not info_dict
.get('id'):
2371 raise ExtractorError('Extractor failed to obtain "id"', ie
=info_dict
['extractor'])
2373 def report_force_conversion(field
, field_not
, conversion
):
2374 self
.report_warning(
2375 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
2376 % (field
, field_not
, conversion
))
2378 def sanitize_string_field(info
, string_field
):
2379 field
= info
.get(string_field
)
2380 if field
is None or isinstance(field
, compat_str
):
2382 report_force_conversion(string_field
, 'a string', 'string')
2383 info
[string_field
] = compat_str(field
)
2385 def sanitize_numeric_fields(info
):
2386 for numeric_field
in self
._NUMERIC
_FIELDS
:
2387 field
= info
.get(numeric_field
)
2388 if field
is None or isinstance(field
, (int, float)):
2390 report_force_conversion(numeric_field
, 'numeric', 'int')
2391 info
[numeric_field
] = int_or_none(field
)
2393 sanitize_string_field(info_dict
, 'id')
2394 sanitize_numeric_fields(info_dict
)
2395 if (info_dict
.get('duration') or 0) <= 0 and info_dict
.pop('duration', None):
2396 self
.report_warning('"duration" field is negative, there is an error in extractor')
2398 if 'playlist' not in info_dict
:
2399 # It isn't part of a playlist
2400 info_dict
['playlist'] = None
2401 info_dict
['playlist_index'] = None
2403 self
._sanitize
_thumbnails
(info_dict
)
2405 thumbnail
= info_dict
.get('thumbnail')
2406 thumbnails
= info_dict
.get('thumbnails')
2408 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
2410 info_dict
['thumbnail'] = thumbnails
[-1]['url']
2412 if info_dict
.get('display_id') is None and 'id' in info_dict
:
2413 info_dict
['display_id'] = info_dict
['id']
2415 self
._fill
_common
_fields
(info_dict
)
2417 for cc_kind
in ('subtitles', 'automatic_captions'):
2418 cc
= info_dict
.get(cc_kind
)
2420 for _
, subtitle
in cc
.items():
2421 for subtitle_format
in subtitle
:
2422 if subtitle_format
.get('url'):
2423 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
2424 if subtitle_format
.get('ext') is None:
2425 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
2427 automatic_captions
= info_dict
.get('automatic_captions')
2428 subtitles
= info_dict
.get('subtitles')
2430 info_dict
['requested_subtitles'] = self
.process_subtitles(
2431 info_dict
['id'], subtitles
, automatic_captions
)
2433 if info_dict
.get('formats') is None:
2434 # There's only one format available
2435 formats
= [info_dict
]
2437 formats
= info_dict
['formats']
2439 info_dict
['__has_drm'] = any(f
.get('has_drm') for f
in formats
)
2440 if not self
.params
.get('allow_unplayable_formats'):
2441 formats
= [f
for f
in formats
if not f
.get('has_drm')]
2442 if info_dict
['__has_drm'] and all(
2443 f
.get('acodec') == f
.get('vcodec') == 'none' for f
in formats
):
2444 self
.report_warning(
2445 'This video is DRM protected and only images are available for download. '
2446 'Use --list-formats to see them')
2448 get_from_start
= not info_dict
.get('is_live') or bool(self
.params
.get('live_from_start'))
2449 if not get_from_start
:
2450 info_dict
['title'] += ' ' + datetime
.datetime
.now().strftime('%Y-%m-%d %H:%M')
2451 if info_dict
.get('is_live') and formats
:
2452 formats
= [f
for f
in formats
if bool(f
.get('is_from_start')) == get_from_start
]
2453 if get_from_start
and not formats
:
2454 self
.raise_no_formats(info_dict
, msg
=(
2455 '--live-from-start is passed, but there are no formats that can be downloaded from the start. '
2456 'If you want to download from the current time, use --no-live-from-start'))
2459 self
.raise_no_formats(info_dict
)
2461 def is_wellformed(f
):
2464 self
.report_warning(
2465 '"url" field is missing or empty - skipping format, '
2466 'there is an error in extractor')
2468 if isinstance(url
, bytes):
2469 sanitize_string_field(f
, 'url')
2472 # Filter out malformed formats for better extraction robustness
2473 formats
= list(filter(is_wellformed
, formats
))
2477 # We check that all the formats have the format and format_id fields
2478 for i
, format
in enumerate(formats
):
2479 sanitize_string_field(format
, 'format_id')
2480 sanitize_numeric_fields(format
)
2481 format
['url'] = sanitize_url(format
['url'])
2482 if not format
.get('format_id'):
2483 format
['format_id'] = compat_str(i
)
2485 # Sanitize format_id from characters used in format selector expression
2486 format
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', format
['format_id'])
2487 format_id
= format
['format_id']
2488 if format_id
not in formats_dict
:
2489 formats_dict
[format_id
] = []
2490 formats_dict
[format_id
].append(format
)
2492 # Make sure all formats have unique format_id
2493 common_exts
= set(itertools
.chain(*self
._format
_selection
_exts
.values()))
2494 for format_id
, ambiguous_formats
in formats_dict
.items():
2495 ambigious_id
= len(ambiguous_formats
) > 1
2496 for i
, format
in enumerate(ambiguous_formats
):
2498 format
['format_id'] = '%s-%d' % (format_id
, i
)
2499 if format
.get('ext') is None:
2500 format
['ext'] = determine_ext(format
['url']).lower()
2501 # Ensure there is no conflict between id and ext in format selection
2502 # See https://github.com/yt-dlp/yt-dlp/issues/1282
2503 if format
['format_id'] != format
['ext'] and format
['format_id'] in common_exts
:
2504 format
['format_id'] = 'f%s' % format
['format_id']
2506 for i
, format
in enumerate(formats
):
2507 if format
.get('format') is None:
2508 format
['format'] = '{id} - {res}{note}'.format(
2509 id=format
['format_id'],
2510 res
=self
.format_resolution(format
),
2511 note
=format_field(format
, 'format_note', ' (%s)'),
2513 if format
.get('protocol') is None:
2514 format
['protocol'] = determine_protocol(format
)
2515 if format
.get('resolution') is None:
2516 format
['resolution'] = self
.format_resolution(format
, default
=None)
2517 if format
.get('dynamic_range') is None and format
.get('vcodec') != 'none':
2518 format
['dynamic_range'] = 'SDR'
2519 if (info_dict
.get('duration') and format
.get('tbr')
2520 and not format
.get('filesize') and not format
.get('filesize_approx')):
2521 format
['filesize_approx'] = info_dict
['duration'] * format
['tbr'] * (1024 / 8)
2523 # Add HTTP headers, so that external programs can use them from the
2525 full_format_info
= info_dict
.copy()
2526 full_format_info
.update(format
)
2527 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
2528 # Remove private housekeeping stuff
2529 if '__x_forwarded_for_ip' in info_dict
:
2530 del info_dict
['__x_forwarded_for_ip']
2532 if self
.params
.get('check_formats') is True:
2533 formats
= LazyList(self
._check
_formats
(formats
[::-1]), reverse
=True)
2535 if not formats
or formats
[0] is not info_dict
:
2536 # only set the 'formats' fields if the original info_dict list them
2537 # otherwise we end up with a circular reference, the first (and unique)
2538 # element in the 'formats' field in info_dict is info_dict itself,
2539 # which can't be exported to json
2540 info_dict
['formats'] = formats
2542 info_dict
, _
= self
.pre_process(info_dict
)
2544 if self
._match
_entry
(info_dict
, incomplete
=self
._format
_fields
) is not None:
2547 self
.post_extract(info_dict
)
2548 info_dict
, _
= self
.pre_process(info_dict
, 'after_filter')
2550 # The pre-processors may have modified the formats
2551 formats
= info_dict
.get('formats', [info_dict
])
2553 list_only
= self
.params
.get('simulate') is None and (
2554 self
.params
.get('list_thumbnails') or self
.params
.get('listformats') or self
.params
.get('listsubtitles'))
2555 interactive_format_selection
= not list_only
and self
.format_selector
== '-'
2556 if self
.params
.get('list_thumbnails'):
2557 self
.list_thumbnails(info_dict
)
2558 if self
.params
.get('listsubtitles'):
2559 if 'automatic_captions' in info_dict
:
2560 self
.list_subtitles(
2561 info_dict
['id'], automatic_captions
, 'automatic captions')
2562 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
2563 if self
.params
.get('listformats') or interactive_format_selection
:
2564 self
.list_formats(info_dict
)
2566 # Without this printing, -F --print-json will not work
2567 self
.__forced
_printings
(info_dict
, self
.prepare_filename(info_dict
), incomplete
=True)
2570 format_selector
= self
.format_selector
2571 if format_selector
is None:
2572 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
2573 self
.write_debug('Default format spec: %s' % req_format
)
2574 format_selector
= self
.build_format_selector(req_format
)
2577 if interactive_format_selection
:
2579 self
._format
_screen
('\nEnter format selector: ', self
.Styles
.EMPHASIS
))
2581 format_selector
= self
.build_format_selector(req_format
)
2582 except SyntaxError as err
:
2583 self
.report_error(err
, tb
=False, is_error
=False)
2586 formats_to_download
= list(format_selector({
2588 'has_merged_format': any('none' not in (f
.get('acodec'), f
.get('vcodec')) for f
in formats
),
2589 'incomplete_formats': (
2590 # All formats are video-only or
2591 all(f
.get('vcodec') != 'none' and f
.get('acodec') == 'none' for f
in formats
)
2592 # all formats are audio-only
2593 or all(f
.get('vcodec') == 'none' and f
.get('acodec') != 'none' for f
in formats
)),
2595 if interactive_format_selection
and not formats_to_download
:
2596 self
.report_error('Requested format is not available', tb
=False, is_error
=False)
2600 if not formats_to_download
:
2601 if not self
.params
.get('ignore_no_formats_error'):
2602 raise ExtractorError(
2603 'Requested format is not available. Use --list-formats for a list of available formats',
2604 expected
=True, video_id
=info_dict
['id'], ie
=info_dict
['extractor'])
2605 self
.report_warning('Requested format is not available')
2606 # Process what we can, even without any available formats.
2607 formats_to_download
= [{}]
2609 best_format
= formats_to_download
[-1]
2613 f
'[info] {info_dict["id"]}: Downloading {len(formats_to_download)} format(s): '
2614 + ', '.join([f
['format_id'] for f
in formats_to_download
]))
2615 max_downloads_reached
= False
2616 for i
, fmt
in enumerate(formats_to_download
):
2617 formats_to_download
[i
] = new_info
= self
._copy
_infodict
(info_dict
)
2618 new_info
.update(fmt
)
2620 self
.process_info(new_info
)
2621 except MaxDownloadsReached
:
2622 max_downloads_reached
= True
2623 # Remove copied info
2624 for key
, val
in tuple(new_info
.items()):
2625 if info_dict
.get(key
) == val
:
2627 if max_downloads_reached
:
2630 write_archive
= {f.get('__write_download_archive', False) for f in formats_to_download}
2631 assert write_archive
.issubset({True, False, 'ignore'}
)
2632 if True in write_archive
and False not in write_archive
:
2633 self
.record_download_archive(info_dict
)
2635 info_dict
['requested_downloads'] = formats_to_download
2636 info_dict
= self
.run_all_pps('after_video', info_dict
)
2637 if max_downloads_reached
:
2638 raise MaxDownloadsReached()
2640 # We update the info dict with the selected best quality format (backwards compatibility)
2641 info_dict
.update(best_format
)
2644 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
2645 """Select the requested subtitles and their format"""
2646 available_subs
, normal_sub_langs
= {}, []
2647 if normal_subtitles
and self
.params
.get('writesubtitles'):
2648 available_subs
.update(normal_subtitles
)
2649 normal_sub_langs
= tuple(normal_subtitles
.keys())
2650 if automatic_captions
and self
.params
.get('writeautomaticsub'):
2651 for lang
, cap_info
in automatic_captions
.items():
2652 if lang
not in available_subs
:
2653 available_subs
[lang
] = cap_info
2655 if (not self
.params
.get('writesubtitles') and not
2656 self
.params
.get('writeautomaticsub') or not
2660 all_sub_langs
= tuple(available_subs
.keys())
2661 if self
.params
.get('allsubtitles', False):
2662 requested_langs
= all_sub_langs
2663 elif self
.params
.get('subtitleslangs', False):
2664 # A list is used so that the order of languages will be the same as
2665 # given in subtitleslangs. See https://github.com/yt-dlp/yt-dlp/issues/1041
2666 requested_langs
= []
2667 for lang_re
in self
.params
.get('subtitleslangs'):
2668 discard
= lang_re
[0] == '-'
2670 lang_re
= lang_re
[1:]
2671 if lang_re
== 'all':
2673 requested_langs
= []
2675 requested_langs
.extend(all_sub_langs
)
2677 current_langs
= filter(re
.compile(lang_re
+ '$').match
, all_sub_langs
)
2679 for lang
in current_langs
:
2680 while lang
in requested_langs
:
2681 requested_langs
.remove(lang
)
2683 requested_langs
.extend(current_langs
)
2684 requested_langs
= orderedSet(requested_langs
)
2685 elif normal_sub_langs
:
2686 requested_langs
= ['en'] if 'en' in normal_sub_langs
else normal_sub_langs
[:1]
2688 requested_langs
= ['en'] if 'en' in all_sub_langs
else all_sub_langs
[:1]
2690 self
.write_debug('Downloading subtitles: %s' % ', '.join(requested_langs
))
2692 formats_query
= self
.params
.get('subtitlesformat', 'best')
2693 formats_preference
= formats_query
.split('/') if formats_query
else []
2695 for lang
in requested_langs
:
2696 formats
= available_subs
.get(lang
)
2698 self
.report_warning(f
'{lang} subtitles not available for {video_id}')
2700 for ext
in formats_preference
:
2704 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
2710 self
.report_warning(
2711 'No subtitle format found matching "%s" for language %s, '
2712 'using %s' % (formats_query
, lang
, f
['ext']))
2716 def _forceprint(self
, key
, info_dict
):
2717 if info_dict
is None:
2719 info_copy
= info_dict
.copy()
2720 info_copy
['formats_table'] = self
.render_formats_table(info_dict
)
2721 info_copy
['thumbnails_table'] = self
.render_thumbnails_table(info_dict
)
2722 info_copy
['subtitles_table'] = self
.render_subtitles_table(info_dict
.get('id'), info_dict
.get('subtitles'))
2723 info_copy
['automatic_captions_table'] = self
.render_subtitles_table(info_dict
.get('id'), info_dict
.get('automatic_captions'))
2725 def format_tmpl(tmpl
):
2726 mobj
= re
.match(r
'\w+(=?)$', tmpl
)
2727 if mobj
and mobj
.group(1):
2728 return f
'{tmpl[:-1]} = %({tmpl[:-1]})r'
2730 return f
'%({tmpl})s'
2733 for tmpl
in self
.params
['forceprint'].get(key
, []):
2734 self
.to_stdout(self
.evaluate_outtmpl(format_tmpl(tmpl
), info_copy
))
2736 for tmpl
, file_tmpl
in self
.params
['print_to_file'].get(key
, []):
2737 filename
= self
.prepare_filename(info_dict
, outtmpl
=file_tmpl
)
2738 tmpl
= format_tmpl(tmpl
)
2739 self
.to_screen(f
'[info] Writing {tmpl!r} to: {filename}')
2740 if self
._ensure
_dir
_exists
(filename
):
2741 with open(filename
, 'a', encoding
='utf-8') as f
:
2742 f
.write(self
.evaluate_outtmpl(tmpl
, info_copy
) + '\n')
2744 def __forced_printings(self
, info_dict
, filename
, incomplete
):
2745 def print_mandatory(field
, actual_field
=None):
2746 if actual_field
is None:
2747 actual_field
= field
2748 if (self
.params
.get('force%s' % field
, False)
2749 and (not incomplete
or info_dict
.get(actual_field
) is not None)):
2750 self
.to_stdout(info_dict
[actual_field
])
2752 def print_optional(field
):
2753 if (self
.params
.get('force%s' % field
, False)
2754 and info_dict
.get(field
) is not None):
2755 self
.to_stdout(info_dict
[field
])
2757 info_dict
= info_dict
.copy()
2758 if filename
is not None:
2759 info_dict
['filename'] = filename
2760 if info_dict
.get('requested_formats') is not None:
2761 # For RTMP URLs, also include the playpath
2762 info_dict
['urls'] = '\n'.join(f
['url'] + f
.get('play_path', '') for f
in info_dict
['requested_formats'])
2763 elif info_dict
.get('url'):
2764 info_dict
['urls'] = info_dict
['url'] + info_dict
.get('play_path', '')
2766 if (self
.params
.get('forcejson')
2767 or self
.params
['forceprint'].get('video')
2768 or self
.params
['print_to_file'].get('video')):
2769 self
.post_extract(info_dict
)
2770 self
._forceprint
('video', info_dict
)
2772 print_mandatory('title')
2773 print_mandatory('id')
2774 print_mandatory('url', 'urls')
2775 print_optional('thumbnail')
2776 print_optional('description')
2777 print_optional('filename')
2778 if self
.params
.get('forceduration') and info_dict
.get('duration') is not None:
2779 self
.to_stdout(formatSeconds(info_dict
['duration']))
2780 print_mandatory('format')
2782 if self
.params
.get('forcejson'):
2783 self
.to_stdout(json
.dumps(self
.sanitize_info(info_dict
)))
2785 def dl(self
, name
, info
, subtitle
=False, test
=False):
2786 if not info
.get('url'):
2787 self
.raise_no_formats(info
, True)
2790 verbose
= self
.params
.get('verbose')
2793 'quiet': self
.params
.get('quiet') or not verbose
,
2795 'noprogress': not verbose
,
2797 'skip_unavailable_fragments': False,
2798 'keep_fragments': False,
2800 '_no_ytdl_file': True,
2803 params
= self
.params
2804 fd
= get_suitable_downloader(info
, params
, to_stdout
=(name
== '-'))(self
, params
)
2806 for ph
in self
._progress
_hooks
:
2807 fd
.add_progress_hook(ph
)
2809 (f
['url'].split(',')[0] + ',<data>' if f
['url'].startswith('data:') else f
['url'])
2810 for f
in info
.get('requested_formats', []) or [info
])
2811 self
.write_debug('Invoking downloader on "%s"' % urls
)
2813 # Note: Ideally info should be a deep-copied so that hooks cannot modify it.
2814 # But it may contain objects that are not deep-copyable
2815 new_info
= self
._copy
_infodict
(info
)
2816 if new_info
.get('http_headers') is None:
2817 new_info
['http_headers'] = self
._calc
_headers
(new_info
)
2818 return fd
.download(name
, new_info
, subtitle
)
2820 def existing_file(self
, filepaths
, *, default_overwrite
=True):
2821 existing_files
= list(filter(os
.path
.exists
, orderedSet(filepaths
)))
2822 if existing_files
and not self
.params
.get('overwrites', default_overwrite
):
2823 return existing_files
[0]
2825 for file in existing_files
:
2826 self
.report_file_delete(file)
2830 def process_info(self
, info_dict
):
2831 """Process a single resolved IE result. (Modifies it in-place)"""
2833 assert info_dict
.get('_type', 'video') == 'video'
2834 original_infodict
= info_dict
2836 if 'format' not in info_dict
and 'ext' in info_dict
:
2837 info_dict
['format'] = info_dict
['ext']
2839 # This is mostly just for backward compatibility of process_info
2840 # As a side-effect, this allows for format-specific filters
2841 if self
._match
_entry
(info_dict
) is not None:
2842 info_dict
['__write_download_archive'] = 'ignore'
2845 # Does nothing under normal operation - for backward compatibility of process_info
2846 self
.post_extract(info_dict
)
2847 self
._num
_downloads
+= 1
2849 # info_dict['_filename'] needs to be set for backward compatibility
2850 info_dict
['_filename'] = full_filename
= self
.prepare_filename(info_dict
, warn
=True)
2851 temp_filename
= self
.prepare_filename(info_dict
, 'temp')
2855 self
.__forced
_printings
(info_dict
, full_filename
, incomplete
=('format' not in info_dict
))
2857 if self
.params
.get('simulate'):
2858 info_dict
['__write_download_archive'] = self
.params
.get('force_write_download_archive')
2861 if full_filename
is None:
2863 if not self
._ensure
_dir
_exists
(encodeFilename(full_filename
)):
2865 if not self
._ensure
_dir
_exists
(encodeFilename(temp_filename
)):
2868 if self
._write
_description
('video', info_dict
,
2869 self
.prepare_filename(info_dict
, 'description')) is None:
2872 sub_files
= self
._write
_subtitles
(info_dict
, temp_filename
)
2873 if sub_files
is None:
2875 files_to_move
.update(dict(sub_files
))
2877 thumb_files
= self
._write
_thumbnails
(
2878 'video', info_dict
, temp_filename
, self
.prepare_filename(info_dict
, 'thumbnail'))
2879 if thumb_files
is None:
2881 files_to_move
.update(dict(thumb_files
))
2883 infofn
= self
.prepare_filename(info_dict
, 'infojson')
2884 _infojson_written
= self
._write
_info
_json
('video', info_dict
, infofn
)
2885 if _infojson_written
:
2886 info_dict
['infojson_filename'] = infofn
2887 # For backward compatibility, even though it was a private field
2888 info_dict
['__infojson_filename'] = infofn
2889 elif _infojson_written
is None:
2892 # Note: Annotations are deprecated
2894 if self
.params
.get('writeannotations', False):
2895 annofn
= self
.prepare_filename(info_dict
, 'annotation')
2897 if not self
._ensure
_dir
_exists
(encodeFilename(annofn
)):
2899 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(annofn
)):
2900 self
.to_screen('[info] Video annotations are already present')
2901 elif not info_dict
.get('annotations'):
2902 self
.report_warning('There are no annotations to write.')
2905 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
2906 with open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
2907 annofile
.write(info_dict
['annotations'])
2908 except (KeyError, TypeError):
2909 self
.report_warning('There are no annotations to write.')
2911 self
.report_error('Cannot write annotations file: ' + annofn
)
2914 # Write internet shortcut files
2915 def _write_link_file(link_type
):
2916 url
= try_get(info_dict
['webpage_url'], iri_to_uri
)
2918 self
.report_warning(
2919 f
'Cannot write internet shortcut file because the actual URL of "{info_dict["webpage_url"]}" is unknown')
2921 linkfn
= replace_extension(self
.prepare_filename(info_dict
, 'link'), link_type
, info_dict
.get('ext'))
2922 if not self
._ensure
_dir
_exists
(encodeFilename(linkfn
)):
2924 if self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(linkfn
)):
2925 self
.to_screen(f
'[info] Internet shortcut (.{link_type}) is already present')
2928 self
.to_screen(f
'[info] Writing internet shortcut (.{link_type}) to: {linkfn}')
2929 with open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8',
2930 newline
='\r\n' if link_type
== 'url' else '\n') as linkfile
:
2931 template_vars
= {'url': url}
2932 if link_type
== 'desktop':
2933 template_vars
['filename'] = linkfn
[:-(len(link_type
) + 1)]
2934 linkfile
.write(LINK_TEMPLATES
[link_type
] % template_vars
)
2936 self
.report_error(f
'Cannot write internet shortcut {linkfn}')
2941 'url': self
.params
.get('writeurllink'),
2942 'webloc': self
.params
.get('writewebloclink'),
2943 'desktop': self
.params
.get('writedesktoplink'),
2945 if self
.params
.get('writelink'):
2946 link_type
= ('webloc' if sys
.platform
== 'darwin'
2947 else 'desktop' if sys
.platform
.startswith('linux')
2949 write_links
[link_type
] = True
2951 if any(should_write
and not _write_link_file(link_type
)
2952 for link_type
, should_write
in write_links
.items()):
2955 def replace_info_dict(new_info
):
2957 if new_info
== info_dict
:
2960 info_dict
.update(new_info
)
2963 new_info
, files_to_move
= self
.pre_process(info_dict
, 'before_dl', files_to_move
)
2964 replace_info_dict(new_info
)
2965 except PostProcessingError
as err
:
2966 self
.report_error('Preprocessing: %s' % str(err
))
2969 if self
.params
.get('skip_download'):
2970 info_dict
['filepath'] = temp_filename
2971 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2972 info_dict
['__files_to_move'] = files_to_move
2973 replace_info_dict(self
.run_pp(MoveFilesAfterDownloadPP(self
, False), info_dict
))
2974 info_dict
['__write_download_archive'] = self
.params
.get('force_write_download_archive')
2977 info_dict
.setdefault('__postprocessors', [])
2980 def existing_video_file(*filepaths
):
2981 ext
= info_dict
.get('ext')
2982 converted
= lambda file: replace_extension(file, self
.params
.get('final_ext') or ext
, ext
)
2983 file = self
.existing_file(itertools
.chain(*zip(map(converted
, filepaths
), filepaths
)),
2984 default_overwrite
=False)
2986 info_dict
['ext'] = os
.path
.splitext(file)[1][1:]
2990 if info_dict
.get('requested_formats') is not None:
2992 def compatible_formats(formats
):
2993 # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
2994 video_formats
= [format
for format
in formats
if format
.get('vcodec') != 'none']
2995 audio_formats
= [format
for format
in formats
if format
.get('acodec') != 'none']
2996 if len(video_formats
) > 2 or len(audio_formats
) > 2:
3000 exts
= {format.get('ext') for format in formats}
3002 {'mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'}
,
3005 for ext_sets
in COMPATIBLE_EXTS
:
3006 if ext_sets
.issuperset(exts
):
3008 # TODO: Check acodec/vcodec
3011 requested_formats
= info_dict
['requested_formats']
3012 old_ext
= info_dict
['ext']
3013 if self
.params
.get('merge_output_format') is None:
3014 if not compatible_formats(requested_formats
):
3015 info_dict
['ext'] = 'mkv'
3016 self
.report_warning(
3017 'Requested formats are incompatible for merge and will be merged into mkv')
3018 if (info_dict
['ext'] == 'webm'
3019 and info_dict
.get('thumbnails')
3020 # check with type instead of pp_key, __name__, or isinstance
3021 # since we dont want any custom PPs to trigger this
3022 and any(type(pp
) == EmbedThumbnailPP
for pp
in self
._pps
['post_process'])):
3023 info_dict
['ext'] = 'mkv'
3024 self
.report_warning(
3025 'webm doesn\'t support embedding a thumbnail, mkv will be used')
3026 new_ext
= info_dict
['ext']
3028 def correct_ext(filename
, ext
=new_ext
):
3031 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
3033 os
.path
.splitext(filename
)[0]
3034 if filename_real_ext
in (old_ext
, new_ext
)
3036 return f
'{filename_wo_ext}.{ext}'
3038 # Ensure filename always has a correct extension for successful merge
3039 full_filename
= correct_ext(full_filename
)
3040 temp_filename
= correct_ext(temp_filename
)
3041 dl_filename
= existing_video_file(full_filename
, temp_filename
)
3042 info_dict
['__real_download'] = False
3045 merger
= FFmpegMergerPP(self
)
3047 fd
= get_suitable_downloader(info_dict
, self
.params
, to_stdout
=temp_filename
== '-')
3048 if dl_filename
is not None:
3049 self
.report_file_already_downloaded(dl_filename
)
3051 for f
in requested_formats
if fd
!= FFmpegFD
else []:
3052 f
['filepath'] = fname
= prepend_extension(
3053 correct_ext(temp_filename
, info_dict
['ext']),
3054 'f%s' % f
['format_id'], info_dict
['ext'])
3055 downloaded
.append(fname
)
3056 info_dict
['url'] = '\n'.join(f
['url'] for f
in requested_formats
)
3057 success
, real_download
= self
.dl(temp_filename
, info_dict
)
3058 info_dict
['__real_download'] = real_download
3060 if self
.params
.get('allow_unplayable_formats'):
3061 self
.report_warning(
3062 'You have requested merging of multiple formats '
3063 'while also allowing unplayable formats to be downloaded. '
3064 'The formats won\'t be merged to prevent data corruption.')
3065 elif not merger
.available
:
3066 msg
= 'You have requested merging of multiple formats but ffmpeg is not installed'
3067 if not self
.params
.get('ignoreerrors'):
3068 self
.report_error(f
'{msg}. Aborting due to --abort-on-error')
3070 self
.report_warning(f
'{msg}. The formats won\'t be merged')
3072 if temp_filename
== '-':
3073 reason
= ('using a downloader other than ffmpeg' if FFmpegFD
.can_merge_formats(info_dict
, self
.params
)
3074 else 'but the formats are incompatible for simultaneous download' if merger
.available
3075 else 'but ffmpeg is not installed')
3076 self
.report_warning(
3077 f
'You have requested downloading multiple formats to stdout {reason}. '
3078 'The formats will be streamed one after the other')
3079 fname
= temp_filename
3080 for f
in requested_formats
:
3081 new_info
= dict(info_dict
)
3082 del new_info
['requested_formats']
3084 if temp_filename
!= '-':
3085 fname
= prepend_extension(
3086 correct_ext(temp_filename
, new_info
['ext']),
3087 'f%s' % f
['format_id'], new_info
['ext'])
3088 if not self
._ensure
_dir
_exists
(fname
):
3090 f
['filepath'] = fname
3091 downloaded
.append(fname
)
3092 partial_success
, real_download
= self
.dl(fname
, new_info
)
3093 info_dict
['__real_download'] = info_dict
['__real_download'] or real_download
3094 success
= success
and partial_success
3096 if downloaded
and merger
.available
and not self
.params
.get('allow_unplayable_formats'):
3097 info_dict
['__postprocessors'].append(merger
)
3098 info_dict
['__files_to_merge'] = downloaded
3099 # Even if there were no downloads, it is being merged only now
3100 info_dict
['__real_download'] = True
3102 for file in downloaded
:
3103 files_to_move
[file] = None
3105 # Just a single file
3106 dl_filename
= existing_video_file(full_filename
, temp_filename
)
3107 if dl_filename
is None or dl_filename
== temp_filename
:
3108 # dl_filename == temp_filename could mean that the file was partially downloaded with --no-part.
3109 # So we should try to resume the download
3110 success
, real_download
= self
.dl(temp_filename
, info_dict
)
3111 info_dict
['__real_download'] = real_download
3113 self
.report_file_already_downloaded(dl_filename
)
3115 dl_filename
= dl_filename
or temp_filename
3116 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
3118 except network_exceptions
as err
:
3119 self
.report_error('unable to download video data: %s' % error_to_compat_str(err
))
3121 except OSError as err
:
3122 raise UnavailableVideoError(err
)
3123 except (ContentTooShortError
, ) as err
:
3124 self
.report_error(f
'content too short (expected {err.expected} bytes and served {err.downloaded})')
3127 if success
and full_filename
!= '-':
3131 fixup_policy
= self
.params
.get('fixup')
3132 vid
= info_dict
['id']
3134 if fixup_policy
in ('ignore', 'never'):
3136 elif fixup_policy
== 'warn':
3138 elif fixup_policy
!= 'force':
3139 assert fixup_policy
in ('detect_or_warn', None)
3140 if not info_dict
.get('__real_download'):
3143 def ffmpeg_fixup(cndn
, msg
, cls
):
3147 self
.report_warning(f
'{vid}: {msg}')
3151 info_dict
['__postprocessors'].append(pp
)
3153 self
.report_warning(f
'{vid}: {msg}. Install ffmpeg to fix this automatically')
3155 stretched_ratio
= info_dict
.get('stretched_ratio')
3157 stretched_ratio
not in (1, None),
3158 f
'Non-uniform pixel ratio {stretched_ratio}',
3159 FFmpegFixupStretchedPP
)
3162 (info_dict
.get('requested_formats') is None
3163 and info_dict
.get('container') == 'm4a_dash'
3164 and info_dict
.get('ext') == 'm4a'),
3165 'writing DASH m4a. Only some players support this container',
3168 downloader
= get_suitable_downloader(info_dict
, self
.params
) if 'protocol' in info_dict
else None
3169 downloader
= downloader
.__name
__ if downloader
else None
3171 if info_dict
.get('requested_formats') is None: # Not necessary if doing merger
3172 ffmpeg_fixup(downloader
== 'HlsFD',
3173 'Possible MPEG-TS in MP4 container or malformed AAC timestamps',
3175 ffmpeg_fixup(info_dict
.get('is_live') and downloader
== 'DashSegmentsFD',
3176 'Possible duplicate MOOV atoms', FFmpegFixupDuplicateMoovPP
)
3178 ffmpeg_fixup(downloader
== 'WebSocketFragmentFD', 'Malformed timestamps detected', FFmpegFixupTimestampPP
)
3179 ffmpeg_fixup(downloader
== 'WebSocketFragmentFD', 'Malformed duration detected', FFmpegFixupDurationPP
)
3183 replace_info_dict(self
.post_process(dl_filename
, info_dict
, files_to_move
))
3184 except PostProcessingError
as err
:
3185 self
.report_error('Postprocessing: %s' % str(err
))
3188 for ph
in self
._post
_hooks
:
3189 ph(info_dict
['filepath'])
3190 except Exception as err
:
3191 self
.report_error('post hooks: %s' % str(err
))
3193 info_dict
['__write_download_archive'] = True
3195 if self
.params
.get('force_write_download_archive'):
3196 info_dict
['__write_download_archive'] = True
3198 # Make sure the info_dict was modified in-place
3199 assert info_dict
is original_infodict
3201 max_downloads
= self
.params
.get('max_downloads')
3202 if max_downloads
is not None and self
._num
_downloads
>= int(max_downloads
):
3203 raise MaxDownloadsReached()
3205 def __download_wrapper(self
, func
):
3206 @functools.wraps(func
)
3207 def wrapper(*args
, **kwargs
):
3209 res
= func(*args
, **kwargs
)
3210 except UnavailableVideoError
as e
:
3211 self
.report_error(e
)
3212 except MaxDownloadsReached
as e
:
3213 self
.to_screen(f
'[info] {e}')
3215 except DownloadCancelled
as e
:
3216 self
.to_screen(f
'[info] {e}')
3217 if not self
.params
.get('break_per_url'):
3220 if self
.params
.get('dump_single_json', False):
3221 self
.post_extract(res
)
3222 self
.to_stdout(json
.dumps(self
.sanitize_info(res
)))
3225 def download(self
, url_list
):
3226 """Download a given list of URLs."""
3227 url_list
= variadic(url_list
) # Passing a single URL is a common mistake
3228 outtmpl
= self
.outtmpl_dict
['default']
3229 if (len(url_list
) > 1
3231 and '%' not in outtmpl
3232 and self
.params
.get('max_downloads') != 1):
3233 raise SameFileError(outtmpl
)
3235 for url
in url_list
:
3236 self
.__download
_wrapper
(self
.extract_info
)(
3237 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
3239 return self
._download
_retcode
3241 def download_with_info_file(self
, info_filename
):
3242 with contextlib
.closing(fileinput
.FileInput(
3243 [info_filename
], mode
='r',
3244 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
3245 # FileInput doesn't have a read method, we can't call json.load
3246 info
= self
.sanitize_info(json
.loads('\n'.join(f
)), self
.params
.get('clean_infojson', True))
3248 self
.__download
_wrapper
(self
.process_ie_result
)(info
, download
=True)
3249 except (DownloadError
, EntryNotInPlaylist
, ReExtractInfo
) as e
:
3250 if not isinstance(e
, EntryNotInPlaylist
):
3251 self
.to_stderr('\r')
3252 webpage_url
= info
.get('webpage_url')
3253 if webpage_url
is not None:
3254 self
.report_warning(f
'The info failed to download: {e}; trying with URL {webpage_url}')
3255 return self
.download([webpage_url
])
3258 return self
._download
_retcode
3261 def sanitize_info(info_dict
, remove_private_keys
=False):
3262 ''' Sanitize the infodict for converting to json '''
3263 if info_dict
is None:
3265 info_dict
.setdefault('epoch', int(time
.time()))
3266 info_dict
.setdefault('_type', 'video')
3268 if remove_private_keys
:
3269 reject
= lambda k
, v
: v
is None or (k
.startswith('_') and k
!= '_type') or k
in {
3270 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
3271 'entries', 'filepath', 'infojson_filename', 'original_url', 'playlist_autonumber',
3274 reject
= lambda k
, v
: False
3277 if isinstance(obj
, dict):
3278 return {k: filter_fn(v) for k, v in obj.items() if not reject(k, v)}
3279 elif isinstance(obj
, (list, tuple, set, LazyList
)):
3280 return list(map(filter_fn
, obj
))
3281 elif obj
is None or isinstance(obj
, (str, int, float, bool)):
3286 return filter_fn(info_dict
)
3289 def filter_requested_info(info_dict
, actually_filter
=True):
3290 ''' Alias of sanitize_info for backward compatibility '''
3291 return YoutubeDL
.sanitize_info(info_dict
, actually_filter
)
3294 def post_extract(info_dict
):
3295 def actual_post_extract(info_dict
):
3296 if info_dict
.get('_type') in ('playlist', 'multi_video'):
3297 for video_dict
in info_dict
.get('entries', {}):
3298 actual_post_extract(video_dict
or {})
3301 post_extractor
= info_dict
.pop('__post_extractor', None) or (lambda: {})
3302 info_dict
.update(post_extractor())
3304 actual_post_extract(info_dict
or {})
3306 def run_pp(self
, pp
, infodict
):
3307 files_to_delete
= []
3308 if '__files_to_move' not in infodict
:
3309 infodict
['__files_to_move'] = {}
3311 files_to_delete
, infodict
= pp
.run(infodict
)
3312 except PostProcessingError
as e
:
3313 # Must be True and not 'only_download'
3314 if self
.params
.get('ignoreerrors') is True:
3315 self
.report_error(e
)
3319 if not files_to_delete
:
3321 if self
.params
.get('keepvideo', False):
3322 for f
in files_to_delete
:
3323 infodict
['__files_to_move'].setdefault(f
, '')
3325 for old_filename
in set(files_to_delete
):
3326 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
3328 os
.remove(encodeFilename(old_filename
))
3330 self
.report_warning('Unable to remove downloaded original file')
3331 if old_filename
in infodict
['__files_to_move']:
3332 del infodict
['__files_to_move'][old_filename
]
3335 def run_all_pps(self
, key
, info
, *, additional_pps
=None):
3336 self
._forceprint
(key
, info
)
3337 for pp
in (additional_pps
or []) + self
._pps
[key
]:
3338 info
= self
.run_pp(pp
, info
)
3341 def pre_process(self
, ie_info
, key
='pre_process', files_to_move
=None):
3342 info
= dict(ie_info
)
3343 info
['__files_to_move'] = files_to_move
or {}
3344 info
= self
.run_all_pps(key
, info
)
3345 return info
, info
.pop('__files_to_move', None)
3347 def post_process(self
, filename
, info
, files_to_move
=None):
3348 """Run all the postprocessors on the given file."""
3349 info
['filepath'] = filename
3350 info
['__files_to_move'] = files_to_move
or {}
3351 info
= self
.run_all_pps('post_process', info
, additional_pps
=info
.get('__postprocessors'))
3352 info
= self
.run_pp(MoveFilesAfterDownloadPP(self
), info
)
3353 del info
['__files_to_move']
3354 return self
.run_all_pps('after_move', info
)
3356 def _make_archive_id(self
, info_dict
):
3357 video_id
= info_dict
.get('id')
3360 # Future-proof against any change in case
3361 # and backwards compatibility with prior versions
3362 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
3363 if extractor
is None:
3364 url
= str_or_none(info_dict
.get('url'))
3367 # Try to find matching extractor for the URL and take its ie_key
3368 for ie_key
, ie
in self
._ies
.items():
3369 if ie
.suitable(url
):
3374 return f
'{extractor.lower()} {video_id}'
3376 def in_download_archive(self
, info_dict
):
3377 fn
= self
.params
.get('download_archive')
3381 vid_id
= self
._make
_archive
_id
(info_dict
)
3383 return False # Incomplete video information
3385 return vid_id
in self
.archive
3387 def record_download_archive(self
, info_dict
):
3388 fn
= self
.params
.get('download_archive')
3391 vid_id
= self
._make
_archive
_id
(info_dict
)
3393 self
.write_debug(f
'Adding to archive: {vid_id}')
3394 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
3395 archive_file
.write(vid_id
+ '\n')
3396 self
.archive
.add(vid_id
)
3399 def format_resolution(format
, default
='unknown'):
3400 if format
.get('vcodec') == 'none' and format
.get('acodec') != 'none':
3402 if format
.get('resolution') is not None:
3403 return format
['resolution']
3404 if format
.get('width') and format
.get('height'):
3405 return '%dx%d' % (format
['width'], format
['height'])
3406 elif format
.get('height'):
3407 return '%sp' % format
['height']
3408 elif format
.get('width'):
3409 return '%dx?' % format
['width']
3412 def _list_format_headers(self
, *headers
):
3413 if self
.params
.get('listformats_table', True) is not False:
3414 return [self
._format
_screen
(header
, self
.Styles
.HEADERS
) for header
in headers
]
3417 def _format_note(self
, fdict
):
3419 if fdict
.get('ext') in ['f4f', 'f4m']:
3420 res
+= '(unsupported)'
3421 if fdict
.get('language'):
3424 res
+= '[%s]' % fdict
['language']
3425 if fdict
.get('format_note') is not None:
3428 res
+= fdict
['format_note']
3429 if fdict
.get('tbr') is not None:
3432 res
+= '%4dk' % fdict
['tbr']
3433 if fdict
.get('container') is not None:
3436 res
+= '%s container' % fdict
['container']
3437 if (fdict
.get('vcodec') is not None
3438 and fdict
.get('vcodec') != 'none'):
3441 res
+= fdict
['vcodec']
3442 if fdict
.get('vbr') is not None:
3444 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
3446 if fdict
.get('vbr') is not None:
3447 res
+= '%4dk' % fdict
['vbr']
3448 if fdict
.get('fps') is not None:
3451 res
+= '%sfps' % fdict
['fps']
3452 if fdict
.get('acodec') is not None:
3455 if fdict
['acodec'] == 'none':
3458 res
+= '%-5s' % fdict
['acodec']
3459 elif fdict
.get('abr') is not None:
3463 if fdict
.get('abr') is not None:
3464 res
+= '@%3dk' % fdict
['abr']
3465 if fdict
.get('asr') is not None:
3466 res
+= ' (%5dHz)' % fdict
['asr']
3467 if fdict
.get('filesize') is not None:
3470 res
+= format_bytes(fdict
['filesize'])
3471 elif fdict
.get('filesize_approx') is not None:
3474 res
+= '~' + format_bytes(fdict
['filesize_approx'])
3477 def render_formats_table(self
, info_dict
):
3478 if not info_dict
.get('formats') and not info_dict
.get('url'):
3481 formats
= info_dict
.get('formats', [info_dict
])
3482 if not self
.params
.get('listformats_table', True) is not False:
3485 format_field(f
, 'format_id'),
3486 format_field(f
, 'ext'),
3487 self
.format_resolution(f
),
3488 self
._format
_note
(f
)
3489 ] for f
in formats
if f
.get('preference') is None or f
['preference'] >= -1000]
3490 return render_table(['format code', 'extension', 'resolution', 'note'], table
, extra_gap
=1)
3492 delim
= self
._format
_screen
('\u2502', self
.Styles
.DELIM
, '|', test_encoding
=True)
3495 self
._format
_screen
(format_field(f
, 'format_id'), self
.Styles
.ID
),
3496 format_field(f
, 'ext'),
3497 format_field(f
, func
=self
.format_resolution
, ignore
=('audio only', 'images')),
3498 format_field(f
, 'fps', '\t%d'),
3499 format_field(f
, 'dynamic_range', '%s', ignore
=(None, 'SDR')).replace('HDR', ''),
3501 format_field(f
, 'filesize', ' \t%s', func
=format_bytes
) + format_field(f
, 'filesize_approx', '~\t%s', func
=format_bytes
),
3502 format_field(f
, 'tbr', '\t%dk'),
3503 shorten_protocol_name(f
.get('protocol', '')),
3505 format_field(f
, 'vcodec', default
='unknown').replace(
3506 'none', 'images' if f
.get('acodec') == 'none'
3507 else self
._format
_screen
('audio only', self
.Styles
.SUPPRESS
)),
3508 format_field(f
, 'vbr', '\t%dk'),
3509 format_field(f
, 'acodec', default
='unknown').replace(
3510 'none', '' if f
.get('vcodec') == 'none'
3511 else self
._format
_screen
('video only', self
.Styles
.SUPPRESS
)),
3512 format_field(f
, 'abr', '\t%dk'),
3513 format_field(f
, 'asr', '\t%dHz'),
3515 self
._format
_screen
('UNSUPPORTED', 'light red') if f
.get('ext') in ('f4f', 'f4m') else None,
3516 format_field(f
, 'language', '[%s]'),
3517 join_nonempty(format_field(f
, 'format_note'),
3518 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
3521 ] for f
in formats
if f
.get('preference') is None or f
['preference'] >= -1000]
3522 header_line
= self
._list
_format
_headers
(
3523 'ID', 'EXT', 'RESOLUTION', '\tFPS', 'HDR', delim
, '\tFILESIZE', '\tTBR', 'PROTO',
3524 delim
, 'VCODEC', '\tVBR', 'ACODEC', '\tABR', '\tASR', 'MORE INFO')
3526 return render_table(
3527 header_line
, table
, hide_empty
=True,
3528 delim
=self
._format
_screen
('\u2500', self
.Styles
.DELIM
, '-', test_encoding
=True))
3530 def render_thumbnails_table(self
, info_dict
):
3531 thumbnails
= list(info_dict
.get('thumbnails') or [])
3534 return render_table(
3535 self
._list
_format
_headers
('ID', 'Width', 'Height', 'URL'),
3536 [[t
.get('id'), t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
])
3538 def render_subtitles_table(self
, video_id
, subtitles
):
3539 def _row(lang
, formats
):
3540 exts
, names
= zip(*((f
['ext'], f
.get('name') or 'unknown') for f
in reversed(formats
)))
3541 if len(set(names
)) == 1:
3542 names
= [] if names
[0] == 'unknown' else names
[:1]
3543 return [lang
, ', '.join(names
), ', '.join(exts
)]
3547 return render_table(
3548 self
._list
_format
_headers
('Language', 'Name', 'Formats'),
3549 [_row(lang
, formats
) for lang
, formats
in subtitles
.items()],
3552 def __list_table(self
, video_id
, name
, func
, *args
):
3555 self
.to_screen(f
'{video_id} has no {name}')
3557 self
.to_screen(f
'[info] Available {name} for {video_id}:')
3558 self
.to_stdout(table
)
3560 def list_formats(self
, info_dict
):
3561 self
.__list
_table
(info_dict
['id'], 'formats', self
.render_formats_table
, info_dict
)
3563 def list_thumbnails(self
, info_dict
):
3564 self
.__list
_table
(info_dict
['id'], 'thumbnails', self
.render_thumbnails_table
, info_dict
)
3566 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
3567 self
.__list
_table
(video_id
, name
, self
.render_subtitles_table
, video_id
, subtitles
)
3569 def urlopen(self
, req
):
3570 """ Start an HTTP download """
3571 if isinstance(req
, str):
3572 req
= sanitized_Request(req
)
3573 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
3575 def print_debug_header(self
):
3576 if not self
.params
.get('verbose'):
3579 def get_encoding(stream
):
3580 ret
= str(getattr(stream
, 'encoding', 'missing (%s)' % type(stream
).__name
__))
3581 if not supports_terminal_sequences(stream
):
3582 from .compat
import WINDOWS_VT_MODE
3583 ret
+= ' (No VT)' if WINDOWS_VT_MODE
is False else ' (No ANSI)'
3586 encoding_str
= 'Encodings: locale %s, fs %s, out %s, err %s, pref %s' % (
3587 locale
.getpreferredencoding(),
3588 sys
.getfilesystemencoding(),
3589 get_encoding(self
._out
_files
['screen']), get_encoding(self
._out
_files
['error']),
3590 self
.get_encoding())
3592 logger
= self
.params
.get('logger')
3594 write_debug
= lambda msg
: logger
.debug(f
'[debug] {msg}')
3595 write_debug(encoding_str
)
3597 write_string(f
'[debug] {encoding_str}\n', encoding
=None)
3598 write_debug
= lambda msg
: self
._write
_string
(f
'[debug] {msg}\n')
3600 source
= detect_variant()
3601 write_debug(join_nonempty(
3602 'yt-dlp version', __version__
,
3603 f
'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD
else '',
3604 '' if source
== 'unknown' else f
'({source})',
3606 if not _LAZY_LOADER
:
3607 if os
.environ
.get('YTDLP_NO_LAZY_EXTRACTORS'):
3608 write_debug('Lazy loading extractors is forcibly disabled')
3610 write_debug('Lazy loading extractors is disabled')
3611 if plugin_extractors
or plugin_postprocessors
:
3612 write_debug('Plugins: %s' % [
3613 '%s%s' % (klass
.__name
__, '' if klass
.__name
__ == name
else f
' as {name}')
3614 for name
, klass
in itertools
.chain(plugin_extractors
.items(), plugin_postprocessors
.items())])
3615 if self
.params
.get('compat_opts'):
3616 write_debug('Compatibility options: %s' % ', '.join(self
.params
.get('compat_opts')))
3618 if source
== 'source':
3621 ['git', 'rev-parse', '--short', 'HEAD'],
3622 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
3623 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
3624 out
, err
= sp
.communicate_or_kill()
3625 out
= out
.decode().strip()
3626 if re
.match('[0-9a-f]+', out
):
3627 write_debug('Git HEAD: %s' % out
)
3629 with contextlib
.suppress(Exception):
3632 def python_implementation():
3633 impl_name
= platform
.python_implementation()
3634 if impl_name
== 'PyPy' and hasattr(sys
, 'pypy_version_info'):
3635 return impl_name
+ ' version %d.%d.%d' % sys
.pypy_version_info
[:3]
3638 write_debug('Python version %s (%s %s) - %s' % (
3639 platform
.python_version(),
3640 python_implementation(),
3641 platform
.architecture()[0],
3644 exe_versions
, ffmpeg_features
= FFmpegPostProcessor
.get_versions_and_features(self
)
3645 ffmpeg_features
= {key for key, val in ffmpeg_features.items() if val}
3647 exe_versions
['ffmpeg'] += ' (%s)' % ','.join(sorted(ffmpeg_features
))
3649 exe_versions
['rtmpdump'] = rtmpdump_version()
3650 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
3651 exe_str
= ', '.join(
3652 f
'{exe} {v}' for exe
, v
in sorted(exe_versions
.items()) if v
3654 write_debug('exe versions: %s' % exe_str
)
3656 from .dependencies
import available_dependencies
3658 write_debug('Optional libraries: %s' % (', '.join(sorted({
3659 module
.__name
__.split('.')[0] for module
in available_dependencies
.values()
3662 self
._setup
_opener
()
3664 for handler
in self
._opener
.handlers
:
3665 if hasattr(handler
, 'proxies'):
3666 proxy_map
.update(handler
.proxies
)
3667 write_debug(f
'Proxy map: {proxy_map}')
3670 if False and self
.params
.get('call_home'):
3671 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
3672 write_debug('Public IP address: %s' % ipaddr
)
3673 latest_version
= self
.urlopen(
3674 'https://yt-dl.org/latest/version').read().decode('utf-8')
3675 if version_tuple(latest_version
) > version_tuple(__version__
):
3676 self
.report_warning(
3677 'You are using an outdated version (newest version: %s)! '
3678 'See https://yt-dl.org/update if you need help updating.' %
3681 def _setup_opener(self
):
3682 if hasattr(self
, '_opener'):
3684 timeout_val
= self
.params
.get('socket_timeout')
3685 self
._socket
_timeout
= 20 if timeout_val
is None else float(timeout_val
)
3687 opts_cookiesfrombrowser
= self
.params
.get('cookiesfrombrowser')
3688 opts_cookiefile
= self
.params
.get('cookiefile')
3689 opts_proxy
= self
.params
.get('proxy')
3691 self
.cookiejar
= load_cookies(opts_cookiefile
, opts_cookiesfrombrowser
, self
)
3693 cookie_processor
= YoutubeDLCookieProcessor(self
.cookiejar
)
3694 if opts_proxy
is not None:
3695 if opts_proxy
== '':
3698 proxies
= {'http': opts_proxy, 'https': opts_proxy}
3700 proxies
= compat_urllib_request
.getproxies()
3701 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
3702 if 'http' in proxies
and 'https' not in proxies
:
3703 proxies
['https'] = proxies
['http']
3704 proxy_handler
= PerRequestProxyHandler(proxies
)
3706 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
3707 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
3708 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
3709 redirect_handler
= YoutubeDLRedirectHandler()
3710 data_handler
= urllib
.request
.DataHandler()
3712 # When passing our own FileHandler instance, build_opener won't add the
3713 # default FileHandler and allows us to disable the file protocol, which
3714 # can be used for malicious purposes (see
3715 # https://github.com/ytdl-org/youtube-dl/issues/8227)
3716 file_handler
= compat_urllib_request
.FileHandler()
3718 def file_open(*args
, **kwargs
):
3719 raise compat_urllib_error
.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
3720 file_handler
.file_open
= file_open
3722 opener
= compat_urllib_request
.build_opener(
3723 proxy_handler
, https_handler
, cookie_processor
, ydlh
, redirect_handler
, data_handler
, file_handler
)
3725 # Delete the default user-agent header, which would otherwise apply in
3726 # cases where our custom HTTP handler doesn't come into play
3727 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
3728 opener
.addheaders
= []
3729 self
._opener
= opener
3731 def encode(self
, s
):
3732 if isinstance(s
, bytes):
3733 return s
# Already encoded
3736 return s
.encode(self
.get_encoding())
3737 except UnicodeEncodeError as err
:
3738 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
3741 def get_encoding(self
):
3742 encoding
= self
.params
.get('encoding')
3743 if encoding
is None:
3744 encoding
= preferredencoding()
3747 def _write_info_json(self
, label
, ie_result
, infofn
, overwrite
=None):
3748 ''' Write infojson and returns True = written, 'exists' = Already exists, False = skip, None = error '''
3749 if overwrite
is None:
3750 overwrite
= self
.params
.get('overwrites', True)
3751 if not self
.params
.get('writeinfojson'):
3754 self
.write_debug(f
'Skipping writing {label} infojson')
3756 elif not self
._ensure
_dir
_exists
(infofn
):
3758 elif not overwrite
and os
.path
.exists(infofn
):
3759 self
.to_screen(f
'[info] {label.title()} metadata is already present')
3762 self
.to_screen(f
'[info] Writing {label} metadata as JSON to: {infofn}')
3764 write_json_file(self
.sanitize_info(ie_result
, self
.params
.get('clean_infojson', True)), infofn
)
3767 self
.report_error(f
'Cannot write {label} metadata to JSON file {infofn}')
3770 def _write_description(self
, label
, ie_result
, descfn
):
3771 ''' Write description and returns True = written, False = skip, None = error '''
3772 if not self
.params
.get('writedescription'):
3775 self
.write_debug(f
'Skipping writing {label} description')
3777 elif not self
._ensure
_dir
_exists
(descfn
):
3779 elif not self
.params
.get('overwrites', True) and os
.path
.exists(descfn
):
3780 self
.to_screen(f
'[info] {label.title()} description is already present')
3781 elif ie_result
.get('description') is None:
3782 self
.report_warning(f
'There\'s no {label} description to write')
3786 self
.to_screen(f
'[info] Writing {label} description to: {descfn}')
3787 with open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
3788 descfile
.write(ie_result
['description'])
3790 self
.report_error(f
'Cannot write {label} description file {descfn}')
3794 def _write_subtitles(self
, info_dict
, filename
):
3795 ''' Write subtitles to file and return list of (sub_filename, final_sub_filename); or None if error'''
3797 subtitles
= info_dict
.get('requested_subtitles')
3798 if not subtitles
or not (self
.params
.get('writesubtitles') or self
.params
.get('writeautomaticsub')):
3799 # subtitles download errors are already managed as troubles in relevant IE
3800 # that way it will silently go on when used with unsupporting IE
3803 sub_filename_base
= self
.prepare_filename(info_dict
, 'subtitle')
3804 if not sub_filename_base
:
3805 self
.to_screen('[info] Skipping writing video subtitles')
3807 for sub_lang
, sub_info
in subtitles
.items():
3808 sub_format
= sub_info
['ext']
3809 sub_filename
= subtitles_filename(filename
, sub_lang
, sub_format
, info_dict
.get('ext'))
3810 sub_filename_final
= subtitles_filename(sub_filename_base
, sub_lang
, sub_format
, info_dict
.get('ext'))
3811 existing_sub
= self
.existing_file((sub_filename_final
, sub_filename
))
3813 self
.to_screen(f
'[info] Video subtitle {sub_lang}.{sub_format} is already present')
3814 sub_info
['filepath'] = existing_sub
3815 ret
.append((existing_sub
, sub_filename_final
))
3818 self
.to_screen(f
'[info] Writing video subtitles to: {sub_filename}')
3819 if sub_info
.get('data') is not None:
3821 # Use newline='' to prevent conversion of newline characters
3822 # See https://github.com/ytdl-org/youtube-dl/issues/10268
3823 with open(sub_filename
, 'w', encoding
='utf-8', newline
='') as subfile
:
3824 subfile
.write(sub_info
['data'])
3825 sub_info
['filepath'] = sub_filename
3826 ret
.append((sub_filename
, sub_filename_final
))
3829 self
.report_error(f
'Cannot write video subtitles file {sub_filename}')
3833 sub_copy
= sub_info
.copy()
3834 sub_copy
.setdefault('http_headers', info_dict
.get('http_headers'))
3835 self
.dl(sub_filename
, sub_copy
, subtitle
=True)
3836 sub_info
['filepath'] = sub_filename
3837 ret
.append((sub_filename
, sub_filename_final
))
3838 except (DownloadError
, ExtractorError
, IOError, OSError, ValueError) + network_exceptions
as err
:
3839 msg
= f
'Unable to download video subtitles for {sub_lang!r}: {err}'
3840 if self
.params
.get('ignoreerrors') is not True: # False or 'only_download'
3841 if not self
.params
.get('ignoreerrors'):
3842 self
.report_error(msg
)
3843 raise DownloadError(msg
)
3844 self
.report_warning(msg
)
3847 def _write_thumbnails(self
, label
, info_dict
, filename
, thumb_filename_base
=None):
3848 ''' Write thumbnails to file and return list of (thumb_filename, final_thumb_filename) '''
3849 write_all
= self
.params
.get('write_all_thumbnails', False)
3850 thumbnails
, ret
= [], []
3851 if write_all
or self
.params
.get('writethumbnail', False):
3852 thumbnails
= info_dict
.get('thumbnails') or []
3853 multiple
= write_all
and len(thumbnails
) > 1
3855 if thumb_filename_base
is None:
3856 thumb_filename_base
= filename
3857 if thumbnails
and not thumb_filename_base
:
3858 self
.write_debug(f
'Skipping writing {label} thumbnail')
3861 for idx
, t
in list(enumerate(thumbnails
))[::-1]:
3862 thumb_ext
= (f
'{t["id"]}.' if multiple
else '') + determine_ext(t
['url'], 'jpg')
3863 thumb_display_id
= f
'{label} thumbnail {t["id"]}'
3864 thumb_filename
= replace_extension(filename
, thumb_ext
, info_dict
.get('ext'))
3865 thumb_filename_final
= replace_extension(thumb_filename_base
, thumb_ext
, info_dict
.get('ext'))
3867 existing_thumb
= self
.existing_file((thumb_filename_final
, thumb_filename
))
3869 self
.to_screen('[info] %s is already present' % (
3870 thumb_display_id
if multiple
else f
'{label} thumbnail').capitalize())
3871 t
['filepath'] = existing_thumb
3872 ret
.append((existing_thumb
, thumb_filename_final
))
3874 self
.to_screen(f
'[info] Downloading {thumb_display_id} ...')
3876 uf
= self
.urlopen(sanitized_Request(t
['url'], headers
=t
.get('http_headers', {})))
3877 self
.to_screen(f
'[info] Writing {thumb_display_id} to: {thumb_filename}')
3878 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
3879 shutil
.copyfileobj(uf
, thumbf
)
3880 ret
.append((thumb_filename
, thumb_filename_final
))
3881 t
['filepath'] = thumb_filename
3882 except network_exceptions
as err
:
3884 self
.report_warning(f
'Unable to download {thumb_display_id}: {err}')
3885 if ret
and not write_all
: