4 from __future__
import absolute_import
, unicode_literals
28 from string
import ascii_letters
29 from zipimport
import zipimporter
34 compat_get_terminal_size
,
39 compat_tokenize_tokenize
,
41 compat_urllib_request
,
42 compat_urllib_request_DataHandler
,
54 DOT_DESKTOP_LINK_TEMPLATE
,
55 DOT_URL_LINK_TEMPLATE
,
56 DOT_WEBLOC_LINK_TEMPLATE
,
82 PerRequestProxyHandler
,
88 register_socks_protocols
,
103 UnavailableVideoError
,
109 YoutubeDLCookieProcessor
,
111 YoutubeDLRedirectHandler
,
112 process_communicate_or_kill
,
114 from .cache
import Cache
115 from .extractor
import (
116 gen_extractor_classes
,
121 from .extractor
.openload
import PhantomJSwrapper
122 from .downloader
import (
123 get_suitable_downloader
,
124 shorten_protocol_name
126 from .downloader
.rtmp
import rtmpdump_version
127 from .postprocessor
import (
130 FFmpegFixupStretchedPP
,
133 # FFmpegSubtitlesConvertorPP,
135 MoveFilesAfterDownloadPP
,
137 from .version
import __version__
139 if compat_os_name
== 'nt':
143 class YoutubeDL(object):
146 YoutubeDL objects are the ones responsible of downloading the
147 actual video file and writing it to disk if the user has requested
148 it, among some other tasks. In most cases there should be one per
149 program. As, given a video URL, the downloader doesn't know how to
150 extract all the needed information, task that InfoExtractors do, it
151 has to pass the URL to one of them.
153 For this, YoutubeDL objects have a method that allows
154 InfoExtractors to be registered in a given order. When it is passed
155 a URL, the YoutubeDL object handles it to the first InfoExtractor it
156 finds that reports being able to handle it. The InfoExtractor extracts
157 all the information about the video or videos the URL refers to, and
158 YoutubeDL process the extracted information, possibly using a File
159 Downloader to download the video.
161 YoutubeDL objects accept a lot of parameters. In order not to saturate
162 the object constructor with arguments, it receives a dictionary of
163 options instead. These options are available through the params
164 attribute for the InfoExtractors to use. The YoutubeDL also
165 registers itself as the downloader in charge for the InfoExtractors
166 that are added to it, so this is a "mutual registration".
170 username: Username for authentication purposes.
171 password: Password for authentication purposes.
172 videopassword: Password for accessing a video.
173 ap_mso: Adobe Pass multiple-system operator identifier.
174 ap_username: Multiple-system operator account username.
175 ap_password: Multiple-system operator account password.
176 usenetrc: Use netrc for authentication instead.
177 verbose: Print additional info to stdout.
178 quiet: Do not print messages to stdout.
179 no_warnings: Do not print out anything for warnings.
180 forceurl: Force printing final URL.
181 forcetitle: Force printing title.
182 forceid: Force printing ID.
183 forcethumbnail: Force printing thumbnail URL.
184 forcedescription: Force printing description.
185 forcefilename: Force printing final filename.
186 forceduration: Force printing duration.
187 forcejson: Force printing info_dict as JSON.
188 dump_single_json: Force printing the info_dict of the whole playlist
189 (or video) as a single JSON line.
190 force_write_download_archive: Force writing download archive regardless
191 of 'skip_download' or 'simulate'.
192 simulate: Do not download the video files.
193 format: Video format code. see "FORMAT SELECTION" for more details.
194 allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
195 ignore_no_formats_error: Ignore "No video formats" error. Usefull for
196 extracting metadata even if the video is not actually
197 available for download (experimental)
198 format_sort: How to sort the video formats. see "Sorting Formats"
200 format_sort_force: Force the given format_sort. see "Sorting Formats"
202 allow_multiple_video_streams: Allow multiple video streams to be merged
204 allow_multiple_audio_streams: Allow multiple audio streams to be merged
206 paths: Dictionary of output paths. The allowed keys are 'home'
207 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
208 outtmpl: Dictionary of templates for output names. Allowed keys
209 are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
210 A string a also accepted for backward compatibility
211 outtmpl_na_placeholder: Placeholder for unavailable meta fields.
212 restrictfilenames: Do not allow "&" and spaces in file names
213 trim_file_name: Limit length of filename (extension excluded)
214 windowsfilenames: Force the filenames to be windows compatible
215 ignoreerrors: Do not stop on download errors
216 (Default True when running yt-dlp,
217 but False when directly accessing YoutubeDL class)
218 skip_playlist_after_errors: Number of allowed failures until the rest of
219 the playlist is skipped
220 force_generic_extractor: Force downloader to use the generic extractor
221 overwrites: Overwrite all video and metadata files if True,
222 overwrite only non-video files if None
223 and don't overwrite any file if False
224 playliststart: Playlist item to start at.
225 playlistend: Playlist item to end at.
226 playlist_items: Specific indices of playlist to download.
227 playlistreverse: Download playlist items in reverse order.
228 playlistrandom: Download playlist items in random order.
229 matchtitle: Download only matching titles.
230 rejecttitle: Reject downloads for matching titles.
231 logger: Log messages to a logging.Logger instance.
232 logtostderr: Log messages to stderr instead of stdout.
233 writedescription: Write the video description to a .description file
234 writeinfojson: Write the video description to a .info.json file
235 clean_infojson: Remove private fields from the infojson
236 writecomments: Extract video comments. This will not be written to disk
237 unless writeinfojson is also given
238 writeannotations: Write the video annotations to a .annotations.xml file
239 writethumbnail: Write the thumbnail image to a file
240 allow_playlist_files: Whether to write playlists' description, infojson etc
241 also to disk when using the 'write*' options
242 write_all_thumbnails: Write all thumbnail formats to files
243 writelink: Write an internet shortcut file, depending on the
244 current platform (.url/.webloc/.desktop)
245 writeurllink: Write a Windows internet shortcut file (.url)
246 writewebloclink: Write a macOS internet shortcut file (.webloc)
247 writedesktoplink: Write a Linux internet shortcut file (.desktop)
248 writesubtitles: Write the video subtitles to a file
249 writeautomaticsub: Write the automatically generated subtitles to a file
250 allsubtitles: Deprecated - Use subtitlelangs = ['all']
251 Downloads all the subtitles of the video
252 (requires writesubtitles or writeautomaticsub)
253 listsubtitles: Lists all available subtitles for the video
254 subtitlesformat: The format code for subtitles
255 subtitleslangs: List of languages of the subtitles to download (can be regex).
256 The list may contain "all" to refer to all the available
257 subtitles. The language can be prefixed with a "-" to
258 exclude it from the requested languages. Eg: ['all', '-live_chat']
259 keepvideo: Keep the video file after post-processing
260 daterange: A DateRange object, download only if the upload_date is in the range.
261 skip_download: Skip the actual download of the video file
262 cachedir: Location of the cache files in the filesystem.
263 False to disable filesystem cache.
264 noplaylist: Download single video instead of a playlist if in doubt.
265 age_limit: An integer representing the user's age in years.
266 Unsuitable videos for the given age are skipped.
267 min_views: An integer representing the minimum view count the video
268 must have in order to not be skipped.
269 Videos without view count information are always
270 downloaded. None for no limit.
271 max_views: An integer representing the maximum view count.
272 Videos that are more popular than that are not
274 Videos without view count information are always
275 downloaded. None for no limit.
276 download_archive: File name of a file where all downloads are recorded.
277 Videos already present in the file are not downloaded
279 break_on_existing: Stop the download process after attempting to download a
280 file that is in the archive.
281 break_on_reject: Stop the download process when encountering a video that
282 has been filtered out.
283 cookiefile: File name where cookies should be read from and dumped to
284 nocheckcertificate:Do not verify SSL certificates
285 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
286 At the moment, this is only supported by YouTube.
287 proxy: URL of the proxy server to use
288 geo_verification_proxy: URL of the proxy to use for IP address verification
289 on geo-restricted sites.
290 socket_timeout: Time to wait for unresponsive hosts, in seconds
291 bidi_workaround: Work around buggy terminals without bidirectional text
292 support, using fridibi
293 debug_printtraffic:Print out sent and received HTTP traffic
294 include_ads: Download ads as well
295 default_search: Prepend this string if an input url is not valid.
296 'auto' for elaborate guessing
297 encoding: Use this encoding instead of the system-specified.
298 extract_flat: Do not resolve URLs, return the immediate result.
299 Pass in 'in_playlist' to only show this behavior for
301 postprocessors: A list of dictionaries, each with an entry
302 * key: The name of the postprocessor. See
303 yt_dlp/postprocessor/__init__.py for a list.
304 * when: When to run the postprocessor. Can be one of
305 pre_process|before_dl|post_process|after_move.
306 Assumed to be 'post_process' if not given
307 post_hooks: A list of functions that get called as the final step
308 for each video file, after all postprocessors have been
309 called. The filename will be passed as the only argument.
310 progress_hooks: A list of functions that get called on download
311 progress, with a dictionary with the entries
312 * status: One of "downloading", "error", or "finished".
313 Check this first and ignore unknown values.
315 If status is one of "downloading", or "finished", the
316 following properties may also be present:
317 * filename: The final filename (always present)
318 * tmpfilename: The filename we're currently writing to
319 * downloaded_bytes: Bytes on disk
320 * total_bytes: Size of the whole file, None if unknown
321 * total_bytes_estimate: Guess of the eventual file size,
323 * elapsed: The number of seconds since download started.
324 * eta: The estimated time in seconds, None if unknown
325 * speed: The download speed in bytes/second, None if
327 * fragment_index: The counter of the currently
328 downloaded video fragment.
329 * fragment_count: The number of fragments (= individual
330 files that will be merged)
332 Progress hooks are guaranteed to be called at least once
333 (with status "finished") if the download is successful.
334 merge_output_format: Extension to use when merging formats.
335 final_ext: Expected final extension; used to detect when the file was
336 already downloaded and converted. "merge_output_format" is
337 replaced by this extension when given
338 fixup: Automatically correct known faults of the file.
340 - "never": do nothing
341 - "warn": only emit a warning
342 - "detect_or_warn": check whether we can do anything
343 about it, warn otherwise (default)
344 source_address: Client-side IP address to bind to.
345 call_home: Boolean, true iff we are allowed to contact the
346 yt-dlp servers for debugging. (BROKEN)
347 sleep_interval_requests: Number of seconds to sleep between requests
349 sleep_interval: Number of seconds to sleep before each download when
350 used alone or a lower bound of a range for randomized
351 sleep before each download (minimum possible number
352 of seconds to sleep) when used along with
354 max_sleep_interval:Upper bound of a range for randomized sleep before each
355 download (maximum possible number of seconds to sleep).
356 Must only be used along with sleep_interval.
357 Actual sleep time will be a random float from range
358 [sleep_interval; max_sleep_interval].
359 sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
360 listformats: Print an overview of available video formats and exit.
361 list_thumbnails: Print a table of all thumbnails and exit.
362 match_filter: A function that gets called with the info_dict of
364 If it returns a message, the video is ignored.
365 If it returns None, the video is downloaded.
366 match_filter_func in utils.py is one example for this.
367 no_color: Do not emit color codes in output.
368 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
371 Two-letter ISO 3166-2 country code that will be used for
372 explicit geographic restriction bypassing via faking
373 X-Forwarded-For HTTP header
375 IP range in CIDR notation that will be used similarly to
378 The following options determine which downloader is picked:
379 external_downloader: A dictionary of protocol keys and the executable of the
380 external downloader to use for it. The allowed protocols
381 are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
382 Set the value to 'native' to use the native downloader
383 hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
384 or {'m3u8': 'ffmpeg'} instead.
385 Use the native HLS downloader instead of ffmpeg/avconv
386 if True, otherwise use ffmpeg/avconv if False, otherwise
387 use downloader suggested by extractor if None.
388 compat_opts: Compatibility options. See "Differences in default behavior".
389 Note that only format-sort, format-spec, no-live-chat, no-attach-info-json
390 playlist-index, list-formats, no-youtube-channel-redirect
391 and no-youtube-unavailable-videos works when used via the API
393 The following parameters are not used by YoutubeDL itself, they are used by
394 the downloader (see yt_dlp/downloader/common.py):
395 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
396 noresizebuffer, retries, continuedl, noprogress, consoletitle,
397 xattr_set_filesize, external_downloader_args, hls_use_mpegts,
400 The following options are used by the post processors:
401 prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
402 otherwise prefer ffmpeg. (avconv support is deprecated)
403 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
404 to the binary or its containing directory.
405 postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
406 and a list of additional command-line arguments for the
407 postprocessor/executable. The dict can also have "PP+EXE" keys
408 which are used when the given exe is used by the given PP.
409 Use 'default' as the name for arguments to passed to all PP
411 The following options are used by the extractors:
412 extractor_retries: Number of times to retry for known errors
413 dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
414 hls_split_discontinuity: Split HLS playlists to different formats at
415 discontinuities such as ad breaks (default: False)
416 youtube_include_dash_manifest: If True (default), DASH manifests and related
417 data will be downloaded and processed by extractor.
418 You can reduce network I/O by disabling it if you don't
419 care about DASH. (only for youtube)
420 youtube_include_hls_manifest: If True (default), HLS manifests and related
421 data will be downloaded and processed by extractor.
422 You can reduce network I/O by disabling it if you don't
423 care about HLS. (only for youtube)
426 _NUMERIC_FIELDS
= set((
427 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
428 'timestamp', 'upload_year', 'upload_month', 'upload_day',
429 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
430 'average_rating', 'comment_count', 'age_limit',
431 'start_time', 'end_time',
432 'chapter_number', 'season_number', 'episode_number',
433 'track_number', 'disc_number', 'release_year',
439 _pps
= {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
440 __prepare_filename_warned
= False
441 _first_webpage_request
= True
442 _download_retcode
= None
443 _num_downloads
= None
445 _playlist_urls
= set()
448 def __init__(self
, params
=None, auto_init
=True):
449 """Create a FileDownloader object with the given options."""
453 self
._ies
_instances
= {}
454 self
._pps
= {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
455 self
.__prepare
_filename
_warned
= False
456 self
._first
_webpage
_request
= True
457 self
._post
_hooks
= []
458 self
._progress
_hooks
= []
459 self
._download
_retcode
= 0
460 self
._num
_downloads
= 0
461 self
._screen
_file
= [sys
.stdout
, sys
.stderr
][params
.get('logtostderr', False)]
462 self
._err
_file
= sys
.stderr
465 'nocheckcertificate': False,
467 self
.params
.update(params
)
468 self
.cache
= Cache(self
)
470 if sys
.version_info
< (3, 6):
472 'Support for Python version %d.%d have been deprecated and will break in future versions of yt-dlp! '
473 'Update to Python 3.6 or above' % sys
.version_info
[:2])
475 def check_deprecated(param
, option
, suggestion
):
476 if self
.params
.get(param
) is not None:
477 self
.report_warning('%s is deprecated. Use %s instead' % (option
, suggestion
))
481 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
482 if self
.params
.get('geo_verification_proxy') is None:
483 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
485 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
486 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
487 check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
489 for msg
in self
.params
.get('warnings', []):
490 self
.report_warning(msg
)
492 if self
.params
.get('final_ext'):
493 if self
.params
.get('merge_output_format'):
494 self
.report_warning('--merge-output-format will be ignored since --remux-video or --recode-video is given')
495 self
.params
['merge_output_format'] = self
.params
['final_ext']
497 if 'overwrites' in self
.params
and self
.params
['overwrites'] is None:
498 del self
.params
['overwrites']
500 if params
.get('bidi_workaround', False):
503 master
, slave
= pty
.openpty()
504 width
= compat_get_terminal_size().columns
508 width_args
= ['-w', str(width
)]
510 stdin
=subprocess
.PIPE
,
512 stderr
=self
._err
_file
)
514 self
._output
_process
= subprocess
.Popen(
515 ['bidiv'] + width_args
, **sp_kwargs
518 self
._output
_process
= subprocess
.Popen(
519 ['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
520 self
._output
_channel
= os
.fdopen(master
, 'rb')
521 except OSError as ose
:
522 if ose
.errno
== errno
.ENOENT
:
523 self
.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
527 if (sys
.platform
!= 'win32'
528 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
529 and not params
.get('restrictfilenames', False)):
530 # Unicode filesystem API will throw errors (#1474, #13027)
532 'Assuming --restrict-filenames since file system encoding '
533 'cannot encode all characters. '
534 'Set the LC_ALL environment variable to fix this.')
535 self
.params
['restrictfilenames'] = True
537 self
.outtmpl_dict
= self
.parse_outtmpl()
541 """Preload the archive, if any is specified"""
542 def preload_download_archive(fn
):
545 if self
.params
.get('verbose'):
546 self
._write
_string
('[debug] Loading archive file %r\n' % fn
)
548 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
549 for line
in archive_file
:
550 self
.archive
.add(line
.strip())
551 except IOError as ioe
:
552 if ioe
.errno
!= errno
.ENOENT
:
558 preload_download_archive(self
.params
.get('download_archive'))
561 self
.print_debug_header()
562 self
.add_default_info_extractors()
564 for pp_def_raw
in self
.params
.get('postprocessors', []):
565 pp_class
= get_postprocessor(pp_def_raw
['key'])
566 pp_def
= dict(pp_def_raw
)
569 when
= pp_def
['when']
572 when
= 'post_process'
573 pp
= pp_class(self
, **compat_kwargs(pp_def
))
574 self
.add_post_processor(pp
, when
=when
)
576 for ph
in self
.params
.get('post_hooks', []):
577 self
.add_post_hook(ph
)
579 for ph
in self
.params
.get('progress_hooks', []):
580 self
.add_progress_hook(ph
)
582 register_socks_protocols()
584 def warn_if_short_id(self
, argv
):
585 # short YouTube ID starting with dash?
587 i
for i
, a
in enumerate(argv
)
588 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
592 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
593 + ['--'] + [argv
[i
] for i
in idxs
]
596 'Long argument string detected. '
597 'Use -- to separate parameters and URLs, like this:\n%s\n' %
598 args_to_str(correct_argv
))
600 def add_info_extractor(self
, ie
):
601 """Add an InfoExtractor object to the end of the list."""
603 if not isinstance(ie
, type):
604 self
._ies
_instances
[ie
.ie_key()] = ie
605 ie
.set_downloader(self
)
607 def get_info_extractor(self
, ie_key
):
609 Get an instance of an IE with name ie_key, it will try to get one from
610 the _ies list, if there's no instance it will create a new one and add
611 it to the extractor list.
613 ie
= self
._ies
_instances
.get(ie_key
)
615 ie
= get_info_extractor(ie_key
)()
616 self
.add_info_extractor(ie
)
619 def add_default_info_extractors(self
):
621 Add the InfoExtractors returned by gen_extractors to the end of the list
623 for ie
in gen_extractor_classes():
624 self
.add_info_extractor(ie
)
626 def add_post_processor(self
, pp
, when
='post_process'):
627 """Add a PostProcessor object to the end of the chain."""
628 self
._pps
[when
].append(pp
)
629 pp
.set_downloader(self
)
631 def add_post_hook(self
, ph
):
632 """Add the post hook"""
633 self
._post
_hooks
.append(ph
)
635 def add_progress_hook(self
, ph
):
636 """Add the progress hook (currently only for the file downloader)"""
637 self
._progress
_hooks
.append(ph
)
639 def _bidi_workaround(self
, message
):
640 if not hasattr(self
, '_output_channel'):
643 assert hasattr(self
, '_output_process')
644 assert isinstance(message
, compat_str
)
645 line_count
= message
.count('\n') + 1
646 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
647 self
._output
_process
.stdin
.flush()
648 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
649 for _
in range(line_count
))
650 return res
[:-len('\n')]
652 def to_screen(self
, message
, skip_eol
=False):
653 """Print message to stdout if not in quiet mode."""
654 return self
.to_stdout(
656 quiet
=self
.params
.get('quiet', False))
658 def _write_string(self
, s
, out
=None):
659 write_string(s
, out
=out
, encoding
=self
.params
.get('encoding'))
661 def to_stdout(self
, message
, skip_eol
=False, quiet
=False):
662 """Print message to stdout if not in quiet mode."""
663 if self
.params
.get('logger'):
664 self
.params
['logger'].debug(message
)
666 message
= self
._bidi
_workaround
(message
)
667 terminator
= ['\n', ''][skip_eol
]
668 output
= message
+ terminator
670 self
._write
_string
(output
, self
._screen
_file
)
672 def to_stderr(self
, message
):
673 """Print message to stderr."""
674 assert isinstance(message
, compat_str
)
675 if self
.params
.get('logger'):
676 self
.params
['logger'].error(message
)
678 message
= self
._bidi
_workaround
(message
)
679 output
= message
+ '\n'
680 self
._write
_string
(output
, self
._err
_file
)
682 def to_console_title(self
, message
):
683 if not self
.params
.get('consoletitle', False):
685 if compat_os_name
== 'nt':
686 if ctypes
.windll
.kernel32
.GetConsoleWindow():
687 # c_wchar_p() might not be necessary if `message` is
688 # already of type unicode()
689 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
690 elif 'TERM' in os
.environ
:
691 self
._write
_string
('\033]0;%s\007' % message
, self
._screen
_file
)
693 def save_console_title(self
):
694 if not self
.params
.get('consoletitle', False):
696 if self
.params
.get('simulate', False):
698 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
699 # Save the title on stack
700 self
._write
_string
('\033[22;0t', self
._screen
_file
)
702 def restore_console_title(self
):
703 if not self
.params
.get('consoletitle', False):
705 if self
.params
.get('simulate', False):
707 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
708 # Restore the title from stack
709 self
._write
_string
('\033[23;0t', self
._screen
_file
)
712 self
.save_console_title()
715 def __exit__(self
, *args
):
716 self
.restore_console_title()
718 if self
.params
.get('cookiefile') is not None:
719 self
.cookiejar
.save(ignore_discard
=True, ignore_expires
=True)
721 def trouble(self
, message
=None, tb
=None):
722 """Determine action to take when a download problem appears.
724 Depending on if the downloader has been configured to ignore
725 download errors or not, this method may throw an exception or
726 not when errors are found, after printing the message.
728 tb, if given, is additional traceback information.
730 if message
is not None:
731 self
.to_stderr(message
)
732 if self
.params
.get('verbose'):
734 if sys
.exc_info()[0]: # if .trouble has been called from an except block
736 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
737 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
738 tb
+= encode_compat_str(traceback
.format_exc())
740 tb_data
= traceback
.format_list(traceback
.extract_stack())
741 tb
= ''.join(tb_data
)
743 if not self
.params
.get('ignoreerrors', False):
744 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
745 exc_info
= sys
.exc_info()[1].exc_info
747 exc_info
= sys
.exc_info()
748 raise DownloadError(message
, exc_info
)
749 self
._download
_retcode
= 1
751 def report_warning(self
, message
):
753 Print the message to stderr, it will be prefixed with 'WARNING:'
754 If stderr is a tty file the 'WARNING:' will be colored
756 if self
.params
.get('logger') is not None:
757 self
.params
['logger'].warning(message
)
759 if self
.params
.get('no_warnings'):
761 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
762 _msg_header
= '\033[0;33mWARNING:\033[0m'
764 _msg_header
= 'WARNING:'
765 warning_message
= '%s %s' % (_msg_header
, message
)
766 self
.to_stderr(warning_message
)
768 def report_error(self
, message
, tb
=None):
770 Do the same as trouble, but prefixes the message with 'ERROR:', colored
771 in red if stderr is a tty file.
773 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
774 _msg_header
= '\033[0;31mERROR:\033[0m'
776 _msg_header
= 'ERROR:'
777 error_message
= '%s %s' % (_msg_header
, message
)
778 self
.trouble(error_message
, tb
)
780 def report_file_already_downloaded(self
, file_name
):
781 """Report file has already been fully downloaded."""
783 self
.to_screen('[download] %s has already been downloaded' % file_name
)
784 except UnicodeEncodeError:
785 self
.to_screen('[download] The file has already been downloaded')
787 def report_file_delete(self
, file_name
):
788 """Report that existing file will be deleted."""
790 self
.to_screen('Deleting existing file %s' % file_name
)
791 except UnicodeEncodeError:
792 self
.to_screen('Deleting existing file')
794 def parse_outtmpl(self
):
795 outtmpl_dict
= self
.params
.get('outtmpl', {})
796 if not isinstance(outtmpl_dict
, dict):
797 outtmpl_dict
= {'default': outtmpl_dict}
798 outtmpl_dict
.update({
799 k
: v
for k
, v
in DEFAULT_OUTTMPL
.items()
800 if not outtmpl_dict
.get(k
)})
801 for key
, val
in outtmpl_dict
.items():
802 if isinstance(val
, bytes):
804 'Parameter outtmpl is bytes, but should be a unicode string. '
805 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
808 def prepare_outtmpl(self
, outtmpl
, info_dict
, sanitize
=None):
809 """ Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
810 template_dict
= dict(info_dict
)
811 na
= self
.params
.get('outtmpl_na_placeholder', 'NA')
814 template_dict
['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
815 formatSeconds(info_dict
['duration'], '-')
816 if info_dict
.get('duration', None) is not None
820 template_dict
['epoch'] = int(time
.time())
823 autonumber_size
= self
.params
.get('autonumber_size')
824 if autonumber_size
is None:
826 template_dict
['autonumber'] = self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
828 # resolution if not defined
829 if template_dict
.get('resolution') is None:
830 if template_dict
.get('width') and template_dict
.get('height'):
831 template_dict
['resolution'] = '%dx%d' % (template_dict
['width'], template_dict
['height'])
832 elif template_dict
.get('height'):
833 template_dict
['resolution'] = '%sp' % template_dict
['height']
834 elif template_dict
.get('width'):
835 template_dict
['resolution'] = '%dx?' % template_dict
['width']
837 # For fields playlist_index and autonumber convert all occurrences
838 # of %(field)s to %(field)0Nd for backward compatibility
839 field_size_compat_map
= {
840 'playlist_index': len(str(template_dict
.get('_last_playlist_index') or '')),
841 'autonumber': autonumber_size
,
843 FIELD_SIZE_COMPAT_RE
= r
'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
844 mobj
= re
.search(FIELD_SIZE_COMPAT_RE
, outtmpl
)
847 FIELD_SIZE_COMPAT_RE
,
848 r
'%%(\1)0%dd' % field_size_compat_map
[mobj
.group('field')],
851 numeric_fields
= list(self
._NUMERIC
_FIELDS
)
853 sanitize
= lambda k
, v
: v
855 EXTERNAL_FORMAT_RE
= FORMAT_RE
.format('(?P<key>[^)]*)')
856 # Field is of the form key1.key2...
857 # where keys (except first) can be string, int or slice
858 FIELD_RE
= r
'\w+(?:\.(?:\w+|[-\d]*(?::[-\d]*){0,2}))*'
859 INTERNAL_FORMAT_RE
= re
.compile(r
'''(?x)
862 (?P<maths>(?:[-+]-?(?:\d+(?:\.\d+)?|{0}))*)
863 (?:>(?P<strf_format>.+?))?
864 (?:\|(?P<default>.*?))?
865 $'''.format(FIELD_RE
))
866 MATH_OPERATORS_RE
= re
.compile(r
'(?<![-+])([-+])')
871 for outer_mobj
in re
.finditer(EXTERNAL_FORMAT_RE
, outtmpl
):
872 final_key
= outer_mobj
.group('key')
873 str_type
= outer_mobj
.group('type')
875 mobj
= re
.match(INTERNAL_FORMAT_RE
, final_key
)
877 mobj
= mobj
.groupdict()
879 fields
= mobj
['fields'].split('.')
880 value
= traverse_dict(template_dict
, fields
)
883 value
= float_or_none(value
)
884 if value
is not None:
888 value
= float_or_none(value
)
890 for item
in MATH_OPERATORS_RE
.split(mobj
['maths'])[1:]:
896 item
, multiplier
= (item
[1:], -1) if item
[0] == '-' else (item
, 1)
897 offset
= float_or_none(item
)
899 offset
= float_or_none(traverse_dict(template_dict
, item
.split('.')))
901 value
= operator(value
, multiplier
* offset
)
902 except (TypeError, ZeroDivisionError):
906 operator
= MATH_FUNCTIONS
[item
]
907 # Datetime formatting
908 if mobj
['strf_format']:
909 value
= strftime_or_none(value
, mobj
['strf_format'])
911 if value
is None and mobj
['default'] is not None:
912 value
= mobj
['default']
914 if str_type
in 'crs' and value
is not None: # string
915 value
= sanitize('%{}'.format(str_type
) % fields
[-1], value
)
917 numeric_fields
.append(final_key
)
918 value
= float_or_none(value
)
919 if value
is not None:
920 template_dict
[final_key
] = value
922 # Missing numeric fields used together with integer presentation types
923 # in format specification will break the argument substitution since
924 # string NA placeholder is returned for missing fields. We will patch
925 # output template for missing fields to meet string presentation type.
926 for numeric_field
in numeric_fields
:
927 if template_dict
.get(numeric_field
) is None:
929 FORMAT_RE
.format(re
.escape(numeric_field
)),
930 r
'%({0})s'.format(numeric_field
), outtmpl
)
932 template_dict
= collections
.defaultdict(lambda: na
, (
933 (k
, v
if isinstance(v
, compat_numeric_types
) else sanitize(k
, v
))
934 for k
, v
in template_dict
.items() if v
is not None))
935 return outtmpl
, template_dict
937 def _prepare_filename(self
, info_dict
, tmpl_type
='default'):
939 sanitize
= lambda k
, v
: sanitize_filename(
941 restricted
=self
.params
.get('restrictfilenames'),
942 is_id
=(k
== 'id' or k
.endswith('_id')))
943 outtmpl
= self
.outtmpl_dict
.get(tmpl_type
, self
.outtmpl_dict
['default'])
944 outtmpl
, template_dict
= self
.prepare_outtmpl(outtmpl
, info_dict
, sanitize
)
946 # expand_path translates '%%' into '%' and '$$' into '$'
947 # correspondingly that is not what we want since we need to keep
948 # '%%' intact for template dict substitution step. Working around
949 # with boundary-alike separator hack.
950 sep
= ''.join([random
.choice(ascii_letters
) for _
in range(32)])
951 outtmpl
= outtmpl
.replace('%%', '%{0}%'.format(sep
)).replace('$$', '${0}$'.format(sep
))
953 # outtmpl should be expand_path'ed before template dict substitution
954 # because meta fields may contain env variables we don't want to
955 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
956 # title "Hello $PATH", we don't want `$PATH` to be expanded.
957 filename
= expand_path(outtmpl
).replace(sep
, '') % template_dict
959 force_ext
= OUTTMPL_TYPES
.get(tmpl_type
)
960 if force_ext
is not None:
961 filename
= replace_extension(filename
, force_ext
, template_dict
.get('ext'))
963 # https://github.com/blackjack4494/youtube-dlc/issues/85
964 trim_file_name
= self
.params
.get('trim_file_name', False)
966 fn_groups
= filename
.rsplit('.')
969 if len(fn_groups
) > 2:
970 sub_ext
= fn_groups
[-2]
971 filename
= '.'.join(filter(None, [fn_groups
[0][:trim_file_name
], sub_ext
, ext
]))
974 except ValueError as err
:
975 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
978 def prepare_filename(self
, info_dict
, dir_type
='', warn
=False):
979 """Generate the output filename."""
980 paths
= self
.params
.get('paths', {})
981 assert isinstance(paths
, dict)
982 filename
= self
._prepare
_filename
(info_dict
, dir_type
or 'default')
984 if warn
and not self
.__prepare
_filename
_warned
:
987 elif filename
== '-':
988 self
.report_warning('--paths is ignored when an outputting to stdout')
989 elif os
.path
.isabs(filename
):
990 self
.report_warning('--paths is ignored since an absolute path is given in output template')
991 self
.__prepare
_filename
_warned
= True
992 if filename
== '-' or not filename
:
995 homepath
= expand_path(paths
.get('home', '').strip())
996 assert isinstance(homepath
, compat_str
)
997 subdir
= expand_path(paths
.get(dir_type
, '').strip()) if dir_type
else ''
998 assert isinstance(subdir
, compat_str
)
999 path
= os
.path
.join(homepath
, subdir
, filename
)
1001 # Temporary fix for #4787
1002 # 'Treat' all problem characters by passing filename through preferredencoding
1003 # to workaround encoding issues with subprocess on python2 @ Windows
1004 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
1005 path
= encodeFilename(path
, True).decode(preferredencoding())
1006 return sanitize_path(path
, force
=self
.params
.get('windowsfilenames'))
1008 def _match_entry(self
, info_dict
, incomplete
):
1009 """ Returns None if the file should be downloaded """
1012 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
1013 if 'title' in info_dict
:
1014 # This can happen when we're just evaluating the playlist
1015 title
= info_dict
['title']
1016 matchtitle
= self
.params
.get('matchtitle', False)
1018 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
1019 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
1020 rejecttitle
= self
.params
.get('rejecttitle', False)
1022 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
1023 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
1024 date
= info_dict
.get('upload_date')
1025 if date
is not None:
1026 dateRange
= self
.params
.get('daterange', DateRange())
1027 if date
not in dateRange
:
1028 return '%s upload date is not in range %s' % (date_from_str(date
).isoformat(), dateRange
)
1029 view_count
= info_dict
.get('view_count')
1030 if view_count
is not None:
1031 min_views
= self
.params
.get('min_views')
1032 if min_views
is not None and view_count
< min_views
:
1033 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
1034 max_views
= self
.params
.get('max_views')
1035 if max_views
is not None and view_count
> max_views
:
1036 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
1037 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
1038 return 'Skipping "%s" because it is age restricted' % video_title
1039 if self
.in_download_archive(info_dict
):
1040 return '%s has already been recorded in archive' % video_title
1043 match_filter
= self
.params
.get('match_filter')
1044 if match_filter
is not None:
1045 ret
= match_filter(info_dict
)
1050 reason
= check_filter()
1051 if reason
is not None:
1052 self
.to_screen('[download] ' + reason
)
1053 if reason
.endswith('has already been recorded in the archive') and self
.params
.get('break_on_existing', False):
1054 raise ExistingVideoReached()
1055 elif self
.params
.get('break_on_reject', False):
1056 raise RejectedVideoReached()
1060 def add_extra_info(info_dict
, extra_info
):
1061 '''Set the keys from extra_info in info dict if they are missing'''
1062 for key
, value
in extra_info
.items():
1063 info_dict
.setdefault(key
, value
)
1065 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
={},
1066 process
=True, force_generic_extractor
=False):
1068 Return a list with a dictionary for each video extracted.
1071 url -- URL to extract
1074 download -- whether to download videos during extraction
1075 ie_key -- extractor key hint
1076 extra_info -- dictionary containing the extra values to add to each result
1077 process -- whether to resolve all unresolved references (URLs, playlist items),
1078 must be True for download to work.
1079 force_generic_extractor -- force using the generic extractor
1082 if not ie_key
and force_generic_extractor
:
1086 ies
= [self
.get_info_extractor(ie_key
)]
1091 if not ie
.suitable(url
):
1094 ie_key
= ie
.ie_key()
1095 ie
= self
.get_info_extractor(ie_key
)
1096 if not ie
.working():
1097 self
.report_warning('The program functionality for this site has been marked as broken, '
1098 'and will probably not work.')
1101 temp_id
= str_or_none(
1102 ie
.extract_id(url
) if callable(getattr(ie
, 'extract_id', None))
1103 else ie
._match
_id
(url
))
1104 except (AssertionError, IndexError, AttributeError):
1106 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': ie_key}
):
1107 self
.to_screen("[%s] %s: has already been recorded in archive" % (
1110 return self
.__extract
_info
(url
, ie
, download
, extra_info
, process
)
1112 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
1114 def __handle_extraction_exceptions(func
):
1115 def wrapper(self
, *args
, **kwargs
):
1117 return func(self
, *args
, **kwargs
)
1118 except GeoRestrictedError
as e
:
1121 msg
+= '\nThis video is available in %s.' % ', '.join(
1122 map(ISO3166Utils
.short2full
, e
.countries
))
1123 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
1124 self
.report_error(msg
)
1125 except ExtractorError
as e
: # An error we somewhat expected
1126 self
.report_error(compat_str(e
), e
.format_traceback())
1127 except (MaxDownloadsReached
, ExistingVideoReached
, RejectedVideoReached
):
1129 except Exception as e
:
1130 if self
.params
.get('ignoreerrors', False):
1131 self
.report_error(error_to_compat_str(e
), tb
=encode_compat_str(traceback
.format_exc()))
1136 @__handle_extraction_exceptions
1137 def __extract_info(self
, url
, ie
, download
, extra_info
, process
):
1138 ie_result
= ie
.extract(url
)
1139 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
1141 if isinstance(ie_result
, list):
1142 # Backwards compatibility: old IE result format
1144 '_type': 'compat_list',
1145 'entries': ie_result
,
1147 self
.add_default_extra_info(ie_result
, ie
, url
)
1149 return self
.process_ie_result(ie_result
, download
, extra_info
)
1153 def add_default_extra_info(self
, ie_result
, ie
, url
):
1154 self
.add_extra_info(ie_result
, {
1155 'extractor': ie
.IE_NAME
,
1157 'webpage_url_basename': url_basename(url
),
1158 'extractor_key': ie
.ie_key(),
1161 def process_ie_result(self
, ie_result
, download
=True, extra_info
={}):
1163 Take the result of the ie(may be modified) and resolve all unresolved
1164 references (URLs, playlist items).
1166 It will also download the videos if 'download'.
1167 Returns the resolved ie_result.
1169 result_type
= ie_result
.get('_type', 'video')
1171 if result_type
in ('url', 'url_transparent'):
1172 ie_result
['url'] = sanitize_url(ie_result
['url'])
1173 extract_flat
= self
.params
.get('extract_flat', False)
1174 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
1175 or extract_flat
is True):
1176 self
.__forced
_printings
(ie_result
, self
.prepare_filename(ie_result
), incomplete
=True)
1179 if result_type
== 'video':
1180 self
.add_extra_info(ie_result
, extra_info
)
1181 return self
.process_video_result(ie_result
, download
=download
)
1182 elif result_type
== 'url':
1183 # We have to add extra_info to the results because it may be
1184 # contained in a playlist
1185 return self
.extract_info(ie_result
['url'],
1187 ie_key
=ie_result
.get('ie_key'),
1188 extra_info
=extra_info
)
1189 elif result_type
== 'url_transparent':
1190 # Use the information from the embedding page
1191 info
= self
.extract_info(
1192 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
1193 extra_info
=extra_info
, download
=False, process
=False)
1195 # extract_info may return None when ignoreerrors is enabled and
1196 # extraction failed with an error, don't crash and return early
1201 force_properties
= dict(
1202 (k
, v
) for k
, v
in ie_result
.items() if v
is not None)
1203 for f
in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
1204 if f
in force_properties
:
1205 del force_properties
[f
]
1206 new_result
= info
.copy()
1207 new_result
.update(force_properties
)
1209 # Extracted info may not be a video result (i.e.
1210 # info.get('_type', 'video') != video) but rather an url or
1211 # url_transparent. In such cases outer metadata (from ie_result)
1212 # should be propagated to inner one (info). For this to happen
1213 # _type of info should be overridden with url_transparent. This
1214 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
1215 if new_result
.get('_type') == 'url':
1216 new_result
['_type'] = 'url_transparent'
1218 return self
.process_ie_result(
1219 new_result
, download
=download
, extra_info
=extra_info
)
1220 elif result_type
in ('playlist', 'multi_video'):
1221 # Protect from infinite recursion due to recursively nested playlists
1222 # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
1223 webpage_url
= ie_result
['webpage_url']
1224 if webpage_url
in self
._playlist
_urls
:
1226 '[download] Skipping already downloaded playlist: %s'
1227 % ie_result
.get('title') or ie_result
.get('id'))
1230 self
._playlist
_level
+= 1
1231 self
._playlist
_urls
.add(webpage_url
)
1233 return self
.__process
_playlist
(ie_result
, download
)
1235 self
._playlist
_level
-= 1
1236 if not self
._playlist
_level
:
1237 self
._playlist
_urls
.clear()
1238 elif result_type
== 'compat_list':
1239 self
.report_warning(
1240 'Extractor %s returned a compat_list result. '
1241 'It needs to be updated.' % ie_result
.get('extractor'))
1244 self
.add_extra_info(
1247 'extractor': ie_result
['extractor'],
1248 'webpage_url': ie_result
['webpage_url'],
1249 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1250 'extractor_key': ie_result
['extractor_key'],
1254 ie_result
['entries'] = [
1255 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1256 for r
in ie_result
['entries']
1260 raise Exception('Invalid result type: %s' % result_type
)
1262 def _ensure_dir_exists(self
, path
):
1263 return make_dir(path
, self
.report_error
)
1265 def __process_playlist(self
, ie_result
, download
):
1266 # We process each entry in the playlist
1267 playlist
= ie_result
.get('title') or ie_result
.get('id')
1268 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
1270 if 'entries' not in ie_result
:
1271 raise EntryNotInPlaylist()
1272 incomplete_entries
= bool(ie_result
.get('requested_entries'))
1273 if incomplete_entries
:
1274 def fill_missing_entries(entries
, indexes
):
1275 ret
= [None] * max(*indexes
)
1276 for i
, entry
in zip(indexes
, entries
):
1279 ie_result
['entries'] = fill_missing_entries(ie_result
['entries'], ie_result
['requested_entries'])
1281 playlist_results
= []
1283 playliststart
= self
.params
.get('playliststart', 1) - 1
1284 playlistend
= self
.params
.get('playlistend')
1285 # For backwards compatibility, interpret -1 as whole list
1286 if playlistend
== -1:
1289 playlistitems_str
= self
.params
.get('playlist_items')
1290 playlistitems
= None
1291 if playlistitems_str
is not None:
1292 def iter_playlistitems(format
):
1293 for string_segment
in format
.split(','):
1294 if '-' in string_segment
:
1295 start
, end
= string_segment
.split('-')
1296 for item
in range(int(start
), int(end
) + 1):
1299 yield int(string_segment
)
1300 playlistitems
= orderedSet(iter_playlistitems(playlistitems_str
))
1302 ie_entries
= ie_result
['entries']
1304 def make_playlistitems_entries(list_ie_entries
):
1305 num_entries
= len(list_ie_entries
)
1306 for i
in playlistitems
:
1307 if -num_entries
< i
<= num_entries
:
1308 yield list_ie_entries
[i
- 1]
1309 elif incomplete_entries
:
1310 raise EntryNotInPlaylist()
1312 if isinstance(ie_entries
, list):
1313 n_all_entries
= len(ie_entries
)
1315 entries
= list(make_playlistitems_entries(ie_entries
))
1317 entries
= ie_entries
[playliststart
:playlistend
]
1318 n_entries
= len(entries
)
1319 msg
= 'Collected %d videos; downloading %d of them' % (n_all_entries
, n_entries
)
1320 elif isinstance(ie_entries
, PagedList
):
1323 for item
in playlistitems
:
1324 entries
.extend(ie_entries
.getslice(
1328 entries
= ie_entries
.getslice(
1329 playliststart
, playlistend
)
1330 n_entries
= len(entries
)
1331 msg
= 'Downloading %d videos' % n_entries
1334 entries
= list(make_playlistitems_entries(list(itertools
.islice(
1335 ie_entries
, 0, max(playlistitems
)))))
1337 entries
= list(itertools
.islice(
1338 ie_entries
, playliststart
, playlistend
))
1339 n_entries
= len(entries
)
1340 msg
= 'Downloading %d videos' % n_entries
1342 if any((entry
is None for entry
in entries
)):
1343 raise EntryNotInPlaylist()
1344 if not playlistitems
and (playliststart
or playlistend
):
1345 playlistitems
= list(range(1 + playliststart
, 1 + playliststart
+ len(entries
)))
1346 ie_result
['entries'] = entries
1347 ie_result
['requested_entries'] = playlistitems
1349 if self
.params
.get('allow_playlist_files', True):
1351 'playlist': playlist
,
1352 'playlist_id': ie_result
.get('id'),
1353 'playlist_title': ie_result
.get('title'),
1354 'playlist_uploader': ie_result
.get('uploader'),
1355 'playlist_uploader_id': ie_result
.get('uploader_id'),
1356 'playlist_index': 0,
1358 ie_copy
.update(dict(ie_result
))
1360 if self
.params
.get('writeinfojson', False):
1361 infofn
= self
.prepare_filename(ie_copy
, 'pl_infojson')
1362 if not self
._ensure
_dir
_exists
(encodeFilename(infofn
)):
1364 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(infofn
)):
1365 self
.to_screen('[info] Playlist metadata is already present')
1367 self
.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn
)
1369 write_json_file(self
.filter_requested_info(ie_result
, self
.params
.get('clean_infojson', True)), infofn
)
1370 except (OSError, IOError):
1371 self
.report_error('Cannot write playlist metadata to JSON file ' + infofn
)
1373 if self
.params
.get('writedescription', False):
1374 descfn
= self
.prepare_filename(ie_copy
, 'pl_description')
1375 if not self
._ensure
_dir
_exists
(encodeFilename(descfn
)):
1377 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(descfn
)):
1378 self
.to_screen('[info] Playlist description is already present')
1379 elif ie_result
.get('description') is None:
1380 self
.report_warning('There\'s no playlist description to write.')
1383 self
.to_screen('[info] Writing playlist description to: ' + descfn
)
1384 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
1385 descfile
.write(ie_result
['description'])
1386 except (OSError, IOError):
1387 self
.report_error('Cannot write playlist description file ' + descfn
)
1390 # Save playlist_index before re-ordering
1392 ((playlistitems
[i
- 1] if playlistitems
else i
), entry
)
1393 for i
, entry
in enumerate(entries
, 1)]
1395 if self
.params
.get('playlistreverse', False):
1396 entries
= entries
[::-1]
1397 if self
.params
.get('playlistrandom', False):
1398 random
.shuffle(entries
)
1400 x_forwarded_for
= ie_result
.get('__x_forwarded_for_ip')
1402 self
.to_screen('[%s] playlist %s: %s' % (ie_result
['extractor'], playlist
, msg
))
1404 max_failures
= self
.params
.get('skip_playlist_after_errors') or float('inf')
1405 for i
, entry_tuple
in enumerate(entries
, 1):
1406 playlist_index
, entry
= entry_tuple
1407 if 'playlist_index' in self
.params
.get('compat_options', []):
1408 playlist_index
= playlistitems
[i
- 1] if playlistitems
else i
1409 self
.to_screen('[download] Downloading video %s of %s' % (i
, n_entries
))
1410 # This __x_forwarded_for_ip thing is a bit ugly but requires
1413 entry
['__x_forwarded_for_ip'] = x_forwarded_for
1415 'n_entries': n_entries
,
1416 '_last_playlist_index': max(playlistitems
) if playlistitems
else (playlistend
or n_entries
),
1417 'playlist_index': playlist_index
,
1418 'playlist_autonumber': i
,
1419 'playlist': playlist
,
1420 'playlist_id': ie_result
.get('id'),
1421 'playlist_title': ie_result
.get('title'),
1422 'playlist_uploader': ie_result
.get('uploader'),
1423 'playlist_uploader_id': ie_result
.get('uploader_id'),
1424 'extractor': ie_result
['extractor'],
1425 'webpage_url': ie_result
['webpage_url'],
1426 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1427 'extractor_key': ie_result
['extractor_key'],
1430 if self
._match
_entry
(entry
, incomplete
=True) is not None:
1433 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, extra
)
1434 if not entry_result
:
1436 if failures
>= max_failures
:
1438 'Skipping the remaining entries in playlist "%s" since %d items failed extraction' % (playlist
, failures
))
1440 # TODO: skip failed (empty) entries?
1441 playlist_results
.append(entry_result
)
1442 ie_result
['entries'] = playlist_results
1443 self
.to_screen('[download] Finished downloading playlist: %s' % playlist
)
1446 @__handle_extraction_exceptions
1447 def __process_iterable_entry(self
, entry
, download
, extra_info
):
1448 return self
.process_ie_result(
1449 entry
, download
=download
, extra_info
=extra_info
)
1451 def _build_format_filter(self
, filter_spec
):
1452 " Returns a function to filter the formats according to the filter_spec "
1462 operator_rex
= re
.compile(r
'''(?x)\s*
1463 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
1464 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1465 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1467 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
1468 m
= operator_rex
.search(filter_spec
)
1471 comparison_value
= int(m
.group('value'))
1473 comparison_value
= parse_filesize(m
.group('value'))
1474 if comparison_value
is None:
1475 comparison_value
= parse_filesize(m
.group('value') + 'B')
1476 if comparison_value
is None:
1478 'Invalid value %r in format specification %r' % (
1479 m
.group('value'), filter_spec
))
1480 op
= OPERATORS
[m
.group('op')]
1485 '^=': lambda attr
, value
: attr
.startswith(value
),
1486 '$=': lambda attr
, value
: attr
.endswith(value
),
1487 '*=': lambda attr
, value
: value
in attr
,
1489 str_operator_rex
= re
.compile(r
'''(?x)
1490 \s*(?P<key>[a-zA-Z0-9._-]+)
1491 \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?
1492 \s*(?P<value>[a-zA-Z0-9._-]+)
1494 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
1495 m
= str_operator_rex
.search(filter_spec
)
1497 comparison_value
= m
.group('value')
1498 str_op
= STR_OPERATORS
[m
.group('op')]
1499 if m
.group('negation'):
1500 op
= lambda attr
, value
: not str_op(attr
, value
)
1505 raise ValueError('Invalid filter specification %r' % filter_spec
)
1508 actual_value
= f
.get(m
.group('key'))
1509 if actual_value
is None:
1510 return m
.group('none_inclusive')
1511 return op(actual_value
, comparison_value
)
1514 def _default_format_spec(self
, info_dict
, download
=True):
1517 merger
= FFmpegMergerPP(self
)
1518 return merger
.available
and merger
.can_merge()
1521 not self
.params
.get('simulate', False)
1525 or info_dict
.get('is_live', False)
1526 or self
.outtmpl_dict
['default'] == '-'))
1529 or self
.params
.get('allow_multiple_audio_streams', False)
1530 or 'format-spec' in self
.params
.get('compat_opts', []))
1533 'best/bestvideo+bestaudio' if prefer_best
1534 else 'bestvideo*+bestaudio/best' if not compat
1535 else 'bestvideo+bestaudio/best')
1537 def build_format_selector(self
, format_spec
):
1538 def syntax_error(note
, start
):
1540 'Invalid format specification: '
1541 '{0}\n\t{1}\n\t{2}^'.format(note
, format_spec
, ' ' * start
[1]))
1542 return SyntaxError(message
)
1544 PICKFIRST
= 'PICKFIRST'
1548 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1550 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
1551 'video': self
.params
.get('allow_multiple_video_streams', False)}
1553 check_formats
= self
.params
.get('check_formats')
1555 def _parse_filter(tokens
):
1557 for type, string
, start
, _
, _
in tokens
:
1558 if type == tokenize
.OP
and string
== ']':
1559 return ''.join(filter_parts
)
1561 filter_parts
.append(string
)
1563 def _remove_unused_ops(tokens
):
1564 # Remove operators that we don't use and join them with the surrounding strings
1565 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1566 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
1567 last_string
, last_start
, last_end
, last_line
= None, None, None, None
1568 for type, string
, start
, end
, line
in tokens
:
1569 if type == tokenize
.OP
and string
== '[':
1571 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1573 yield type, string
, start
, end
, line
1574 # everything inside brackets will be handled by _parse_filter
1575 for type, string
, start
, end
, line
in tokens
:
1576 yield type, string
, start
, end
, line
1577 if type == tokenize
.OP
and string
== ']':
1579 elif type == tokenize
.OP
and string
in ALLOWED_OPS
:
1581 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1583 yield type, string
, start
, end
, line
1584 elif type in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
1586 last_string
= string
1590 last_string
+= string
1592 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1594 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
1596 current_selector
= None
1597 for type, string
, start
, _
, _
in tokens
:
1598 # ENCODING is only defined in python 3.x
1599 if type == getattr(tokenize
, 'ENCODING', None):
1601 elif type in [tokenize
.NAME
, tokenize
.NUMBER
]:
1602 current_selector
= FormatSelector(SINGLE
, string
, [])
1603 elif type == tokenize
.OP
:
1605 if not inside_group
:
1606 # ')' will be handled by the parentheses group
1607 tokens
.restore_last_token()
1609 elif inside_merge
and string
in ['/', ',']:
1610 tokens
.restore_last_token()
1612 elif inside_choice
and string
== ',':
1613 tokens
.restore_last_token()
1616 if not current_selector
:
1617 raise syntax_error('"," must follow a format selector', start
)
1618 selectors
.append(current_selector
)
1619 current_selector
= None
1621 if not current_selector
:
1622 raise syntax_error('"/" must follow a format selector', start
)
1623 first_choice
= current_selector
1624 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
1625 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
1627 if not current_selector
:
1628 current_selector
= FormatSelector(SINGLE
, 'best', [])
1629 format_filter
= _parse_filter(tokens
)
1630 current_selector
.filters
.append(format_filter
)
1632 if current_selector
:
1633 raise syntax_error('Unexpected "("', start
)
1634 group
= _parse_format_selection(tokens
, inside_group
=True)
1635 current_selector
= FormatSelector(GROUP
, group
, [])
1637 if not current_selector
:
1638 raise syntax_error('Unexpected "+"', start
)
1639 selector_1
= current_selector
1640 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
1642 raise syntax_error('Expected a selector', start
)
1643 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
1645 raise syntax_error('Operator not recognized: "{0}"'.format(string
), start
)
1646 elif type == tokenize
.ENDMARKER
:
1648 if current_selector
:
1649 selectors
.append(current_selector
)
1652 def _merge(formats_pair
):
1653 format_1
, format_2
= formats_pair
1656 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
1657 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
1659 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
1660 get_no_more
= {"video": False, "audio": False}
1661 for (i
, fmt_info
) in enumerate(formats_info
):
1662 for aud_vid
in ["audio", "video"]:
1663 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
1664 if get_no_more
[aud_vid
]:
1666 get_no_more
[aud_vid
] = True
1668 if len(formats_info
) == 1:
1669 return formats_info
[0]
1671 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
1672 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
1674 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
1675 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
1677 output_ext
= self
.params
.get('merge_output_format')
1680 output_ext
= the_only_video
['ext']
1681 elif the_only_audio
and not video_fmts
:
1682 output_ext
= the_only_audio
['ext']
1687 'requested_formats': formats_info
,
1688 'format': '+'.join(fmt_info
.get('format') for fmt_info
in formats_info
),
1689 'format_id': '+'.join(fmt_info
.get('format_id') for fmt_info
in formats_info
),
1695 'width': the_only_video
.get('width'),
1696 'height': the_only_video
.get('height'),
1697 'resolution': the_only_video
.get('resolution') or self
.format_resolution(the_only_video
),
1698 'fps': the_only_video
.get('fps'),
1699 'vcodec': the_only_video
.get('vcodec'),
1700 'vbr': the_only_video
.get('vbr'),
1701 'stretched_ratio': the_only_video
.get('stretched_ratio'),
1706 'acodec': the_only_audio
.get('acodec'),
1707 'abr': the_only_audio
.get('abr'),
1712 def _check_formats(formats
):
1714 self
.to_screen('[info] Testing format %s' % f
['format_id'])
1715 paths
= self
.params
.get('paths', {})
1716 temp_file
= os
.path
.join(
1717 expand_path(paths
.get('home', '').strip()),
1718 expand_path(paths
.get('temp', '').strip()),
1719 'ytdl.%s.f%s.check-format' % (random_uuidv4(), f
['format_id']))
1720 dl
, _
= self
.dl(temp_file
, f
, test
=True)
1721 if os
.path
.exists(temp_file
):
1722 os
.remove(temp_file
)
1726 self
.to_screen('[info] Unable to download format %s. Skipping...' % f
['format_id'])
1728 def _build_selector_function(selector
):
1729 if isinstance(selector
, list): # ,
1730 fs
= [_build_selector_function(s
) for s
in selector
]
1732 def selector_function(ctx
):
1734 for format
in f(ctx
):
1736 return selector_function
1738 elif selector
.type == GROUP
: # ()
1739 selector_function
= _build_selector_function(selector
.selector
)
1741 elif selector
.type == PICKFIRST
: # /
1742 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
1744 def selector_function(ctx
):
1746 picked_formats
= list(f(ctx
))
1748 return picked_formats
1751 elif selector
.type == SINGLE
: # atom
1752 format_spec
= selector
.selector
or 'best'
1754 # TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
1755 if format_spec
== 'all':
1756 def selector_function(ctx
):
1757 formats
= list(ctx
['formats'])
1759 formats
= _check_formats(formats
)
1762 elif format_spec
== 'mergeall':
1763 def selector_function(ctx
):
1764 formats
= list(_check_formats(ctx
['formats']))
1767 merged_format
= formats
[-1]
1768 for f
in formats
[-2::-1]:
1769 merged_format
= _merge((merged_format
, f
))
1773 format_fallback
, format_reverse
, format_idx
= False, True, 1
1775 r
'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
1777 if mobj
is not None:
1778 format_idx
= int_or_none(mobj
.group('n'), default
=1)
1779 format_reverse
= mobj
.group('bw')[0] == 'b'
1780 format_type
= (mobj
.group('type') or [None])[0]
1781 not_format_type
= {'v': 'a', 'a': 'v'}
.get(format_type
)
1782 format_modified
= mobj
.group('mod') is not None
1784 format_fallback
= not format_type
and not format_modified
# for b, w
1786 (lambda f
: f
.get('%scodec' % format_type
) != 'none')
1787 if format_type
and format_modified
# bv*, ba*, wv*, wa*
1788 else (lambda f
: f
.get('%scodec' % not_format_type
) == 'none')
1789 if format_type
# bv, ba, wv, wa
1790 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
1791 if not format_modified
# b, w
1794 filter_f
= ((lambda f
: f
.get('ext') == format_spec
)
1795 if format_spec
in ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav'] # extension
1796 else (lambda f
: f
.get('format_id') == format_spec
)) # id
1798 def selector_function(ctx
):
1799 formats
= list(ctx
['formats'])
1802 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
1803 if format_fallback
and ctx
['incomplete_formats'] and not matches
:
1804 # for extractors with incomplete formats (audio only (soundcloud)
1805 # or video only (imgur)) best/worst will fallback to
1806 # best/worst {video,audio}-only format
1809 matches
= matches
[::-1]
1811 matches
= list(itertools
.islice(_check_formats(matches
), format_idx
))
1813 if -n
<= format_idx
- 1 < n
:
1814 yield matches
[format_idx
- 1]
1816 elif selector
.type == MERGE
: # +
1817 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
1819 def selector_function(ctx
):
1820 for pair
in itertools
.product(
1821 selector_1(copy
.deepcopy(ctx
)), selector_2(copy
.deepcopy(ctx
))):
1824 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
1826 def final_selector(ctx
):
1827 ctx_copy
= copy
.deepcopy(ctx
)
1828 for _filter
in filters
:
1829 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
1830 return selector_function(ctx_copy
)
1831 return final_selector
1833 stream
= io
.BytesIO(format_spec
.encode('utf-8'))
1835 tokens
= list(_remove_unused_ops(compat_tokenize_tokenize(stream
.readline
)))
1836 except tokenize
.TokenError
:
1837 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
1839 class TokenIterator(object):
1840 def __init__(self
, tokens
):
1841 self
.tokens
= tokens
1848 if self
.counter
>= len(self
.tokens
):
1849 raise StopIteration()
1850 value
= self
.tokens
[self
.counter
]
1856 def restore_last_token(self
):
1859 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
1860 return _build_selector_function(parsed_selector
)
1862 def _calc_headers(self
, info_dict
):
1863 res
= std_headers
.copy()
1865 add_headers
= info_dict
.get('http_headers')
1867 res
.update(add_headers
)
1869 cookies
= self
._calc
_cookies
(info_dict
)
1871 res
['Cookie'] = cookies
1873 if 'X-Forwarded-For' not in res
:
1874 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
1875 if x_forwarded_for_ip
:
1876 res
['X-Forwarded-For'] = x_forwarded_for_ip
1880 def _calc_cookies(self
, info_dict
):
1881 pr
= sanitized_Request(info_dict
['url'])
1882 self
.cookiejar
.add_cookie_header(pr
)
1883 return pr
.get_header('Cookie')
1885 def process_video_result(self
, info_dict
, download
=True):
1886 assert info_dict
.get('_type', 'video') == 'video'
1888 if 'id' not in info_dict
:
1889 raise ExtractorError('Missing "id" field in extractor result')
1890 if 'title' not in info_dict
:
1891 raise ExtractorError('Missing "title" field in extractor result')
1893 def report_force_conversion(field
, field_not
, conversion
):
1894 self
.report_warning(
1895 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1896 % (field
, field_not
, conversion
))
1898 def sanitize_string_field(info
, string_field
):
1899 field
= info
.get(string_field
)
1900 if field
is None or isinstance(field
, compat_str
):
1902 report_force_conversion(string_field
, 'a string', 'string')
1903 info
[string_field
] = compat_str(field
)
1905 def sanitize_numeric_fields(info
):
1906 for numeric_field
in self
._NUMERIC
_FIELDS
:
1907 field
= info
.get(numeric_field
)
1908 if field
is None or isinstance(field
, compat_numeric_types
):
1910 report_force_conversion(numeric_field
, 'numeric', 'int')
1911 info
[numeric_field
] = int_or_none(field
)
1913 sanitize_string_field(info_dict
, 'id')
1914 sanitize_numeric_fields(info_dict
)
1916 if 'playlist' not in info_dict
:
1917 # It isn't part of a playlist
1918 info_dict
['playlist'] = None
1919 info_dict
['playlist_index'] = None
1921 thumbnails
= info_dict
.get('thumbnails')
1922 if thumbnails
is None:
1923 thumbnail
= info_dict
.get('thumbnail')
1925 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
1927 thumbnails
.sort(key
=lambda t
: (
1928 t
.get('preference') if t
.get('preference') is not None else -1,
1929 t
.get('width') if t
.get('width') is not None else -1,
1930 t
.get('height') if t
.get('height') is not None else -1,
1931 t
.get('id') if t
.get('id') is not None else '', t
.get('url')))
1932 for i
, t
in enumerate(thumbnails
):
1933 t
['url'] = sanitize_url(t
['url'])
1934 if t
.get('width') and t
.get('height'):
1935 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
1936 if t
.get('id') is None:
1939 if self
.params
.get('list_thumbnails'):
1940 self
.list_thumbnails(info_dict
)
1943 thumbnail
= info_dict
.get('thumbnail')
1945 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
1947 info_dict
['thumbnail'] = thumbnails
[-1]['url']
1949 if 'display_id' not in info_dict
and 'id' in info_dict
:
1950 info_dict
['display_id'] = info_dict
['id']
1952 for ts_key
, date_key
in (
1953 ('timestamp', 'upload_date'),
1954 ('release_timestamp', 'release_date'),
1956 if info_dict
.get(date_key
) is None and info_dict
.get(ts_key
) is not None:
1957 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1958 # see http://bugs.python.org/issue1646728)
1960 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
[ts_key
])
1961 info_dict
[date_key
] = upload_date
.strftime('%Y%m%d')
1962 except (ValueError, OverflowError, OSError):
1965 # Auto generate title fields corresponding to the *_number fields when missing
1966 # in order to always have clean titles. This is very common for TV series.
1967 for field
in ('chapter', 'season', 'episode'):
1968 if info_dict
.get('%s_number' % field
) is not None and not info_dict
.get(field
):
1969 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
['%s_number' % field
])
1971 for cc_kind
in ('subtitles', 'automatic_captions'):
1972 cc
= info_dict
.get(cc_kind
)
1974 for _
, subtitle
in cc
.items():
1975 for subtitle_format
in subtitle
:
1976 if subtitle_format
.get('url'):
1977 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
1978 if subtitle_format
.get('ext') is None:
1979 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
1981 automatic_captions
= info_dict
.get('automatic_captions')
1982 subtitles
= info_dict
.get('subtitles')
1984 if self
.params
.get('listsubtitles', False):
1985 if 'automatic_captions' in info_dict
:
1986 self
.list_subtitles(
1987 info_dict
['id'], automatic_captions
, 'automatic captions')
1988 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
1991 info_dict
['requested_subtitles'] = self
.process_subtitles(
1992 info_dict
['id'], subtitles
, automatic_captions
)
1994 # We now pick which formats have to be downloaded
1995 if info_dict
.get('formats') is None:
1996 # There's only one format available
1997 formats
= [info_dict
]
1999 formats
= info_dict
['formats']
2002 if not self
.params
.get('ignore_no_formats_error'):
2003 raise ExtractorError('No video formats found!')
2005 self
.report_warning('No video formats found!')
2007 def is_wellformed(f
):
2010 self
.report_warning(
2011 '"url" field is missing or empty - skipping format, '
2012 'there is an error in extractor')
2014 if isinstance(url
, bytes):
2015 sanitize_string_field(f
, 'url')
2018 # Filter out malformed formats for better extraction robustness
2019 formats
= list(filter(is_wellformed
, formats
))
2023 # We check that all the formats have the format and format_id fields
2024 for i
, format
in enumerate(formats
):
2025 sanitize_string_field(format
, 'format_id')
2026 sanitize_numeric_fields(format
)
2027 format
['url'] = sanitize_url(format
['url'])
2028 if not format
.get('format_id'):
2029 format
['format_id'] = compat_str(i
)
2031 # Sanitize format_id from characters used in format selector expression
2032 format
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', format
['format_id'])
2033 format_id
= format
['format_id']
2034 if format_id
not in formats_dict
:
2035 formats_dict
[format_id
] = []
2036 formats_dict
[format_id
].append(format
)
2038 # Make sure all formats have unique format_id
2039 for format_id
, ambiguous_formats
in formats_dict
.items():
2040 if len(ambiguous_formats
) > 1:
2041 for i
, format
in enumerate(ambiguous_formats
):
2042 format
['format_id'] = '%s-%d' % (format_id
, i
)
2044 for i
, format
in enumerate(formats
):
2045 if format
.get('format') is None:
2046 format
['format'] = '{id} - {res}{note}'.format(
2047 id=format
['format_id'],
2048 res
=self
.format_resolution(format
),
2049 note
=' ({0})'.format(format
['format_note']) if format
.get('format_note') is not None else '',
2051 # Automatically determine file extension if missing
2052 if format
.get('ext') is None:
2053 format
['ext'] = determine_ext(format
['url']).lower()
2054 # Automatically determine protocol if missing (useful for format
2055 # selection purposes)
2056 if format
.get('protocol') is None:
2057 format
['protocol'] = determine_protocol(format
)
2058 # Add HTTP headers, so that external programs can use them from the
2060 full_format_info
= info_dict
.copy()
2061 full_format_info
.update(format
)
2062 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
2063 # Remove private housekeeping stuff
2064 if '__x_forwarded_for_ip' in info_dict
:
2065 del info_dict
['__x_forwarded_for_ip']
2067 # TODO Central sorting goes here
2069 if formats
and formats
[0] is not info_dict
:
2070 # only set the 'formats' fields if the original info_dict list them
2071 # otherwise we end up with a circular reference, the first (and unique)
2072 # element in the 'formats' field in info_dict is info_dict itself,
2073 # which can't be exported to json
2074 info_dict
['formats'] = formats
2075 if self
.params
.get('listformats'):
2076 if not info_dict
.get('formats'):
2077 raise ExtractorError('No video formats found', expected
=True)
2078 self
.list_formats(info_dict
)
2081 req_format
= self
.params
.get('format')
2082 if req_format
is None:
2083 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
2084 if self
.params
.get('verbose'):
2085 self
.to_screen('[debug] Default format spec: %s' % req_format
)
2087 format_selector
= self
.build_format_selector(req_format
)
2089 # While in format selection we may need to have an access to the original
2090 # format set in order to calculate some metrics or do some processing.
2091 # For now we need to be able to guess whether original formats provided
2092 # by extractor are incomplete or not (i.e. whether extractor provides only
2093 # video-only or audio-only formats) for proper formats selection for
2094 # extractors with such incomplete formats (see
2095 # https://github.com/ytdl-org/youtube-dl/pull/5556).
2096 # Since formats may be filtered during format selection and may not match
2097 # the original formats the results may be incorrect. Thus original formats
2098 # or pre-calculated metrics should be passed to format selection routines
2100 # We will pass a context object containing all necessary additional data
2101 # instead of just formats.
2102 # This fixes incorrect format selection issue (see
2103 # https://github.com/ytdl-org/youtube-dl/issues/10083).
2104 incomplete_formats
= (
2105 # All formats are video-only or
2106 all(f
.get('vcodec') != 'none' and f
.get('acodec') == 'none' for f
in formats
)
2107 # all formats are audio-only
2108 or all(f
.get('vcodec') == 'none' and f
.get('acodec') != 'none' for f
in formats
))
2112 'incomplete_formats': incomplete_formats
,
2115 formats_to_download
= list(format_selector(ctx
))
2116 if not formats_to_download
:
2117 if not self
.params
.get('ignore_no_formats_error'):
2118 raise ExtractorError('Requested format is not available', expected
=True)
2120 self
.report_warning('Requested format is not available')
2123 '[info] %s: Downloading format(s) %s'
2124 % (info_dict
['id'], ", ".join([f
['format_id'] for f
in formats_to_download
])))
2125 if len(formats_to_download
) > 1:
2127 '[info] %s: Downloading video in %s formats'
2128 % (info_dict
['id'], len(formats_to_download
)))
2129 for fmt
in formats_to_download
:
2130 new_info
= dict(info_dict
)
2131 new_info
.update(fmt
)
2132 self
.process_info(new_info
)
2133 # We update the info dict with the best quality format (backwards compatibility)
2134 if formats_to_download
:
2135 info_dict
.update(formats_to_download
[-1])
2138 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
2139 """Select the requested subtitles and their format"""
2141 if normal_subtitles
and self
.params
.get('writesubtitles'):
2142 available_subs
.update(normal_subtitles
)
2143 if automatic_captions
and self
.params
.get('writeautomaticsub'):
2144 for lang
, cap_info
in automatic_captions
.items():
2145 if lang
not in available_subs
:
2146 available_subs
[lang
] = cap_info
2148 if (not self
.params
.get('writesubtitles') and not
2149 self
.params
.get('writeautomaticsub') or not
2153 all_sub_langs
= available_subs
.keys()
2154 if self
.params
.get('allsubtitles', False):
2155 requested_langs
= all_sub_langs
2156 elif self
.params
.get('subtitleslangs', False):
2157 requested_langs
= set()
2158 for lang
in self
.params
.get('subtitleslangs'):
2160 requested_langs
.update(all_sub_langs
)
2162 discard
= lang
[0] == '-'
2165 current_langs
= filter(re
.compile(lang
+ '$').match
, all_sub_langs
)
2167 for lang
in current_langs
:
2168 requested_langs
.discard(lang
)
2170 requested_langs
.update(current_langs
)
2171 elif 'en' in available_subs
:
2172 requested_langs
= ['en']
2174 requested_langs
= [list(all_sub_langs
)[0]]
2176 formats_query
= self
.params
.get('subtitlesformat', 'best')
2177 formats_preference
= formats_query
.split('/') if formats_query
else []
2179 for lang
in requested_langs
:
2180 formats
= available_subs
.get(lang
)
2182 self
.report_warning('%s subtitles not available for %s' % (lang
, video_id
))
2184 for ext
in formats_preference
:
2188 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
2194 self
.report_warning(
2195 'No subtitle format found matching "%s" for language %s, '
2196 'using %s' % (formats_query
, lang
, f
['ext']))
2200 def __forced_printings(self
, info_dict
, filename
, incomplete
):
2201 def print_mandatory(field
):
2202 if (self
.params
.get('force%s' % field
, False)
2203 and (not incomplete
or info_dict
.get(field
) is not None)):
2204 self
.to_stdout(info_dict
[field
])
2206 def print_optional(field
):
2207 if (self
.params
.get('force%s' % field
, False)
2208 and info_dict
.get(field
) is not None):
2209 self
.to_stdout(info_dict
[field
])
2211 print_mandatory('title')
2212 print_mandatory('id')
2213 if self
.params
.get('forceurl', False) and not incomplete
:
2214 if info_dict
.get('requested_formats') is not None:
2215 for f
in info_dict
['requested_formats']:
2216 self
.to_stdout(f
['url'] + f
.get('play_path', ''))
2218 # For RTMP URLs, also include the playpath
2219 self
.to_stdout(info_dict
['url'] + info_dict
.get('play_path', ''))
2220 print_optional('thumbnail')
2221 print_optional('description')
2222 if self
.params
.get('forcefilename', False) and filename
is not None:
2223 self
.to_stdout(filename
)
2224 if self
.params
.get('forceduration', False) and info_dict
.get('duration') is not None:
2225 self
.to_stdout(formatSeconds(info_dict
['duration']))
2226 print_mandatory('format')
2227 if self
.params
.get('forcejson', False):
2228 self
.post_extract(info_dict
)
2229 self
.to_stdout(json
.dumps(info_dict
, default
=repr))
2231 def dl(self
, name
, info
, subtitle
=False, test
=False):
2234 verbose
= self
.params
.get('verbose')
2237 'quiet': not verbose
,
2239 'noprogress': not verbose
,
2241 'skip_unavailable_fragments': False,
2242 'keep_fragments': False,
2244 '_no_ytdl_file': True,
2247 params
= self
.params
2248 fd
= get_suitable_downloader(info
, params
)(self
, params
)
2250 for ph
in self
._progress
_hooks
:
2251 fd
.add_progress_hook(ph
)
2252 if self
.params
.get('verbose'):
2253 self
.to_screen('[debug] Invoking downloader on %r' % info
.get('url'))
2254 new_info
= dict(info
)
2255 if new_info
.get('http_headers') is None:
2256 new_info
['http_headers'] = self
._calc
_headers
(new_info
)
2257 return fd
.download(name
, new_info
, subtitle
)
2259 def process_info(self
, info_dict
):
2260 """Process a single resolved IE result."""
2262 assert info_dict
.get('_type', 'video') == 'video'
2264 info_dict
.setdefault('__postprocessors', [])
2266 max_downloads
= self
.params
.get('max_downloads')
2267 if max_downloads
is not None:
2268 if self
._num
_downloads
>= int(max_downloads
):
2269 raise MaxDownloadsReached()
2271 # TODO: backward compatibility, to be removed
2272 info_dict
['fulltitle'] = info_dict
['title']
2274 if 'format' not in info_dict
:
2275 info_dict
['format'] = info_dict
['ext']
2277 if self
._match
_entry
(info_dict
, incomplete
=False) is not None:
2280 self
.post_extract(info_dict
)
2281 self
._num
_downloads
+= 1
2283 info_dict
, _
= self
.pre_process(info_dict
)
2285 # info_dict['_filename'] needs to be set for backward compatibility
2286 info_dict
['_filename'] = full_filename
= self
.prepare_filename(info_dict
, warn
=True)
2287 temp_filename
= self
.prepare_filename(info_dict
, 'temp')
2291 self
.__forced
_printings
(info_dict
, full_filename
, incomplete
=False)
2293 if self
.params
.get('simulate', False):
2294 if self
.params
.get('force_write_download_archive', False):
2295 self
.record_download_archive(info_dict
)
2297 # Do nothing else if in simulate mode
2300 if full_filename
is None:
2303 if not self
._ensure
_dir
_exists
(encodeFilename(full_filename
)):
2305 if not self
._ensure
_dir
_exists
(encodeFilename(temp_filename
)):
2308 if self
.params
.get('writedescription', False):
2309 descfn
= self
.prepare_filename(info_dict
, 'description')
2310 if not self
._ensure
_dir
_exists
(encodeFilename(descfn
)):
2312 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(descfn
)):
2313 self
.to_screen('[info] Video description is already present')
2314 elif info_dict
.get('description') is None:
2315 self
.report_warning('There\'s no description to write.')
2318 self
.to_screen('[info] Writing video description to: ' + descfn
)
2319 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
2320 descfile
.write(info_dict
['description'])
2321 except (OSError, IOError):
2322 self
.report_error('Cannot write description file ' + descfn
)
2325 if self
.params
.get('writeannotations', False):
2326 annofn
= self
.prepare_filename(info_dict
, 'annotation')
2327 if not self
._ensure
_dir
_exists
(encodeFilename(annofn
)):
2329 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(annofn
)):
2330 self
.to_screen('[info] Video annotations are already present')
2331 elif not info_dict
.get('annotations'):
2332 self
.report_warning('There are no annotations to write.')
2335 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
2336 with io
.open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
2337 annofile
.write(info_dict
['annotations'])
2338 except (KeyError, TypeError):
2339 self
.report_warning('There are no annotations to write.')
2340 except (OSError, IOError):
2341 self
.report_error('Cannot write annotations file: ' + annofn
)
2344 subtitles_are_requested
= any([self
.params
.get('writesubtitles', False),
2345 self
.params
.get('writeautomaticsub')])
2347 if subtitles_are_requested
and info_dict
.get('requested_subtitles'):
2348 # subtitles download errors are already managed as troubles in relevant IE
2349 # that way it will silently go on when used with unsupporting IE
2350 subtitles
= info_dict
['requested_subtitles']
2351 # ie = self.get_info_extractor(info_dict['extractor_key'])
2352 for sub_lang
, sub_info
in subtitles
.items():
2353 sub_format
= sub_info
['ext']
2354 sub_filename
= subtitles_filename(temp_filename
, sub_lang
, sub_format
, info_dict
.get('ext'))
2355 sub_filename_final
= subtitles_filename(
2356 self
.prepare_filename(info_dict
, 'subtitle'), sub_lang
, sub_format
, info_dict
.get('ext'))
2357 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(sub_filename
)):
2358 self
.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang
, sub_format
))
2359 sub_info
['filepath'] = sub_filename
2360 files_to_move
[sub_filename
] = sub_filename_final
2362 self
.to_screen('[info] Writing video subtitles to: ' + sub_filename
)
2363 if sub_info
.get('data') is not None:
2365 # Use newline='' to prevent conversion of newline characters
2366 # See https://github.com/ytdl-org/youtube-dl/issues/10268
2367 with io
.open(encodeFilename(sub_filename
), 'w', encoding
='utf-8', newline
='') as subfile
:
2368 subfile
.write(sub_info
['data'])
2369 sub_info
['filepath'] = sub_filename
2370 files_to_move
[sub_filename
] = sub_filename_final
2371 except (OSError, IOError):
2372 self
.report_error('Cannot write subtitles file ' + sub_filename
)
2376 self
.dl(sub_filename
, sub_info
.copy(), subtitle
=True)
2377 sub_info
['filepath'] = sub_filename
2378 files_to_move
[sub_filename
] = sub_filename_final
2379 except tuple([ExtractorError
, IOError, OSError, ValueError] + network_exceptions
) as err
:
2380 self
.report_warning('Unable to download subtitle for "%s": %s' %
2381 (sub_lang
, error_to_compat_str(err
)))
2384 if self
.params
.get('writeinfojson', False):
2385 infofn
= self
.prepare_filename(info_dict
, 'infojson')
2386 if not self
._ensure
_dir
_exists
(encodeFilename(infofn
)):
2388 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(infofn
)):
2389 self
.to_screen('[info] Video metadata is already present')
2391 self
.to_screen('[info] Writing video metadata as JSON to: ' + infofn
)
2393 write_json_file(self
.filter_requested_info(info_dict
, self
.params
.get('clean_infojson', True)), infofn
)
2394 except (OSError, IOError):
2395 self
.report_error('Cannot write video metadata to JSON file ' + infofn
)
2397 info_dict
['__infojson_filename'] = infofn
2399 for thumb_ext
in self
._write
_thumbnails
(info_dict
, temp_filename
):
2400 thumb_filename_temp
= replace_extension(temp_filename
, thumb_ext
, info_dict
.get('ext'))
2401 thumb_filename
= replace_extension(
2402 self
.prepare_filename(info_dict
, 'thumbnail'), thumb_ext
, info_dict
.get('ext'))
2403 files_to_move
[thumb_filename_temp
] = thumb_filename
2405 # Write internet shortcut files
2406 url_link
= webloc_link
= desktop_link
= False
2407 if self
.params
.get('writelink', False):
2408 if sys
.platform
== "darwin": # macOS.
2410 elif sys
.platform
.startswith("linux"):
2412 else: # if sys.platform in ['win32', 'cygwin']:
2414 if self
.params
.get('writeurllink', False):
2416 if self
.params
.get('writewebloclink', False):
2418 if self
.params
.get('writedesktoplink', False):
2421 if url_link
or webloc_link
or desktop_link
:
2422 if 'webpage_url' not in info_dict
:
2423 self
.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information')
2425 ascii_url
= iri_to_uri(info_dict
['webpage_url'])
2427 def _write_link_file(extension
, template
, newline
, embed_filename
):
2428 linkfn
= replace_extension(full_filename
, extension
, info_dict
.get('ext'))
2429 if self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(linkfn
)):
2430 self
.to_screen('[info] Internet shortcut is already present')
2433 self
.to_screen('[info] Writing internet shortcut to: ' + linkfn
)
2434 with io
.open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8', newline
=newline
) as linkfile
:
2435 template_vars
= {'url': ascii_url}
2437 template_vars
['filename'] = linkfn
[:-(len(extension
) + 1)]
2438 linkfile
.write(template
% template_vars
)
2439 except (OSError, IOError):
2440 self
.report_error('Cannot write internet shortcut ' + linkfn
)
2445 if not _write_link_file('url', DOT_URL_LINK_TEMPLATE
, '\r\n', embed_filename
=False):
2448 if not _write_link_file('webloc', DOT_WEBLOC_LINK_TEMPLATE
, '\n', embed_filename
=False):
2451 if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE
, '\n', embed_filename
=True):
2455 info_dict
, files_to_move
= self
.pre_process(info_dict
, 'before_dl', files_to_move
)
2456 except PostProcessingError
as err
:
2457 self
.report_error('Preprocessing: %s' % str(err
))
2460 must_record_download_archive
= False
2461 if self
.params
.get('skip_download', False):
2462 info_dict
['filepath'] = temp_filename
2463 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2464 info_dict
['__files_to_move'] = files_to_move
2465 info_dict
= self
.run_pp(MoveFilesAfterDownloadPP(self
, False), info_dict
)
2470 def existing_file(*filepaths
):
2471 ext
= info_dict
.get('ext')
2472 final_ext
= self
.params
.get('final_ext', ext
)
2474 for file in orderedSet(filepaths
):
2475 if final_ext
!= ext
:
2476 converted
= replace_extension(file, final_ext
, ext
)
2477 if os
.path
.exists(encodeFilename(converted
)):
2478 existing_files
.append(converted
)
2479 if os
.path
.exists(encodeFilename(file)):
2480 existing_files
.append(file)
2482 if not existing_files
or self
.params
.get('overwrites', False):
2483 for file in orderedSet(existing_files
):
2484 self
.report_file_delete(file)
2485 os
.remove(encodeFilename(file))
2488 self
.report_file_already_downloaded(existing_files
[0])
2489 info_dict
['ext'] = os
.path
.splitext(existing_files
[0])[1][1:]
2490 return existing_files
[0]
2493 if info_dict
.get('requested_formats') is not None:
2495 merger
= FFmpegMergerPP(self
)
2496 if self
.params
.get('allow_unplayable_formats'):
2497 self
.report_warning(
2498 'You have requested merging of multiple formats '
2499 'while also allowing unplayable formats to be downloaded. '
2500 'The formats won\'t be merged to prevent data corruption.')
2501 elif not merger
.available
:
2502 self
.report_warning(
2503 'You have requested merging of multiple formats but ffmpeg is not installed. '
2504 'The formats won\'t be merged.')
2506 def compatible_formats(formats
):
2507 # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
2508 video_formats
= [format
for format
in formats
if format
.get('vcodec') != 'none']
2509 audio_formats
= [format
for format
in formats
if format
.get('acodec') != 'none']
2510 if len(video_formats
) > 2 or len(audio_formats
) > 2:
2514 exts
= set(format
.get('ext') for format
in formats
)
2516 set(('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma')),
2519 for ext_sets
in COMPATIBLE_EXTS
:
2520 if ext_sets
.issuperset(exts
):
2522 # TODO: Check acodec/vcodec
2525 requested_formats
= info_dict
['requested_formats']
2526 old_ext
= info_dict
['ext']
2527 if self
.params
.get('merge_output_format') is None:
2528 if not compatible_formats(requested_formats
):
2529 info_dict
['ext'] = 'mkv'
2530 self
.report_warning(
2531 'Requested formats are incompatible for merge and will be merged into mkv.')
2532 if (info_dict
['ext'] == 'webm'
2533 and self
.params
.get('writethumbnail', False)
2534 and info_dict
.get('thumbnails')):
2535 info_dict
['ext'] = 'mkv'
2536 self
.report_warning(
2537 'webm doesn\'t support embedding a thumbnail, mkv will be used.')
2539 def correct_ext(filename
):
2540 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
2542 os
.path
.splitext(filename
)[0]
2543 if filename_real_ext
== old_ext
2545 return '%s.%s' % (filename_wo_ext
, info_dict
['ext'])
2547 # Ensure filename always has a correct extension for successful merge
2548 full_filename
= correct_ext(full_filename
)
2549 temp_filename
= correct_ext(temp_filename
)
2550 dl_filename
= existing_file(full_filename
, temp_filename
)
2551 info_dict
['__real_download'] = False
2552 if dl_filename
is None:
2553 for f
in requested_formats
:
2554 new_info
= dict(info_dict
)
2556 fname
= prepend_extension(
2557 self
.prepare_filename(new_info
, 'temp'),
2558 'f%s' % f
['format_id'], new_info
['ext'])
2559 if not self
._ensure
_dir
_exists
(fname
):
2561 downloaded
.append(fname
)
2562 partial_success
, real_download
= self
.dl(fname
, new_info
)
2563 info_dict
['__real_download'] = info_dict
['__real_download'] or real_download
2564 success
= success
and partial_success
2565 if merger
.available
and not self
.params
.get('allow_unplayable_formats'):
2566 info_dict
['__postprocessors'].append(merger
)
2567 info_dict
['__files_to_merge'] = downloaded
2568 # Even if there were no downloads, it is being merged only now
2569 info_dict
['__real_download'] = True
2571 for file in downloaded
:
2572 files_to_move
[file] = None
2574 # Just a single file
2575 dl_filename
= existing_file(full_filename
, temp_filename
)
2576 if dl_filename
is None:
2577 success
, real_download
= self
.dl(temp_filename
, info_dict
)
2578 info_dict
['__real_download'] = real_download
2580 dl_filename
= dl_filename
or temp_filename
2581 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2583 except network_exceptions
as err
:
2584 self
.report_error('unable to download video data: %s' % error_to_compat_str(err
))
2586 except (OSError, IOError) as err
:
2587 raise UnavailableVideoError(err
)
2588 except (ContentTooShortError
, ) as err
:
2589 self
.report_error('content too short (expected %s bytes and served %s)' % (err
.expected
, err
.downloaded
))
2592 if success
and full_filename
!= '-':
2594 fixup_policy
= self
.params
.get('fixup')
2595 if fixup_policy
is None:
2596 fixup_policy
= 'detect_or_warn'
2598 INSTALL_FFMPEG_MESSAGE
= 'Install ffmpeg to fix this automatically.'
2600 stretched_ratio
= info_dict
.get('stretched_ratio')
2601 if stretched_ratio
is not None and stretched_ratio
!= 1:
2602 if fixup_policy
== 'warn':
2603 self
.report_warning('%s: Non-uniform pixel ratio (%s)' % (
2604 info_dict
['id'], stretched_ratio
))
2605 elif fixup_policy
== 'detect_or_warn':
2606 stretched_pp
= FFmpegFixupStretchedPP(self
)
2607 if stretched_pp
.available
:
2608 info_dict
['__postprocessors'].append(stretched_pp
)
2610 self
.report_warning(
2611 '%s: Non-uniform pixel ratio (%s). %s'
2612 % (info_dict
['id'], stretched_ratio
, INSTALL_FFMPEG_MESSAGE
))
2614 assert fixup_policy
in ('ignore', 'never')
2616 if (info_dict
.get('requested_formats') is None
2617 and info_dict
.get('container') == 'm4a_dash'
2618 and info_dict
.get('ext') == 'm4a'):
2619 if fixup_policy
== 'warn':
2620 self
.report_warning(
2621 '%s: writing DASH m4a. '
2622 'Only some players support this container.'
2624 elif fixup_policy
== 'detect_or_warn':
2625 fixup_pp
= FFmpegFixupM4aPP(self
)
2626 if fixup_pp
.available
:
2627 info_dict
['__postprocessors'].append(fixup_pp
)
2629 self
.report_warning(
2630 '%s: writing DASH m4a. '
2631 'Only some players support this container. %s'
2632 % (info_dict
['id'], INSTALL_FFMPEG_MESSAGE
))
2634 assert fixup_policy
in ('ignore', 'never')
2636 if ('protocol' in info_dict
2637 and get_suitable_downloader(info_dict
, self
.params
).__name
__ == 'HlsFD'):
2638 if fixup_policy
== 'warn':
2639 self
.report_warning('%s: malformed AAC bitstream detected.' % (
2641 elif fixup_policy
== 'detect_or_warn':
2642 fixup_pp
= FFmpegFixupM3u8PP(self
)
2643 if fixup_pp
.available
:
2644 info_dict
['__postprocessors'].append(fixup_pp
)
2646 self
.report_warning(
2647 '%s: malformed AAC bitstream detected. %s'
2648 % (info_dict
['id'], INSTALL_FFMPEG_MESSAGE
))
2650 assert fixup_policy
in ('ignore', 'never')
2653 info_dict
= self
.post_process(dl_filename
, info_dict
, files_to_move
)
2654 except PostProcessingError
as err
:
2655 self
.report_error('Postprocessing: %s' % str(err
))
2658 for ph
in self
._post
_hooks
:
2659 ph(info_dict
['filepath'])
2660 except Exception as err
:
2661 self
.report_error('post hooks: %s' % str(err
))
2663 must_record_download_archive
= True
2665 if must_record_download_archive
or self
.params
.get('force_write_download_archive', False):
2666 self
.record_download_archive(info_dict
)
2667 max_downloads
= self
.params
.get('max_downloads')
2668 if max_downloads
is not None and self
._num
_downloads
>= int(max_downloads
):
2669 raise MaxDownloadsReached()
2671 def download(self
, url_list
):
2672 """Download a given list of URLs."""
2673 outtmpl
= self
.outtmpl_dict
['default']
2674 if (len(url_list
) > 1
2676 and '%' not in outtmpl
2677 and self
.params
.get('max_downloads') != 1):
2678 raise SameFileError(outtmpl
)
2680 for url
in url_list
:
2682 # It also downloads the videos
2683 res
= self
.extract_info(
2684 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
2685 except UnavailableVideoError
:
2686 self
.report_error('unable to download video')
2687 except MaxDownloadsReached
:
2688 self
.to_screen('[info] Maximum number of downloaded files reached')
2690 except ExistingVideoReached
:
2691 self
.to_screen('[info] Encountered a file that is already in the archive, stopping due to --break-on-existing')
2693 except RejectedVideoReached
:
2694 self
.to_screen('[info] Encountered a file that did not match filter, stopping due to --break-on-reject')
2697 if self
.params
.get('dump_single_json', False):
2698 self
.post_extract(res
)
2699 self
.to_stdout(json
.dumps(res
, default
=repr))
2701 return self
._download
_retcode
2703 def download_with_info_file(self
, info_filename
):
2704 with contextlib
.closing(fileinput
.FileInput(
2705 [info_filename
], mode
='r',
2706 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
2707 # FileInput doesn't have a read method, we can't call json.load
2708 info
= self
.filter_requested_info(json
.loads('\n'.join(f
)), self
.params
.get('clean_infojson', True))
2710 self
.process_ie_result(info
, download
=True)
2711 except (DownloadError
, EntryNotInPlaylist
):
2712 webpage_url
= info
.get('webpage_url')
2713 if webpage_url
is not None:
2714 self
.report_warning('The info failed to download, trying with "%s"' % webpage_url
)
2715 return self
.download([webpage_url
])
2718 return self
._download
_retcode
2721 def filter_requested_info(info_dict
, actually_filter
=True):
2722 if not actually_filter
:
2723 info_dict
['epoch'] = int(time
.time())
2726 'remove': ['requested_formats', 'requested_subtitles', 'requested_entries', 'filepath', 'entries'],
2729 keep_key
= lambda k
: k
in exceptions
['keep'] or not (k
.startswith('_') or k
in exceptions
['remove'])
2730 filter_fn
= lambda obj
: (
2731 list(map(filter_fn
, obj
)) if isinstance(obj
, (list, tuple))
2732 else obj
if not isinstance(obj
, dict)
2733 else dict((k
, filter_fn(v
)) for k
, v
in obj
.items() if keep_key(k
)))
2734 return filter_fn(info_dict
)
2736 def run_pp(self
, pp
, infodict
):
2737 files_to_delete
= []
2738 if '__files_to_move' not in infodict
:
2739 infodict
['__files_to_move'] = {}
2740 files_to_delete
, infodict
= pp
.run(infodict
)
2741 if not files_to_delete
:
2744 if self
.params
.get('keepvideo', False):
2745 for f
in files_to_delete
:
2746 infodict
['__files_to_move'].setdefault(f
, '')
2748 for old_filename
in set(files_to_delete
):
2749 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
2751 os
.remove(encodeFilename(old_filename
))
2752 except (IOError, OSError):
2753 self
.report_warning('Unable to remove downloaded original file')
2754 if old_filename
in infodict
['__files_to_move']:
2755 del infodict
['__files_to_move'][old_filename
]
2759 def post_extract(info_dict
):
2760 def actual_post_extract(info_dict
):
2761 if info_dict
.get('_type') in ('playlist', 'multi_video'):
2762 for video_dict
in info_dict
.get('entries', {}):
2763 actual_post_extract(video_dict
or {})
2766 if '__post_extractor' not in info_dict
:
2768 post_extractor
= info_dict
['__post_extractor']
2770 info_dict
.update(post_extractor().items())
2771 del info_dict
['__post_extractor']
2774 actual_post_extract(info_dict
or {})
2776 def pre_process(self
, ie_info
, key
='pre_process', files_to_move
=None):
2777 info
= dict(ie_info
)
2778 info
['__files_to_move'] = files_to_move
or {}
2779 for pp
in self
._pps
[key
]:
2780 info
= self
.run_pp(pp
, info
)
2781 return info
, info
.pop('__files_to_move', None)
2783 def post_process(self
, filename
, ie_info
, files_to_move
=None):
2784 """Run all the postprocessors on the given file."""
2785 info
= dict(ie_info
)
2786 info
['filepath'] = filename
2787 info
['__files_to_move'] = files_to_move
or {}
2789 for pp
in ie_info
.get('__postprocessors', []) + self
._pps
['post_process']:
2790 info
= self
.run_pp(pp
, info
)
2791 info
= self
.run_pp(MoveFilesAfterDownloadPP(self
), info
)
2792 del info
['__files_to_move']
2793 for pp
in self
._pps
['after_move']:
2794 info
= self
.run_pp(pp
, info
)
2797 def _make_archive_id(self
, info_dict
):
2798 video_id
= info_dict
.get('id')
2801 # Future-proof against any change in case
2802 # and backwards compatibility with prior versions
2803 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
2804 if extractor
is None:
2805 url
= str_or_none(info_dict
.get('url'))
2808 # Try to find matching extractor for the URL and take its ie_key
2809 for ie
in self
._ies
:
2810 if ie
.suitable(url
):
2811 extractor
= ie
.ie_key()
2815 return '%s %s' % (extractor
.lower(), video_id
)
2817 def in_download_archive(self
, info_dict
):
2818 fn
= self
.params
.get('download_archive')
2822 vid_id
= self
._make
_archive
_id
(info_dict
)
2824 return False # Incomplete video information
2826 return vid_id
in self
.archive
2828 def record_download_archive(self
, info_dict
):
2829 fn
= self
.params
.get('download_archive')
2832 vid_id
= self
._make
_archive
_id
(info_dict
)
2834 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
2835 archive_file
.write(vid_id
+ '\n')
2836 self
.archive
.add(vid_id
)
2839 def format_resolution(format
, default
='unknown'):
2840 if format
.get('vcodec') == 'none':
2842 if format
.get('resolution') is not None:
2843 return format
['resolution']
2844 if format
.get('width') and format
.get('height'):
2845 res
= '%dx%d' % (format
['width'], format
['height'])
2846 elif format
.get('height'):
2847 res
= '%sp' % format
['height']
2848 elif format
.get('width'):
2849 res
= '%dx?' % format
['width']
2854 def _format_note(self
, fdict
):
2856 if fdict
.get('ext') in ['f4f', 'f4m']:
2857 res
+= '(unsupported) '
2858 if fdict
.get('language'):
2861 res
+= '[%s] ' % fdict
['language']
2862 if fdict
.get('format_note') is not None:
2863 res
+= fdict
['format_note'] + ' '
2864 if fdict
.get('tbr') is not None:
2865 res
+= '%4dk ' % fdict
['tbr']
2866 if fdict
.get('container') is not None:
2869 res
+= '%s container' % fdict
['container']
2870 if (fdict
.get('vcodec') is not None
2871 and fdict
.get('vcodec') != 'none'):
2874 res
+= fdict
['vcodec']
2875 if fdict
.get('vbr') is not None:
2877 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
2879 if fdict
.get('vbr') is not None:
2880 res
+= '%4dk' % fdict
['vbr']
2881 if fdict
.get('fps') is not None:
2884 res
+= '%sfps' % fdict
['fps']
2885 if fdict
.get('acodec') is not None:
2888 if fdict
['acodec'] == 'none':
2891 res
+= '%-5s' % fdict
['acodec']
2892 elif fdict
.get('abr') is not None:
2896 if fdict
.get('abr') is not None:
2897 res
+= '@%3dk' % fdict
['abr']
2898 if fdict
.get('asr') is not None:
2899 res
+= ' (%5dHz)' % fdict
['asr']
2900 if fdict
.get('filesize') is not None:
2903 res
+= format_bytes(fdict
['filesize'])
2904 elif fdict
.get('filesize_approx') is not None:
2907 res
+= '~' + format_bytes(fdict
['filesize_approx'])
2910 def _format_note_table(self
, f
):
2911 def join_fields(*vargs
):
2912 return ', '.join((val
for val
in vargs
if val
!= ''))
2915 'UNSUPPORTED' if f
.get('ext') in ('f4f', 'f4m') else '',
2916 format_field(f
, 'language', '[%s]'),
2917 format_field(f
, 'format_note'),
2918 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
2919 format_field(f
, 'asr', '%5dHz'))
2921 def list_formats(self
, info_dict
):
2922 formats
= info_dict
.get('formats', [info_dict
])
2924 'list-formats' not in self
.params
.get('compat_opts', [])
2925 and self
.params
.get('list_formats_as_table', True) is not False)
2929 format_field(f
, 'format_id'),
2930 format_field(f
, 'ext'),
2931 self
.format_resolution(f
),
2932 format_field(f
, 'fps', '%d'),
2934 format_field(f
, 'filesize', ' %s', func
=format_bytes
) + format_field(f
, 'filesize_approx', '~%s', func
=format_bytes
),
2935 format_field(f
, 'tbr', '%4dk'),
2936 shorten_protocol_name(f
.get('protocol', '').replace("native", "n")),
2938 format_field(f
, 'vcodec', default
='unknown').replace('none', ''),
2939 format_field(f
, 'vbr', '%4dk'),
2940 format_field(f
, 'acodec', default
='unknown').replace('none', ''),
2941 format_field(f
, 'abr', '%3dk'),
2942 format_field(f
, 'asr', '%5dHz'),
2943 self
._format
_note
_table
(f
)]
2945 if f
.get('preference') is None or f
['preference'] >= -1000]
2946 header_line
= ['ID', 'EXT', 'RESOLUTION', 'FPS', '|', ' FILESIZE', ' TBR', 'PROTO',
2947 '|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'NOTE']
2951 format_field(f
, 'format_id'),
2952 format_field(f
, 'ext'),
2953 self
.format_resolution(f
),
2954 self
._format
_note
(f
)]
2956 if f
.get('preference') is None or f
['preference'] >= -1000]
2957 header_line
= ['format code', 'extension', 'resolution', 'note']
2960 '[info] Available formats for %s:\n%s' % (info_dict
['id'], render_table(
2964 extraGap
=(0 if new_format
else 1),
2965 hideEmpty
=new_format
)))
2967 def list_thumbnails(self
, info_dict
):
2968 thumbnails
= info_dict
.get('thumbnails')
2970 self
.to_screen('[info] No thumbnails present for %s' % info_dict
['id'])
2974 '[info] Thumbnails for %s:' % info_dict
['id'])
2975 self
.to_screen(render_table(
2976 ['ID', 'width', 'height', 'URL'],
2977 [[t
['id'], t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
]))
2979 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
2981 self
.to_screen('%s has no %s' % (video_id
, name
))
2984 'Available %s for %s:' % (name
, video_id
))
2985 self
.to_screen(render_table(
2986 ['Language', 'formats'],
2987 [[lang
, ', '.join(f
['ext'] for f
in reversed(formats
))]
2988 for lang
, formats
in subtitles
.items()]))
2990 def urlopen(self
, req
):
2991 """ Start an HTTP download """
2992 if isinstance(req
, compat_basestring
):
2993 req
= sanitized_Request(req
)
2994 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
2996 def print_debug_header(self
):
2997 if not self
.params
.get('verbose'):
3000 if type('') is not compat_str
:
3001 # Python 2.6 on SLES11 SP1 (https://github.com/ytdl-org/youtube-dl/issues/3326)
3002 self
.report_warning(
3003 'Your Python is broken! Update to a newer and supported version')
3005 stdout_encoding
= getattr(
3006 sys
.stdout
, 'encoding', 'missing (%s)' % type(sys
.stdout
).__name
__)
3008 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
3009 locale
.getpreferredencoding(),
3010 sys
.getfilesystemencoding(),
3012 self
.get_encoding()))
3013 write_string(encoding_str
, encoding
=None)
3016 '(exe)' if hasattr(sys
, 'frozen')
3017 else '(zip)' if isinstance(globals().get('__loader__'), zipimporter
)
3018 else '(source)' if os
.path
.basename(sys
.argv
[0]) == '__main__.py'
3020 self
._write
_string
('[debug] yt-dlp version %s %s\n' % (__version__
, source
))
3022 self
._write
_string
('[debug] Lazy loading extractors enabled\n')
3025 '[debug] Plugin Extractors: %s\n' % [ie
.ie_key() for ie
in _PLUGIN_CLASSES
])
3026 if self
.params
.get('compat_opts'):
3028 '[debug] Compatibility options: %s\n' % ', '.join(self
.params
.get('compat_opts')))
3030 sp
= subprocess
.Popen(
3031 ['git', 'rev-parse', '--short', 'HEAD'],
3032 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
3033 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
3034 out
, err
= process_communicate_or_kill(sp
)
3035 out
= out
.decode().strip()
3036 if re
.match('[0-9a-f]+', out
):
3037 self
._write
_string
('[debug] Git HEAD: %s\n' % out
)
3044 def python_implementation():
3045 impl_name
= platform
.python_implementation()
3046 if impl_name
== 'PyPy' and hasattr(sys
, 'pypy_version_info'):
3047 return impl_name
+ ' version %d.%d.%d' % sys
.pypy_version_info
[:3]
3050 self
._write
_string
('[debug] Python version %s (%s %s) - %s\n' % (
3051 platform
.python_version(),
3052 python_implementation(),
3053 platform
.architecture()[0],
3056 exe_versions
= FFmpegPostProcessor
.get_versions(self
)
3057 exe_versions
['rtmpdump'] = rtmpdump_version()
3058 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
3059 exe_str
= ', '.join(
3061 for exe
, v
in sorted(exe_versions
.items())
3066 self
._write
_string
('[debug] exe versions: %s\n' % exe_str
)
3069 for handler
in self
._opener
.handlers
:
3070 if hasattr(handler
, 'proxies'):
3071 proxy_map
.update(handler
.proxies
)
3072 self
._write
_string
('[debug] Proxy map: ' + compat_str(proxy_map
) + '\n')
3074 if self
.params
.get('call_home', False):
3075 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
3076 self
._write
_string
('[debug] Public IP address: %s\n' % ipaddr
)
3078 latest_version
= self
.urlopen(
3079 'https://yt-dl.org/latest/version').read().decode('utf-8')
3080 if version_tuple(latest_version
) > version_tuple(__version__
):
3081 self
.report_warning(
3082 'You are using an outdated version (newest version: %s)! '
3083 'See https://yt-dl.org/update if you need help updating.' %
3086 def _setup_opener(self
):
3087 timeout_val
= self
.params
.get('socket_timeout')
3088 self
._socket
_timeout
= 600 if timeout_val
is None else float(timeout_val
)
3090 opts_cookiefile
= self
.params
.get('cookiefile')
3091 opts_proxy
= self
.params
.get('proxy')
3093 if opts_cookiefile
is None:
3094 self
.cookiejar
= compat_cookiejar
.CookieJar()
3096 opts_cookiefile
= expand_path(opts_cookiefile
)
3097 self
.cookiejar
= YoutubeDLCookieJar(opts_cookiefile
)
3098 if os
.access(opts_cookiefile
, os
.R_OK
):
3099 self
.cookiejar
.load(ignore_discard
=True, ignore_expires
=True)
3101 cookie_processor
= YoutubeDLCookieProcessor(self
.cookiejar
)
3102 if opts_proxy
is not None:
3103 if opts_proxy
== '':
3106 proxies
= {'http': opts_proxy, 'https': opts_proxy}
3108 proxies
= compat_urllib_request
.getproxies()
3109 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
3110 if 'http' in proxies
and 'https' not in proxies
:
3111 proxies
['https'] = proxies
['http']
3112 proxy_handler
= PerRequestProxyHandler(proxies
)
3114 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
3115 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
3116 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
3117 redirect_handler
= YoutubeDLRedirectHandler()
3118 data_handler
= compat_urllib_request_DataHandler()
3120 # When passing our own FileHandler instance, build_opener won't add the
3121 # default FileHandler and allows us to disable the file protocol, which
3122 # can be used for malicious purposes (see
3123 # https://github.com/ytdl-org/youtube-dl/issues/8227)
3124 file_handler
= compat_urllib_request
.FileHandler()
3126 def file_open(*args
, **kwargs
):
3127 raise compat_urllib_error
.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
3128 file_handler
.file_open
= file_open
3130 opener
= compat_urllib_request
.build_opener(
3131 proxy_handler
, https_handler
, cookie_processor
, ydlh
, redirect_handler
, data_handler
, file_handler
)
3133 # Delete the default user-agent header, which would otherwise apply in
3134 # cases where our custom HTTP handler doesn't come into play
3135 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
3136 opener
.addheaders
= []
3137 self
._opener
= opener
3139 def encode(self
, s
):
3140 if isinstance(s
, bytes):
3141 return s
# Already encoded
3144 return s
.encode(self
.get_encoding())
3145 except UnicodeEncodeError as err
:
3146 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
3149 def get_encoding(self
):
3150 encoding
= self
.params
.get('encoding')
3151 if encoding
is None:
3152 encoding
= preferredencoding()
3155 def _write_thumbnails(self
, info_dict
, filename
): # return the extensions
3156 write_all
= self
.params
.get('write_all_thumbnails', False)
3158 if write_all
or self
.params
.get('writethumbnail', False):
3159 thumbnails
= info_dict
.get('thumbnails') or []
3160 multiple
= write_all
and len(thumbnails
) > 1
3163 for t
in thumbnails
[::1 if write_all
else -1]:
3164 thumb_ext
= determine_ext(t
['url'], 'jpg')
3165 suffix
= '%s.' % t
['id'] if multiple
else ''
3166 thumb_display_id
= '%s ' % t
['id'] if multiple
else ''
3167 t
['filepath'] = thumb_filename
= replace_extension(filename
, suffix
+ thumb_ext
, info_dict
.get('ext'))
3169 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(thumb_filename
)):
3170 ret
.append(suffix
+ thumb_ext
)
3171 self
.to_screen('[%s] %s: Thumbnail %sis already present' %
3172 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
3174 self
.to_screen('[%s] %s: Downloading thumbnail %s ...' %
3175 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
3177 uf
= self
.urlopen(t
['url'])
3178 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
3179 shutil
.copyfileobj(uf
, thumbf
)
3180 ret
.append(suffix
+ thumb_ext
)
3181 self
.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
3182 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
, thumb_filename
))
3183 except network_exceptions
as err
:
3184 self
.report_warning('Unable to download thumbnail "%s": %s' %
3185 (t
['url'], error_to_compat_str(err
)))
3186 if ret
and not write_all
: