4 from __future__
import absolute_import
, unicode_literals
29 from string
import ascii_letters
30 from zipimport
import zipimporter
35 compat_get_terminal_size
,
40 compat_tokenize_tokenize
,
42 compat_urllib_request
,
43 compat_urllib_request_DataHandler
,
54 DOT_DESKTOP_LINK_TEMPLATE
,
55 DOT_URL_LINK_TEMPLATE
,
56 DOT_WEBLOC_LINK_TEMPLATE
,
85 PerRequestProxyHandler
,
90 process_communicate_or_kill
,
91 register_socks_protocols
,
108 UnavailableVideoError
,
114 YoutubeDLCookieProcessor
,
116 YoutubeDLRedirectHandler
,
118 from .cache
import Cache
119 from .extractor
import (
120 gen_extractor_classes
,
125 from .extractor
.openload
import PhantomJSwrapper
126 from .downloader
import (
127 get_suitable_downloader
,
128 shorten_protocol_name
130 from .downloader
.rtmp
import rtmpdump_version
131 from .postprocessor
import (
133 FFmpegFixupDurationPP
,
136 FFmpegFixupStretchedPP
,
137 FFmpegFixupTimestampPP
,
140 MoveFilesAfterDownloadPP
,
142 from .version
import __version__
144 if compat_os_name
== 'nt':
148 class YoutubeDL(object):
151 YoutubeDL objects are the ones responsible of downloading the
152 actual video file and writing it to disk if the user has requested
153 it, among some other tasks. In most cases there should be one per
154 program. As, given a video URL, the downloader doesn't know how to
155 extract all the needed information, task that InfoExtractors do, it
156 has to pass the URL to one of them.
158 For this, YoutubeDL objects have a method that allows
159 InfoExtractors to be registered in a given order. When it is passed
160 a URL, the YoutubeDL object handles it to the first InfoExtractor it
161 finds that reports being able to handle it. The InfoExtractor extracts
162 all the information about the video or videos the URL refers to, and
163 YoutubeDL process the extracted information, possibly using a File
164 Downloader to download the video.
166 YoutubeDL objects accept a lot of parameters. In order not to saturate
167 the object constructor with arguments, it receives a dictionary of
168 options instead. These options are available through the params
169 attribute for the InfoExtractors to use. The YoutubeDL also
170 registers itself as the downloader in charge for the InfoExtractors
171 that are added to it, so this is a "mutual registration".
175 username: Username for authentication purposes.
176 password: Password for authentication purposes.
177 videopassword: Password for accessing a video.
178 ap_mso: Adobe Pass multiple-system operator identifier.
179 ap_username: Multiple-system operator account username.
180 ap_password: Multiple-system operator account password.
181 usenetrc: Use netrc for authentication instead.
182 verbose: Print additional info to stdout.
183 quiet: Do not print messages to stdout.
184 no_warnings: Do not print out anything for warnings.
185 forceprint: A list of templates to force print
186 forceurl: Force printing final URL. (Deprecated)
187 forcetitle: Force printing title. (Deprecated)
188 forceid: Force printing ID. (Deprecated)
189 forcethumbnail: Force printing thumbnail URL. (Deprecated)
190 forcedescription: Force printing description. (Deprecated)
191 forcefilename: Force printing final filename. (Deprecated)
192 forceduration: Force printing duration. (Deprecated)
193 forcejson: Force printing info_dict as JSON.
194 dump_single_json: Force printing the info_dict of the whole playlist
195 (or video) as a single JSON line.
196 force_write_download_archive: Force writing download archive regardless
197 of 'skip_download' or 'simulate'.
198 simulate: Do not download the video files.
199 format: Video format code. see "FORMAT SELECTION" for more details.
200 allow_unplayable_formats: Allow unplayable formats to be extracted and downloaded.
201 ignore_no_formats_error: Ignore "No video formats" error. Usefull for
202 extracting metadata even if the video is not actually
203 available for download (experimental)
204 format_sort: How to sort the video formats. see "Sorting Formats"
206 format_sort_force: Force the given format_sort. see "Sorting Formats"
208 allow_multiple_video_streams: Allow multiple video streams to be merged
210 allow_multiple_audio_streams: Allow multiple audio streams to be merged
212 paths: Dictionary of output paths. The allowed keys are 'home'
213 'temp' and the keys of OUTTMPL_TYPES (in utils.py)
214 outtmpl: Dictionary of templates for output names. Allowed keys
215 are 'default' and the keys of OUTTMPL_TYPES (in utils.py).
216 A string a also accepted for backward compatibility
217 outtmpl_na_placeholder: Placeholder for unavailable meta fields.
218 restrictfilenames: Do not allow "&" and spaces in file names
219 trim_file_name: Limit length of filename (extension excluded)
220 windowsfilenames: Force the filenames to be windows compatible
221 ignoreerrors: Do not stop on download errors
222 (Default True when running yt-dlp,
223 but False when directly accessing YoutubeDL class)
224 skip_playlist_after_errors: Number of allowed failures until the rest of
225 the playlist is skipped
226 force_generic_extractor: Force downloader to use the generic extractor
227 overwrites: Overwrite all video and metadata files if True,
228 overwrite only non-video files if None
229 and don't overwrite any file if False
230 playliststart: Playlist item to start at.
231 playlistend: Playlist item to end at.
232 playlist_items: Specific indices of playlist to download.
233 playlistreverse: Download playlist items in reverse order.
234 playlistrandom: Download playlist items in random order.
235 matchtitle: Download only matching titles.
236 rejecttitle: Reject downloads for matching titles.
237 logger: Log messages to a logging.Logger instance.
238 logtostderr: Log messages to stderr instead of stdout.
239 writedescription: Write the video description to a .description file
240 writeinfojson: Write the video description to a .info.json file
241 clean_infojson: Remove private fields from the infojson
242 writecomments: Extract video comments. This will not be written to disk
243 unless writeinfojson is also given
244 writeannotations: Write the video annotations to a .annotations.xml file
245 writethumbnail: Write the thumbnail image to a file
246 allow_playlist_files: Whether to write playlists' description, infojson etc
247 also to disk when using the 'write*' options
248 write_all_thumbnails: Write all thumbnail formats to files
249 writelink: Write an internet shortcut file, depending on the
250 current platform (.url/.webloc/.desktop)
251 writeurllink: Write a Windows internet shortcut file (.url)
252 writewebloclink: Write a macOS internet shortcut file (.webloc)
253 writedesktoplink: Write a Linux internet shortcut file (.desktop)
254 writesubtitles: Write the video subtitles to a file
255 writeautomaticsub: Write the automatically generated subtitles to a file
256 allsubtitles: Deprecated - Use subtitlelangs = ['all']
257 Downloads all the subtitles of the video
258 (requires writesubtitles or writeautomaticsub)
259 listsubtitles: Lists all available subtitles for the video
260 subtitlesformat: The format code for subtitles
261 subtitleslangs: List of languages of the subtitles to download (can be regex).
262 The list may contain "all" to refer to all the available
263 subtitles. The language can be prefixed with a "-" to
264 exclude it from the requested languages. Eg: ['all', '-live_chat']
265 keepvideo: Keep the video file after post-processing
266 daterange: A DateRange object, download only if the upload_date is in the range.
267 skip_download: Skip the actual download of the video file
268 cachedir: Location of the cache files in the filesystem.
269 False to disable filesystem cache.
270 noplaylist: Download single video instead of a playlist if in doubt.
271 age_limit: An integer representing the user's age in years.
272 Unsuitable videos for the given age are skipped.
273 min_views: An integer representing the minimum view count the video
274 must have in order to not be skipped.
275 Videos without view count information are always
276 downloaded. None for no limit.
277 max_views: An integer representing the maximum view count.
278 Videos that are more popular than that are not
280 Videos without view count information are always
281 downloaded. None for no limit.
282 download_archive: File name of a file where all downloads are recorded.
283 Videos already present in the file are not downloaded
285 break_on_existing: Stop the download process after attempting to download a
286 file that is in the archive.
287 break_on_reject: Stop the download process when encountering a video that
288 has been filtered out.
289 cookiefile: File name where cookies should be read from and dumped to
290 nocheckcertificate:Do not verify SSL certificates
291 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
292 At the moment, this is only supported by YouTube.
293 proxy: URL of the proxy server to use
294 geo_verification_proxy: URL of the proxy to use for IP address verification
295 on geo-restricted sites.
296 socket_timeout: Time to wait for unresponsive hosts, in seconds
297 bidi_workaround: Work around buggy terminals without bidirectional text
298 support, using fridibi
299 debug_printtraffic:Print out sent and received HTTP traffic
300 include_ads: Download ads as well
301 default_search: Prepend this string if an input url is not valid.
302 'auto' for elaborate guessing
303 encoding: Use this encoding instead of the system-specified.
304 extract_flat: Do not resolve URLs, return the immediate result.
305 Pass in 'in_playlist' to only show this behavior for
307 postprocessors: A list of dictionaries, each with an entry
308 * key: The name of the postprocessor. See
309 yt_dlp/postprocessor/__init__.py for a list.
310 * when: When to run the postprocessor. Can be one of
311 pre_process|before_dl|post_process|after_move.
312 Assumed to be 'post_process' if not given
313 post_hooks: A list of functions that get called as the final step
314 for each video file, after all postprocessors have been
315 called. The filename will be passed as the only argument.
316 progress_hooks: A list of functions that get called on download
317 progress, with a dictionary with the entries
318 * status: One of "downloading", "error", or "finished".
319 Check this first and ignore unknown values.
321 If status is one of "downloading", or "finished", the
322 following properties may also be present:
323 * filename: The final filename (always present)
324 * tmpfilename: The filename we're currently writing to
325 * downloaded_bytes: Bytes on disk
326 * total_bytes: Size of the whole file, None if unknown
327 * total_bytes_estimate: Guess of the eventual file size,
329 * elapsed: The number of seconds since download started.
330 * eta: The estimated time in seconds, None if unknown
331 * speed: The download speed in bytes/second, None if
333 * fragment_index: The counter of the currently
334 downloaded video fragment.
335 * fragment_count: The number of fragments (= individual
336 files that will be merged)
338 Progress hooks are guaranteed to be called at least once
339 (with status "finished") if the download is successful.
340 merge_output_format: Extension to use when merging formats.
341 final_ext: Expected final extension; used to detect when the file was
342 already downloaded and converted. "merge_output_format" is
343 replaced by this extension when given
344 fixup: Automatically correct known faults of the file.
346 - "never": do nothing
347 - "warn": only emit a warning
348 - "detect_or_warn": check whether we can do anything
349 about it, warn otherwise (default)
350 source_address: Client-side IP address to bind to.
351 call_home: Boolean, true iff we are allowed to contact the
352 yt-dlp servers for debugging. (BROKEN)
353 sleep_interval_requests: Number of seconds to sleep between requests
355 sleep_interval: Number of seconds to sleep before each download when
356 used alone or a lower bound of a range for randomized
357 sleep before each download (minimum possible number
358 of seconds to sleep) when used along with
360 max_sleep_interval:Upper bound of a range for randomized sleep before each
361 download (maximum possible number of seconds to sleep).
362 Must only be used along with sleep_interval.
363 Actual sleep time will be a random float from range
364 [sleep_interval; max_sleep_interval].
365 sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
366 listformats: Print an overview of available video formats and exit.
367 list_thumbnails: Print a table of all thumbnails and exit.
368 match_filter: A function that gets called with the info_dict of
370 If it returns a message, the video is ignored.
371 If it returns None, the video is downloaded.
372 match_filter_func in utils.py is one example for this.
373 no_color: Do not emit color codes in output.
374 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
377 Two-letter ISO 3166-2 country code that will be used for
378 explicit geographic restriction bypassing via faking
379 X-Forwarded-For HTTP header
381 IP range in CIDR notation that will be used similarly to
384 The following options determine which downloader is picked:
385 external_downloader: A dictionary of protocol keys and the executable of the
386 external downloader to use for it. The allowed protocols
387 are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
388 Set the value to 'native' to use the native downloader
389 hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
390 or {'m3u8': 'ffmpeg'} instead.
391 Use the native HLS downloader instead of ffmpeg/avconv
392 if True, otherwise use ffmpeg/avconv if False, otherwise
393 use downloader suggested by extractor if None.
394 compat_opts: Compatibility options. See "Differences in default behavior".
395 The following options do not work when used through the API:
396 filename, abort-on-error, multistreams, no-live-chat,
397 no-playlist-metafiles. Refer __init__.py for their implementation
399 The following parameters are not used by YoutubeDL itself, they are used by
400 the downloader (see yt_dlp/downloader/common.py):
401 nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
402 max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle,
403 xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size.
405 The following options are used by the post processors:
406 prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
407 otherwise prefer ffmpeg. (avconv support is deprecated)
408 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
409 to the binary or its containing directory.
410 postprocessor_args: A dictionary of postprocessor/executable keys (in lower case)
411 and a list of additional command-line arguments for the
412 postprocessor/executable. The dict can also have "PP+EXE" keys
413 which are used when the given exe is used by the given PP.
414 Use 'default' as the name for arguments to passed to all PP
416 The following options are used by the extractors:
417 extractor_retries: Number of times to retry for known errors
418 dynamic_mpd: Whether to process dynamic DASH manifests (default: True)
419 hls_split_discontinuity: Split HLS playlists to different formats at
420 discontinuities such as ad breaks (default: False)
421 extractor_args: A dictionary of arguments to be passed to the extractors.
422 See "EXTRACTOR ARGUMENTS" for details.
423 Eg: {'youtube': {'skip': ['dash', 'hls']}}
424 youtube_include_dash_manifest: Deprecated - Use extractor_args instead.
425 If True (default), DASH manifests and related
426 data will be downloaded and processed by extractor.
427 You can reduce network I/O by disabling it if you don't
428 care about DASH. (only for youtube)
429 youtube_include_hls_manifest: Deprecated - Use extractor_args instead.
430 If True (default), HLS manifests and related
431 data will be downloaded and processed by extractor.
432 You can reduce network I/O by disabling it if you don't
433 care about HLS. (only for youtube)
436 _NUMERIC_FIELDS
= set((
437 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
438 'timestamp', 'upload_year', 'upload_month', 'upload_day',
439 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
440 'average_rating', 'comment_count', 'age_limit',
441 'start_time', 'end_time',
442 'chapter_number', 'season_number', 'episode_number',
443 'track_number', 'disc_number', 'release_year',
449 _pps
= {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
450 __prepare_filename_warned
= False
451 _first_webpage_request
= True
452 _download_retcode
= None
453 _num_downloads
= None
455 _playlist_urls
= set()
458 def __init__(self
, params
=None, auto_init
=True):
459 """Create a FileDownloader object with the given options."""
463 self
._ies
_instances
= {}
464 self
._pps
= {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
465 self
.__prepare
_filename
_warned
= False
466 self
._first
_webpage
_request
= True
467 self
._post
_hooks
= []
468 self
._progress
_hooks
= []
469 self
._download
_retcode
= 0
470 self
._num
_downloads
= 0
471 self
._screen
_file
= [sys
.stdout
, sys
.stderr
][params
.get('logtostderr', False)]
472 self
._err
_file
= sys
.stderr
475 'nocheckcertificate': False,
477 self
.params
.update(params
)
478 self
.cache
= Cache(self
)
480 if sys
.version_info
< (3, 6):
482 'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys
.version_info
[:2])
484 def check_deprecated(param
, option
, suggestion
):
485 if self
.params
.get(param
) is not None:
486 self
.report_warning('%s is deprecated. Use %s instead' % (option
, suggestion
))
490 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
491 if self
.params
.get('geo_verification_proxy') is None:
492 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
494 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
495 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
496 check_deprecated('useid', '--id', '-o "%(id)s.%(ext)s"')
498 for msg
in self
.params
.get('warnings', []):
499 self
.report_warning(msg
)
501 if self
.params
.get('final_ext'):
502 if self
.params
.get('merge_output_format'):
503 self
.report_warning('--merge-output-format will be ignored since --remux-video or --recode-video is given')
504 self
.params
['merge_output_format'] = self
.params
['final_ext']
506 if 'overwrites' in self
.params
and self
.params
['overwrites'] is None:
507 del self
.params
['overwrites']
509 if params
.get('bidi_workaround', False):
512 master
, slave
= pty
.openpty()
513 width
= compat_get_terminal_size().columns
517 width_args
= ['-w', str(width
)]
519 stdin
=subprocess
.PIPE
,
521 stderr
=self
._err
_file
)
523 self
._output
_process
= subprocess
.Popen(
524 ['bidiv'] + width_args
, **sp_kwargs
527 self
._output
_process
= subprocess
.Popen(
528 ['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
529 self
._output
_channel
= os
.fdopen(master
, 'rb')
530 except OSError as ose
:
531 if ose
.errno
== errno
.ENOENT
:
532 self
.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
536 if (sys
.platform
!= 'win32'
537 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
538 and not params
.get('restrictfilenames', False)):
539 # Unicode filesystem API will throw errors (#1474, #13027)
541 'Assuming --restrict-filenames since file system encoding '
542 'cannot encode all characters. '
543 'Set the LC_ALL environment variable to fix this.')
544 self
.params
['restrictfilenames'] = True
546 self
.outtmpl_dict
= self
.parse_outtmpl()
548 # Creating format selector here allows us to catch syntax errors before the extraction
549 self
.format_selector
= (
550 None if self
.params
.get('format') is None
551 else self
.build_format_selector(self
.params
['format']))
555 """Preload the archive, if any is specified"""
556 def preload_download_archive(fn
):
559 self
.write_debug('Loading archive file %r\n' % fn
)
561 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
562 for line
in archive_file
:
563 self
.archive
.add(line
.strip())
564 except IOError as ioe
:
565 if ioe
.errno
!= errno
.ENOENT
:
571 preload_download_archive(self
.params
.get('download_archive'))
574 self
.print_debug_header()
575 self
.add_default_info_extractors()
577 for pp_def_raw
in self
.params
.get('postprocessors', []):
578 pp_def
= dict(pp_def_raw
)
579 when
= pp_def
.pop('when', 'post_process')
580 pp_class
= get_postprocessor(pp_def
.pop('key'))
581 pp
= pp_class(self
, **compat_kwargs(pp_def
))
582 self
.add_post_processor(pp
, when
=when
)
584 for ph
in self
.params
.get('post_hooks', []):
585 self
.add_post_hook(ph
)
587 for ph
in self
.params
.get('progress_hooks', []):
588 self
.add_progress_hook(ph
)
590 register_socks_protocols()
592 def warn_if_short_id(self
, argv
):
593 # short YouTube ID starting with dash?
595 i
for i
, a
in enumerate(argv
)
596 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
600 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
601 + ['--'] + [argv
[i
] for i
in idxs
]
604 'Long argument string detected. '
605 'Use -- to separate parameters and URLs, like this:\n%s\n' %
606 args_to_str(correct_argv
))
608 def add_info_extractor(self
, ie
):
609 """Add an InfoExtractor object to the end of the list."""
611 if not isinstance(ie
, type):
612 self
._ies
_instances
[ie
.ie_key()] = ie
613 ie
.set_downloader(self
)
615 def get_info_extractor(self
, ie_key
):
617 Get an instance of an IE with name ie_key, it will try to get one from
618 the _ies list, if there's no instance it will create a new one and add
619 it to the extractor list.
621 ie
= self
._ies
_instances
.get(ie_key
)
623 ie
= get_info_extractor(ie_key
)()
624 self
.add_info_extractor(ie
)
627 def add_default_info_extractors(self
):
629 Add the InfoExtractors returned by gen_extractors to the end of the list
631 for ie
in gen_extractor_classes():
632 self
.add_info_extractor(ie
)
634 def add_post_processor(self
, pp
, when
='post_process'):
635 """Add a PostProcessor object to the end of the chain."""
636 self
._pps
[when
].append(pp
)
637 pp
.set_downloader(self
)
639 def add_post_hook(self
, ph
):
640 """Add the post hook"""
641 self
._post
_hooks
.append(ph
)
643 def add_progress_hook(self
, ph
):
644 """Add the progress hook (currently only for the file downloader)"""
645 self
._progress
_hooks
.append(ph
)
647 def _bidi_workaround(self
, message
):
648 if not hasattr(self
, '_output_channel'):
651 assert hasattr(self
, '_output_process')
652 assert isinstance(message
, compat_str
)
653 line_count
= message
.count('\n') + 1
654 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
655 self
._output
_process
.stdin
.flush()
656 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
657 for _
in range(line_count
))
658 return res
[:-len('\n')]
660 def _write_string(self
, s
, out
=None):
661 write_string(s
, out
=out
, encoding
=self
.params
.get('encoding'))
663 def to_stdout(self
, message
, skip_eol
=False, quiet
=False):
664 """Print message to stdout"""
665 if self
.params
.get('logger'):
666 self
.params
['logger'].debug(message
)
667 elif not quiet
or self
.params
.get('verbose'):
669 '%s%s' % (self
._bidi
_workaround
(message
), ('' if skip_eol
else '\n')),
670 self
._err
_file
if quiet
else self
._screen
_file
)
672 def to_stderr(self
, message
):
673 """Print message to stderr"""
674 assert isinstance(message
, compat_str
)
675 if self
.params
.get('logger'):
676 self
.params
['logger'].error(message
)
678 self
._write
_string
('%s\n' % self
._bidi
_workaround
(message
), self
._err
_file
)
680 def to_console_title(self
, message
):
681 if not self
.params
.get('consoletitle', False):
683 if compat_os_name
== 'nt':
684 if ctypes
.windll
.kernel32
.GetConsoleWindow():
685 # c_wchar_p() might not be necessary if `message` is
686 # already of type unicode()
687 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
688 elif 'TERM' in os
.environ
:
689 self
._write
_string
('\033]0;%s\007' % message
, self
._screen
_file
)
691 def save_console_title(self
):
692 if not self
.params
.get('consoletitle', False):
694 if self
.params
.get('simulate', False):
696 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
697 # Save the title on stack
698 self
._write
_string
('\033[22;0t', self
._screen
_file
)
700 def restore_console_title(self
):
701 if not self
.params
.get('consoletitle', False):
703 if self
.params
.get('simulate', False):
705 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
706 # Restore the title from stack
707 self
._write
_string
('\033[23;0t', self
._screen
_file
)
710 self
.save_console_title()
713 def __exit__(self
, *args
):
714 self
.restore_console_title()
716 if self
.params
.get('cookiefile') is not None:
717 self
.cookiejar
.save(ignore_discard
=True, ignore_expires
=True)
719 def trouble(self
, message
=None, tb
=None):
720 """Determine action to take when a download problem appears.
722 Depending on if the downloader has been configured to ignore
723 download errors or not, this method may throw an exception or
724 not when errors are found, after printing the message.
726 tb, if given, is additional traceback information.
728 if message
is not None:
729 self
.to_stderr(message
)
730 if self
.params
.get('verbose'):
732 if sys
.exc_info()[0]: # if .trouble has been called from an except block
734 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
735 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
736 tb
+= encode_compat_str(traceback
.format_exc())
738 tb_data
= traceback
.format_list(traceback
.extract_stack())
739 tb
= ''.join(tb_data
)
742 if not self
.params
.get('ignoreerrors', False):
743 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
744 exc_info
= sys
.exc_info()[1].exc_info
746 exc_info
= sys
.exc_info()
747 raise DownloadError(message
, exc_info
)
748 self
._download
_retcode
= 1
750 def to_screen(self
, message
, skip_eol
=False):
751 """Print message to stdout if not in quiet mode"""
753 message
, skip_eol
, quiet
=self
.params
.get('quiet', False))
755 def report_warning(self
, message
):
757 Print the message to stderr, it will be prefixed with 'WARNING:'
758 If stderr is a tty file the 'WARNING:' will be colored
760 if self
.params
.get('logger') is not None:
761 self
.params
['logger'].warning(message
)
763 if self
.params
.get('no_warnings'):
765 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
766 _msg_header
= '\033[0;33mWARNING:\033[0m'
768 _msg_header
= 'WARNING:'
769 warning_message
= '%s %s' % (_msg_header
, message
)
770 self
.to_stderr(warning_message
)
772 def report_error(self
, message
, tb
=None):
774 Do the same as trouble, but prefixes the message with 'ERROR:', colored
775 in red if stderr is a tty file.
777 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
778 _msg_header
= '\033[0;31mERROR:\033[0m'
780 _msg_header
= 'ERROR:'
781 error_message
= '%s %s' % (_msg_header
, message
)
782 self
.trouble(error_message
, tb
)
784 def write_debug(self
, message
):
785 '''Log debug message or Print message to stderr'''
786 if not self
.params
.get('verbose', False):
788 message
= '[debug] %s' % message
789 if self
.params
.get('logger'):
790 self
.params
['logger'].debug(message
)
792 self
._write
_string
('%s\n' % message
)
794 def report_file_already_downloaded(self
, file_name
):
795 """Report file has already been fully downloaded."""
797 self
.to_screen('[download] %s has already been downloaded' % file_name
)
798 except UnicodeEncodeError:
799 self
.to_screen('[download] The file has already been downloaded')
801 def report_file_delete(self
, file_name
):
802 """Report that existing file will be deleted."""
804 self
.to_screen('Deleting existing file %s' % file_name
)
805 except UnicodeEncodeError:
806 self
.to_screen('Deleting existing file')
808 def parse_outtmpl(self
):
809 outtmpl_dict
= self
.params
.get('outtmpl', {})
810 if not isinstance(outtmpl_dict
, dict):
811 outtmpl_dict
= {'default': outtmpl_dict}
812 outtmpl_dict
.update({
813 k
: v
for k
, v
in DEFAULT_OUTTMPL
.items()
814 if not outtmpl_dict
.get(k
)})
815 for key
, val
in outtmpl_dict
.items():
816 if isinstance(val
, bytes):
818 'Parameter outtmpl is bytes, but should be a unicode string. '
819 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
822 def get_output_path(self
, dir_type
='', filename
=None):
823 paths
= self
.params
.get('paths', {})
824 assert isinstance(paths
, dict)
826 expand_path(paths
.get('home', '').strip()),
827 expand_path(paths
.get(dir_type
, '').strip()) if dir_type
else '',
830 # Temporary fix for #4787
831 # 'Treat' all problem characters by passing filename through preferredencoding
832 # to workaround encoding issues with subprocess on python2 @ Windows
833 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
834 path
= encodeFilename(path
, True).decode(preferredencoding())
835 return sanitize_path(path
, force
=self
.params
.get('windowsfilenames'))
838 def validate_outtmpl(tmpl
):
839 ''' @return None or Exception object '''
842 STR_FORMAT_RE
.format(''),
843 lambda mobj
: ('%' if not mobj
.group('has_key') else '') + mobj
.group(0),
845 ) % collections
.defaultdict(int)
847 except ValueError as err
:
850 def prepare_outtmpl(self
, outtmpl
, info_dict
, sanitize
=None):
851 """ Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
852 info_dict
= dict(info_dict
)
853 na
= self
.params
.get('outtmpl_na_placeholder', 'NA')
855 info_dict
['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
856 formatSeconds(info_dict
['duration'], '-' if sanitize
else ':')
857 if info_dict
.get('duration', None) is not None
859 info_dict
['epoch'] = int(time
.time())
860 info_dict
['autonumber'] = self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
861 if info_dict
.get('resolution') is None:
862 info_dict
['resolution'] = self
.format_resolution(info_dict
, default
=None)
864 # For fields playlist_index and autonumber convert all occurrences
865 # of %(field)s to %(field)0Nd for backward compatibility
866 field_size_compat_map
= {
867 'playlist_index': len(str(info_dict
.get('_last_playlist_index') or '')),
868 'autonumber': self
.params
.get('autonumber_size') or 5,
872 EXTERNAL_FORMAT_RE
= re
.compile(STR_FORMAT_RE
.format('[^)]*'))
877 # Field is of the form key1.key2...
878 # where keys (except first) can be string, int or slice
879 FIELD_RE
= r
'\w+(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num
=r
'(?:-?\d+)')
880 MATH_FIELD_RE
= r
'''{field}|{num}'''.format(field
=FIELD_RE
, num
=r
'-?\d+(?:.\d+)?')
881 MATH_OPERATORS_RE
= r
'(?:%s)' % '|'.join(map(re
.escape
, MATH_FUNCTIONS
.keys()))
882 INTERNAL_FORMAT_RE
= re
.compile(r
'''(?x)
885 (?P<maths>(?:{math_op}{math_field})*)
886 (?:>(?P<strf_format>.+?))?
887 (?:\|(?P<default>.*?))?
888 $'''.format(field
=FIELD_RE
, math_op
=MATH_OPERATORS_RE
, math_field
=MATH_FIELD_RE
))
890 get_key
= lambda k
: traverse_obj(
891 info_dict
, k
.split('.'), is_user_input
=True, traverse_string
=True)
893 def get_value(mdict
):
895 value
= get_key(mdict
['fields'])
898 value
= float_or_none(value
)
899 if value
is not None:
902 offset_key
= mdict
['maths']
904 value
= float_or_none(value
)
908 MATH_FIELD_RE
if operator
else MATH_OPERATORS_RE
,
910 offset_key
= offset_key
[len(item
):]
912 operator
= MATH_FUNCTIONS
[item
]
914 item
, multiplier
= (item
[1:], -1) if item
[0] == '-' else (item
, 1)
915 offset
= float_or_none(item
)
917 offset
= float_or_none(get_key(item
))
919 value
= operator(value
, multiplier
* offset
)
920 except (TypeError, ZeroDivisionError):
923 # Datetime formatting
924 if mdict
['strf_format']:
925 value
= strftime_or_none(value
, mdict
['strf_format'])
929 def create_key(outer_mobj
):
930 if not outer_mobj
.group('has_key'):
931 return '%{}'.format(outer_mobj
.group(0))
933 key
= outer_mobj
.group('key')
934 fmt
= outer_mobj
.group('format')
935 mobj
= re
.match(INTERNAL_FORMAT_RE
, key
)
937 value
, default
, mobj
= None, na
, {'fields': ''}
939 mobj
= mobj
.groupdict()
940 default
= mobj
['default'] if mobj
['default'] is not None else na
941 value
= get_value(mobj
)
943 if fmt
== 's' and value
is not None and key
in field_size_compat_map
.keys():
944 fmt
= '0{:d}d'.format(field_size_compat_map
[key
])
946 value
= default
if value
is None else value
949 value
= compat_str(value
)
951 value
, fmt
= default
, 's'
954 elif fmt
[-1] not in 'rs': # numeric
955 value
= float_or_none(value
)
957 value
, fmt
= default
, 's'
960 # If value is an object, sanitize might convert it to a string
961 # So we convert it to repr first
962 value
, fmt
= repr(value
), '%ss' % fmt
[:-1]
964 value
= sanitize(mobj
['fields'].split('.')[-1], value
)
966 TMPL_DICT
[key
] = value
967 return '%({key}){fmt}'.format(key
=key
, fmt
=fmt
)
969 return EXTERNAL_FORMAT_RE
.sub(create_key
, outtmpl
), TMPL_DICT
971 def _prepare_filename(self
, info_dict
, tmpl_type
='default'):
973 sanitize
= lambda k
, v
: sanitize_filename(
975 restricted
=self
.params
.get('restrictfilenames'),
976 is_id
=(k
== 'id' or k
.endswith('_id')))
977 outtmpl
= self
.outtmpl_dict
.get(tmpl_type
, self
.outtmpl_dict
['default'])
978 outtmpl
, template_dict
= self
.prepare_outtmpl(outtmpl
, info_dict
, sanitize
)
980 # expand_path translates '%%' into '%' and '$$' into '$'
981 # correspondingly that is not what we want since we need to keep
982 # '%%' intact for template dict substitution step. Working around
983 # with boundary-alike separator hack.
984 sep
= ''.join([random
.choice(ascii_letters
) for _
in range(32)])
985 outtmpl
= outtmpl
.replace('%%', '%{0}%'.format(sep
)).replace('$$', '${0}$'.format(sep
))
987 # outtmpl should be expand_path'ed before template dict substitution
988 # because meta fields may contain env variables we don't want to
989 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
990 # title "Hello $PATH", we don't want `$PATH` to be expanded.
991 filename
= expand_path(outtmpl
).replace(sep
, '') % template_dict
993 force_ext
= OUTTMPL_TYPES
.get(tmpl_type
)
994 if force_ext
is not None:
995 filename
= replace_extension(filename
, force_ext
, info_dict
.get('ext'))
997 # https://github.com/blackjack4494/youtube-dlc/issues/85
998 trim_file_name
= self
.params
.get('trim_file_name', False)
1000 fn_groups
= filename
.rsplit('.')
1003 if len(fn_groups
) > 2:
1004 sub_ext
= fn_groups
[-2]
1005 filename
= '.'.join(filter(None, [fn_groups
[0][:trim_file_name
], sub_ext
, ext
]))
1008 except ValueError as err
:
1009 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
1012 def prepare_filename(self
, info_dict
, dir_type
='', warn
=False):
1013 """Generate the output filename."""
1015 filename
= self
._prepare
_filename
(info_dict
, dir_type
or 'default')
1017 if warn
and not self
.__prepare
_filename
_warned
:
1018 if not self
.params
.get('paths'):
1020 elif filename
== '-':
1021 self
.report_warning('--paths is ignored when an outputting to stdout')
1022 elif os
.path
.isabs(filename
):
1023 self
.report_warning('--paths is ignored since an absolute path is given in output template')
1024 self
.__prepare
_filename
_warned
= True
1025 if filename
== '-' or not filename
:
1028 return self
.get_output_path(dir_type
, filename
)
1030 def _match_entry(self
, info_dict
, incomplete
=False, silent
=False):
1031 """ Returns None if the file should be downloaded """
1033 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
1036 if 'title' in info_dict
:
1037 # This can happen when we're just evaluating the playlist
1038 title
= info_dict
['title']
1039 matchtitle
= self
.params
.get('matchtitle', False)
1041 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
1042 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
1043 rejecttitle
= self
.params
.get('rejecttitle', False)
1045 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
1046 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
1047 date
= info_dict
.get('upload_date')
1048 if date
is not None:
1049 dateRange
= self
.params
.get('daterange', DateRange())
1050 if date
not in dateRange
:
1051 return '%s upload date is not in range %s' % (date_from_str(date
).isoformat(), dateRange
)
1052 view_count
= info_dict
.get('view_count')
1053 if view_count
is not None:
1054 min_views
= self
.params
.get('min_views')
1055 if min_views
is not None and view_count
< min_views
:
1056 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
1057 max_views
= self
.params
.get('max_views')
1058 if max_views
is not None and view_count
> max_views
:
1059 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
1060 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
1061 return 'Skipping "%s" because it is age restricted' % video_title
1064 match_filter
= self
.params
.get('match_filter')
1065 if match_filter
is not None:
1066 ret
= match_filter(info_dict
)
1071 if self
.in_download_archive(info_dict
):
1072 reason
= '%s has already been recorded in the archive' % video_title
1073 break_opt
, break_err
= 'break_on_existing', ExistingVideoReached
1075 reason
= check_filter()
1076 break_opt
, break_err
= 'break_on_reject', RejectedVideoReached
1077 if reason
is not None:
1079 self
.to_screen('[download] ' + reason
)
1080 if self
.params
.get(break_opt
, False):
1085 def add_extra_info(info_dict
, extra_info
):
1086 '''Set the keys from extra_info in info dict if they are missing'''
1087 for key
, value
in extra_info
.items():
1088 info_dict
.setdefault(key
, value
)
1090 def extract_info(self
, url
, download
=True, ie_key
=None, extra_info
={},
1091 process
=True, force_generic_extractor
=False):
1093 Return a list with a dictionary for each video extracted.
1096 url -- URL to extract
1099 download -- whether to download videos during extraction
1100 ie_key -- extractor key hint
1101 extra_info -- dictionary containing the extra values to add to each result
1102 process -- whether to resolve all unresolved references (URLs, playlist items),
1103 must be True for download to work.
1104 force_generic_extractor -- force using the generic extractor
1107 if not ie_key
and force_generic_extractor
:
1111 ies
= [self
.get_info_extractor(ie_key
)]
1116 if not ie
.suitable(url
):
1119 ie_key
= ie
.ie_key()
1120 ie
= self
.get_info_extractor(ie_key
)
1121 if not ie
.working():
1122 self
.report_warning('The program functionality for this site has been marked as broken, '
1123 'and will probably not work.')
1126 temp_id
= str_or_none(
1127 ie
.extract_id(url
) if callable(getattr(ie
, 'extract_id', None))
1128 else ie
._match
_id
(url
))
1129 except (AssertionError, IndexError, AttributeError):
1131 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': ie_key}
):
1132 self
.to_screen("[%s] %s: has already been recorded in archive" % (
1135 return self
.__extract
_info
(url
, ie
, download
, extra_info
, process
)
1137 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
1139 def __handle_extraction_exceptions(func
):
1140 def wrapper(self
, *args
, **kwargs
):
1142 return func(self
, *args
, **kwargs
)
1143 except GeoRestrictedError
as e
:
1146 msg
+= '\nThis video is available in %s.' % ', '.join(
1147 map(ISO3166Utils
.short2full
, e
.countries
))
1148 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
1149 self
.report_error(msg
)
1150 except ExtractorError
as e
: # An error we somewhat expected
1151 self
.report_error(compat_str(e
), e
.format_traceback())
1152 except ThrottledDownload
:
1153 self
.to_stderr('\r')
1154 self
.report_warning('The download speed is below throttle limit. Re-extracting data')
1155 return wrapper(self
, *args
, **kwargs
)
1156 except (MaxDownloadsReached
, ExistingVideoReached
, RejectedVideoReached
):
1158 except Exception as e
:
1159 if self
.params
.get('ignoreerrors', False):
1160 self
.report_error(error_to_compat_str(e
), tb
=encode_compat_str(traceback
.format_exc()))
1165 @__handle_extraction_exceptions
1166 def __extract_info(self
, url
, ie
, download
, extra_info
, process
):
1167 ie_result
= ie
.extract(url
)
1168 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
1170 if isinstance(ie_result
, list):
1171 # Backwards compatibility: old IE result format
1173 '_type': 'compat_list',
1174 'entries': ie_result
,
1176 self
.add_default_extra_info(ie_result
, ie
, url
)
1178 return self
.process_ie_result(ie_result
, download
, extra_info
)
1182 def add_default_extra_info(self
, ie_result
, ie
, url
):
1184 self
.add_extra_info(ie_result
, {
1186 'original_url': url
,
1187 'webpage_url_basename': url_basename(url
),
1190 self
.add_extra_info(ie_result
, {
1191 'extractor': ie
.IE_NAME
,
1192 'extractor_key': ie
.ie_key(),
1195 def process_ie_result(self
, ie_result
, download
=True, extra_info
={}):
1197 Take the result of the ie(may be modified) and resolve all unresolved
1198 references (URLs, playlist items).
1200 It will also download the videos if 'download'.
1201 Returns the resolved ie_result.
1203 result_type
= ie_result
.get('_type', 'video')
1205 if result_type
in ('url', 'url_transparent'):
1206 ie_result
['url'] = sanitize_url(ie_result
['url'])
1207 extract_flat
= self
.params
.get('extract_flat', False)
1208 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
1209 or extract_flat
is True):
1210 info_copy
= ie_result
.copy()
1211 self
.add_extra_info(info_copy
, extra_info
)
1212 ie
= try_get(ie_result
.get('ie_key'), self
.get_info_extractor
)
1213 self
.add_default_extra_info(info_copy
, ie
, ie_result
['url'])
1214 self
.__forced
_printings
(info_copy
, self
.prepare_filename(info_copy
), incomplete
=True)
1217 if result_type
== 'video':
1218 self
.add_extra_info(ie_result
, extra_info
)
1219 ie_result
= self
.process_video_result(ie_result
, download
=download
)
1220 additional_urls
= (ie_result
or {}).get('additional_urls')
1222 # TODO: Improve MetadataFromFieldPP to allow setting a list
1223 if isinstance(additional_urls
, compat_str
):
1224 additional_urls
= [additional_urls
]
1226 '[info] %s: %d additional URL(s) requested' % (ie_result
['id'], len(additional_urls
)))
1227 self
.write_debug('Additional URLs: "%s"' % '", "'.join(additional_urls
))
1228 ie_result
['additional_entries'] = [
1230 url
, download
, extra_info
,
1231 force_generic_extractor
=self
.params
.get('force_generic_extractor'))
1232 for url
in additional_urls
1235 elif result_type
== 'url':
1236 # We have to add extra_info to the results because it may be
1237 # contained in a playlist
1238 return self
.extract_info(
1239 ie_result
['url'], download
,
1240 ie_key
=ie_result
.get('ie_key'),
1241 extra_info
=extra_info
)
1242 elif result_type
== 'url_transparent':
1243 # Use the information from the embedding page
1244 info
= self
.extract_info(
1245 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
1246 extra_info
=extra_info
, download
=False, process
=False)
1248 # extract_info may return None when ignoreerrors is enabled and
1249 # extraction failed with an error, don't crash and return early
1254 force_properties
= dict(
1255 (k
, v
) for k
, v
in ie_result
.items() if v
is not None)
1256 for f
in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
1257 if f
in force_properties
:
1258 del force_properties
[f
]
1259 new_result
= info
.copy()
1260 new_result
.update(force_properties
)
1262 # Extracted info may not be a video result (i.e.
1263 # info.get('_type', 'video') != video) but rather an url or
1264 # url_transparent. In such cases outer metadata (from ie_result)
1265 # should be propagated to inner one (info). For this to happen
1266 # _type of info should be overridden with url_transparent. This
1267 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
1268 if new_result
.get('_type') == 'url':
1269 new_result
['_type'] = 'url_transparent'
1271 return self
.process_ie_result(
1272 new_result
, download
=download
, extra_info
=extra_info
)
1273 elif result_type
in ('playlist', 'multi_video'):
1274 # Protect from infinite recursion due to recursively nested playlists
1275 # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
1276 webpage_url
= ie_result
['webpage_url']
1277 if webpage_url
in self
._playlist
_urls
:
1279 '[download] Skipping already downloaded playlist: %s'
1280 % ie_result
.get('title') or ie_result
.get('id'))
1283 self
._playlist
_level
+= 1
1284 self
._playlist
_urls
.add(webpage_url
)
1285 self
._sanitize
_thumbnails
(ie_result
)
1287 return self
.__process
_playlist
(ie_result
, download
)
1289 self
._playlist
_level
-= 1
1290 if not self
._playlist
_level
:
1291 self
._playlist
_urls
.clear()
1292 elif result_type
== 'compat_list':
1293 self
.report_warning(
1294 'Extractor %s returned a compat_list result. '
1295 'It needs to be updated.' % ie_result
.get('extractor'))
1298 self
.add_extra_info(
1301 'extractor': ie_result
['extractor'],
1302 'webpage_url': ie_result
['webpage_url'],
1303 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1304 'extractor_key': ie_result
['extractor_key'],
1308 ie_result
['entries'] = [
1309 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1310 for r
in ie_result
['entries']
1314 raise Exception('Invalid result type: %s' % result_type
)
1316 def _ensure_dir_exists(self
, path
):
1317 return make_dir(path
, self
.report_error
)
1319 def __process_playlist(self
, ie_result
, download
):
1320 # We process each entry in the playlist
1321 playlist
= ie_result
.get('title') or ie_result
.get('id')
1322 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
1324 if 'entries' not in ie_result
:
1325 raise EntryNotInPlaylist()
1326 incomplete_entries
= bool(ie_result
.get('requested_entries'))
1327 if incomplete_entries
:
1328 def fill_missing_entries(entries
, indexes
):
1329 ret
= [None] * max(*indexes
)
1330 for i
, entry
in zip(indexes
, entries
):
1333 ie_result
['entries'] = fill_missing_entries(ie_result
['entries'], ie_result
['requested_entries'])
1335 playlist_results
= []
1337 playliststart
= self
.params
.get('playliststart', 1)
1338 playlistend
= self
.params
.get('playlistend')
1339 # For backwards compatibility, interpret -1 as whole list
1340 if playlistend
== -1:
1343 playlistitems_str
= self
.params
.get('playlist_items')
1344 playlistitems
= None
1345 if playlistitems_str
is not None:
1346 def iter_playlistitems(format
):
1347 for string_segment
in format
.split(','):
1348 if '-' in string_segment
:
1349 start
, end
= string_segment
.split('-')
1350 for item
in range(int(start
), int(end
) + 1):
1353 yield int(string_segment
)
1354 playlistitems
= orderedSet(iter_playlistitems(playlistitems_str
))
1356 ie_entries
= ie_result
['entries']
1358 'Downloading %d videos' if not isinstance(ie_entries
, list)
1359 else 'Collected %d videos; downloading %%d of them' % len(ie_entries
))
1360 if not isinstance(ie_entries
, (list, PagedList
)):
1361 ie_entries
= LazyList(ie_entries
)
1364 for i
in playlistitems
or itertools
.count(playliststart
):
1365 if playlistitems
is None and playlistend
is not None and playlistend
< i
:
1369 entry
= ie_entries
[i
- 1]
1371 raise EntryNotInPlaylist()
1372 except (IndexError, EntryNotInPlaylist
):
1373 if incomplete_entries
:
1374 raise EntryNotInPlaylist()
1375 elif not playlistitems
:
1377 entries
.append(entry
)
1379 if entry
is not None:
1380 self
._match
_entry
(entry
, incomplete
=True, silent
=True)
1381 except (ExistingVideoReached
, RejectedVideoReached
):
1383 ie_result
['entries'] = entries
1385 # Save playlist_index before re-ordering
1387 ((playlistitems
[i
- 1] if playlistitems
else i
), entry
)
1388 for i
, entry
in enumerate(entries
, 1)
1389 if entry
is not None]
1390 n_entries
= len(entries
)
1392 if not playlistitems
and (playliststart
or playlistend
):
1393 playlistitems
= list(range(playliststart
, playliststart
+ n_entries
))
1394 ie_result
['requested_entries'] = playlistitems
1396 if self
.params
.get('allow_playlist_files', True):
1398 'playlist': playlist
,
1399 'playlist_id': ie_result
.get('id'),
1400 'playlist_title': ie_result
.get('title'),
1401 'playlist_uploader': ie_result
.get('uploader'),
1402 'playlist_uploader_id': ie_result
.get('uploader_id'),
1403 'playlist_index': 0,
1405 ie_copy
.update(dict(ie_result
))
1407 if self
.params
.get('writeinfojson', False):
1408 infofn
= self
.prepare_filename(ie_copy
, 'pl_infojson')
1409 if not self
._ensure
_dir
_exists
(encodeFilename(infofn
)):
1411 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(infofn
)):
1412 self
.to_screen('[info] Playlist metadata is already present')
1414 self
.to_screen('[info] Writing playlist metadata as JSON to: ' + infofn
)
1416 write_json_file(self
.filter_requested_info(ie_result
, self
.params
.get('clean_infojson', True)), infofn
)
1417 except (OSError, IOError):
1418 self
.report_error('Cannot write playlist metadata to JSON file ' + infofn
)
1420 # TODO: This should be passed to ThumbnailsConvertor if necessary
1421 self
._write
_thumbnails
(ie_copy
, self
.prepare_filename(ie_copy
, 'pl_thumbnail'))
1423 if self
.params
.get('writedescription', False):
1424 descfn
= self
.prepare_filename(ie_copy
, 'pl_description')
1425 if not self
._ensure
_dir
_exists
(encodeFilename(descfn
)):
1427 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(descfn
)):
1428 self
.to_screen('[info] Playlist description is already present')
1429 elif ie_result
.get('description') is None:
1430 self
.report_warning('There\'s no playlist description to write.')
1433 self
.to_screen('[info] Writing playlist description to: ' + descfn
)
1434 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
1435 descfile
.write(ie_result
['description'])
1436 except (OSError, IOError):
1437 self
.report_error('Cannot write playlist description file ' + descfn
)
1440 if self
.params
.get('playlistreverse', False):
1441 entries
= entries
[::-1]
1442 if self
.params
.get('playlistrandom', False):
1443 random
.shuffle(entries
)
1445 x_forwarded_for
= ie_result
.get('__x_forwarded_for_ip')
1447 self
.to_screen('[%s] playlist %s: %s' % (ie_result
['extractor'], playlist
, msg
% n_entries
))
1449 max_failures
= self
.params
.get('skip_playlist_after_errors') or float('inf')
1450 for i
, entry_tuple
in enumerate(entries
, 1):
1451 playlist_index
, entry
= entry_tuple
1452 if 'playlist_index' in self
.params
.get('compat_options', []):
1453 playlist_index
= playlistitems
[i
- 1] if playlistitems
else i
1454 self
.to_screen('[download] Downloading video %s of %s' % (i
, n_entries
))
1455 # This __x_forwarded_for_ip thing is a bit ugly but requires
1458 entry
['__x_forwarded_for_ip'] = x_forwarded_for
1460 'n_entries': n_entries
,
1461 '_last_playlist_index': max(playlistitems
) if playlistitems
else (playlistend
or n_entries
),
1462 'playlist_index': playlist_index
,
1463 'playlist_autonumber': i
,
1464 'playlist': playlist
,
1465 'playlist_id': ie_result
.get('id'),
1466 'playlist_title': ie_result
.get('title'),
1467 'playlist_uploader': ie_result
.get('uploader'),
1468 'playlist_uploader_id': ie_result
.get('uploader_id'),
1469 'extractor': ie_result
['extractor'],
1470 'webpage_url': ie_result
['webpage_url'],
1471 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1472 'extractor_key': ie_result
['extractor_key'],
1475 if self
._match
_entry
(entry
, incomplete
=True) is not None:
1478 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, extra
)
1479 if not entry_result
:
1481 if failures
>= max_failures
:
1483 'Skipping the remaining entries in playlist "%s" since %d items failed extraction' % (playlist
, failures
))
1485 # TODO: skip failed (empty) entries?
1486 playlist_results
.append(entry_result
)
1487 ie_result
['entries'] = playlist_results
1488 self
.to_screen('[download] Finished downloading playlist: %s' % playlist
)
1491 @__handle_extraction_exceptions
1492 def __process_iterable_entry(self
, entry
, download
, extra_info
):
1493 return self
.process_ie_result(
1494 entry
, download
=download
, extra_info
=extra_info
)
1496 def _build_format_filter(self
, filter_spec
):
1497 " Returns a function to filter the formats according to the filter_spec "
1507 operator_rex
= re
.compile(r
'''(?x)\s*
1508 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)\s*
1509 (?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1510 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)\s*
1511 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
1512 m
= operator_rex
.fullmatch(filter_spec
)
1515 comparison_value
= int(m
.group('value'))
1517 comparison_value
= parse_filesize(m
.group('value'))
1518 if comparison_value
is None:
1519 comparison_value
= parse_filesize(m
.group('value') + 'B')
1520 if comparison_value
is None:
1522 'Invalid value %r in format specification %r' % (
1523 m
.group('value'), filter_spec
))
1524 op
= OPERATORS
[m
.group('op')]
1529 '^=': lambda attr
, value
: attr
.startswith(value
),
1530 '$=': lambda attr
, value
: attr
.endswith(value
),
1531 '*=': lambda attr
, value
: value
in attr
,
1533 str_operator_rex
= re
.compile(r
'''(?x)\s*
1534 (?P<key>[a-zA-Z0-9._-]+)\s*
1535 (?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1536 (?P<value>[a-zA-Z0-9._-]+)\s*
1537 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
1538 m
= str_operator_rex
.fullmatch(filter_spec
)
1540 comparison_value
= m
.group('value')
1541 str_op
= STR_OPERATORS
[m
.group('op')]
1542 if m
.group('negation'):
1543 op
= lambda attr
, value
: not str_op(attr
, value
)
1548 raise SyntaxError('Invalid filter specification %r' % filter_spec
)
1551 actual_value
= f
.get(m
.group('key'))
1552 if actual_value
is None:
1553 return m
.group('none_inclusive')
1554 return op(actual_value
, comparison_value
)
1557 def _default_format_spec(self
, info_dict
, download
=True):
1560 merger
= FFmpegMergerPP(self
)
1561 return merger
.available
and merger
.can_merge()
1564 not self
.params
.get('simulate', False)
1568 or info_dict
.get('is_live', False)
1569 or self
.outtmpl_dict
['default'] == '-'))
1572 or self
.params
.get('allow_multiple_audio_streams', False)
1573 or 'format-spec' in self
.params
.get('compat_opts', []))
1576 'best/bestvideo+bestaudio' if prefer_best
1577 else 'bestvideo*+bestaudio/best' if not compat
1578 else 'bestvideo+bestaudio/best')
1580 def build_format_selector(self
, format_spec
):
1581 def syntax_error(note
, start
):
1583 'Invalid format specification: '
1584 '{0}\n\t{1}\n\t{2}^'.format(note
, format_spec
, ' ' * start
[1]))
1585 return SyntaxError(message
)
1587 PICKFIRST
= 'PICKFIRST'
1591 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1593 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
1594 'video': self
.params
.get('allow_multiple_video_streams', False)}
1596 check_formats
= self
.params
.get('check_formats')
1598 def _parse_filter(tokens
):
1600 for type, string
, start
, _
, _
in tokens
:
1601 if type == tokenize
.OP
and string
== ']':
1602 return ''.join(filter_parts
)
1604 filter_parts
.append(string
)
1606 def _remove_unused_ops(tokens
):
1607 # Remove operators that we don't use and join them with the surrounding strings
1608 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1609 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
1610 last_string
, last_start
, last_end
, last_line
= None, None, None, None
1611 for type, string
, start
, end
, line
in tokens
:
1612 if type == tokenize
.OP
and string
== '[':
1614 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1616 yield type, string
, start
, end
, line
1617 # everything inside brackets will be handled by _parse_filter
1618 for type, string
, start
, end
, line
in tokens
:
1619 yield type, string
, start
, end
, line
1620 if type == tokenize
.OP
and string
== ']':
1622 elif type == tokenize
.OP
and string
in ALLOWED_OPS
:
1624 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1626 yield type, string
, start
, end
, line
1627 elif type in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
1629 last_string
= string
1633 last_string
+= string
1635 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1637 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
1639 current_selector
= None
1640 for type, string
, start
, _
, _
in tokens
:
1641 # ENCODING is only defined in python 3.x
1642 if type == getattr(tokenize
, 'ENCODING', None):
1644 elif type in [tokenize
.NAME
, tokenize
.NUMBER
]:
1645 current_selector
= FormatSelector(SINGLE
, string
, [])
1646 elif type == tokenize
.OP
:
1648 if not inside_group
:
1649 # ')' will be handled by the parentheses group
1650 tokens
.restore_last_token()
1652 elif inside_merge
and string
in ['/', ',']:
1653 tokens
.restore_last_token()
1655 elif inside_choice
and string
== ',':
1656 tokens
.restore_last_token()
1659 if not current_selector
:
1660 raise syntax_error('"," must follow a format selector', start
)
1661 selectors
.append(current_selector
)
1662 current_selector
= None
1664 if not current_selector
:
1665 raise syntax_error('"/" must follow a format selector', start
)
1666 first_choice
= current_selector
1667 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
1668 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
1670 if not current_selector
:
1671 current_selector
= FormatSelector(SINGLE
, 'best', [])
1672 format_filter
= _parse_filter(tokens
)
1673 current_selector
.filters
.append(format_filter
)
1675 if current_selector
:
1676 raise syntax_error('Unexpected "("', start
)
1677 group
= _parse_format_selection(tokens
, inside_group
=True)
1678 current_selector
= FormatSelector(GROUP
, group
, [])
1680 if not current_selector
:
1681 raise syntax_error('Unexpected "+"', start
)
1682 selector_1
= current_selector
1683 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
1685 raise syntax_error('Expected a selector', start
)
1686 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
1688 raise syntax_error('Operator not recognized: "{0}"'.format(string
), start
)
1689 elif type == tokenize
.ENDMARKER
:
1691 if current_selector
:
1692 selectors
.append(current_selector
)
1695 def _merge(formats_pair
):
1696 format_1
, format_2
= formats_pair
1699 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
1700 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
1702 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
1703 get_no_more
= {'video': False, 'audio': False}
1704 for (i
, fmt_info
) in enumerate(formats_info
):
1705 if fmt_info
.get('acodec') == fmt_info
.get('vcodec') == 'none':
1708 for aud_vid
in ['audio', 'video']:
1709 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
1710 if get_no_more
[aud_vid
]:
1712 get_no_more
[aud_vid
] = True
1714 if len(formats_info
) == 1:
1715 return formats_info
[0]
1717 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
1718 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
1720 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
1721 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
1723 output_ext
= self
.params
.get('merge_output_format')
1726 output_ext
= the_only_video
['ext']
1727 elif the_only_audio
and not video_fmts
:
1728 output_ext
= the_only_audio
['ext']
1733 'requested_formats': formats_info
,
1734 'format': '+'.join(fmt_info
.get('format') for fmt_info
in formats_info
),
1735 'format_id': '+'.join(fmt_info
.get('format_id') for fmt_info
in formats_info
),
1741 'width': the_only_video
.get('width'),
1742 'height': the_only_video
.get('height'),
1743 'resolution': the_only_video
.get('resolution') or self
.format_resolution(the_only_video
),
1744 'fps': the_only_video
.get('fps'),
1745 'vcodec': the_only_video
.get('vcodec'),
1746 'vbr': the_only_video
.get('vbr'),
1747 'stretched_ratio': the_only_video
.get('stretched_ratio'),
1752 'acodec': the_only_audio
.get('acodec'),
1753 'abr': the_only_audio
.get('abr'),
1758 def _check_formats(formats
):
1759 if not check_formats
:
1763 self
.to_screen('[info] Testing format %s' % f
['format_id'])
1764 temp_file
= tempfile
.NamedTemporaryFile(
1765 suffix
='.tmp', delete
=False,
1766 dir=self
.get_output_path('temp') or None)
1769 success
, _
= self
.dl(temp_file
.name
, f
, test
=True)
1770 except (DownloadError
, IOError, OSError, ValueError) + network_exceptions
:
1773 if os
.path
.exists(temp_file
.name
):
1775 os
.remove(temp_file
.name
)
1777 self
.report_warning('Unable to delete temporary file "%s"' % temp_file
.name
)
1781 self
.to_screen('[info] Unable to download format %s. Skipping...' % f
['format_id'])
1783 def _build_selector_function(selector
):
1784 if isinstance(selector
, list): # ,
1785 fs
= [_build_selector_function(s
) for s
in selector
]
1787 def selector_function(ctx
):
1790 return selector_function
1792 elif selector
.type == GROUP
: # ()
1793 selector_function
= _build_selector_function(selector
.selector
)
1795 elif selector
.type == PICKFIRST
: # /
1796 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
1798 def selector_function(ctx
):
1800 picked_formats
= list(f(ctx
))
1802 return picked_formats
1805 elif selector
.type == MERGE
: # +
1806 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
1808 def selector_function(ctx
):
1809 for pair
in itertools
.product(
1810 selector_1(copy
.deepcopy(ctx
)), selector_2(copy
.deepcopy(ctx
))):
1813 elif selector
.type == SINGLE
: # atom
1814 format_spec
= selector
.selector
or 'best'
1816 # TODO: Add allvideo, allaudio etc by generalizing the code with best/worst selector
1817 if format_spec
== 'all':
1818 def selector_function(ctx
):
1819 yield from _check_formats(ctx
['formats'])
1820 elif format_spec
== 'mergeall':
1821 def selector_function(ctx
):
1822 formats
= list(_check_formats(ctx
['formats']))
1825 merged_format
= formats
[-1]
1826 for f
in formats
[-2::-1]:
1827 merged_format
= _merge((merged_format
, f
))
1831 format_fallback
, format_reverse
, format_idx
= False, True, 1
1833 r
'(?P<bw>best|worst|b|w)(?P<type>video|audio|v|a)?(?P<mod>\*)?(?:\.(?P<n>[1-9]\d*))?$',
1835 if mobj
is not None:
1836 format_idx
= int_or_none(mobj
.group('n'), default
=1)
1837 format_reverse
= mobj
.group('bw')[0] == 'b'
1838 format_type
= (mobj
.group('type') or [None])[0]
1839 not_format_type
= {'v': 'a', 'a': 'v'}
.get(format_type
)
1840 format_modified
= mobj
.group('mod') is not None
1842 format_fallback
= not format_type
and not format_modified
# for b, w
1844 (lambda f
: f
.get('%scodec' % format_type
) != 'none')
1845 if format_type
and format_modified
# bv*, ba*, wv*, wa*
1846 else (lambda f
: f
.get('%scodec' % not_format_type
) == 'none')
1847 if format_type
# bv, ba, wv, wa
1848 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
1849 if not format_modified
# b, w
1850 else lambda f
: True) # b*, w*
1851 filter_f
= lambda f
: _filter_f(f
) and (
1852 f
.get('vcodec') != 'none' or f
.get('acodec') != 'none')
1854 filter_f
= ((lambda f
: f
.get('ext') == format_spec
)
1855 if format_spec
in ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav'] # extension
1856 else (lambda f
: f
.get('format_id') == format_spec
)) # id
1858 def selector_function(ctx
):
1859 formats
= list(ctx
['formats'])
1860 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
1861 if format_fallback
and ctx
['incomplete_formats'] and not matches
:
1862 # for extractors with incomplete formats (audio only (soundcloud)
1863 # or video only (imgur)) best/worst will fallback to
1864 # best/worst {video,audio}-only format
1866 matches
= LazyList(_check_formats(matches
[::-1 if format_reverse
else 1]))
1868 yield matches
[format_idx
- 1]
1872 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
1874 def final_selector(ctx
):
1875 ctx_copy
= copy
.deepcopy(ctx
)
1876 for _filter
in filters
:
1877 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
1878 return selector_function(ctx_copy
)
1879 return final_selector
1881 stream
= io
.BytesIO(format_spec
.encode('utf-8'))
1883 tokens
= list(_remove_unused_ops(compat_tokenize_tokenize(stream
.readline
)))
1884 except tokenize
.TokenError
:
1885 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
1887 class TokenIterator(object):
1888 def __init__(self
, tokens
):
1889 self
.tokens
= tokens
1896 if self
.counter
>= len(self
.tokens
):
1897 raise StopIteration()
1898 value
= self
.tokens
[self
.counter
]
1904 def restore_last_token(self
):
1907 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
1908 return _build_selector_function(parsed_selector
)
1910 def _calc_headers(self
, info_dict
):
1911 res
= std_headers
.copy()
1913 add_headers
= info_dict
.get('http_headers')
1915 res
.update(add_headers
)
1917 cookies
= self
._calc
_cookies
(info_dict
)
1919 res
['Cookie'] = cookies
1921 if 'X-Forwarded-For' not in res
:
1922 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
1923 if x_forwarded_for_ip
:
1924 res
['X-Forwarded-For'] = x_forwarded_for_ip
1928 def _calc_cookies(self
, info_dict
):
1929 pr
= sanitized_Request(info_dict
['url'])
1930 self
.cookiejar
.add_cookie_header(pr
)
1931 return pr
.get_header('Cookie')
1933 def _sanitize_thumbnails(self
, info_dict
):
1934 thumbnails
= info_dict
.get('thumbnails')
1935 if thumbnails
is None:
1936 thumbnail
= info_dict
.get('thumbnail')
1938 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
1940 thumbnails
.sort(key
=lambda t
: (
1941 t
.get('preference') if t
.get('preference') is not None else -1,
1942 t
.get('width') if t
.get('width') is not None else -1,
1943 t
.get('height') if t
.get('height') is not None else -1,
1944 t
.get('id') if t
.get('id') is not None else '',
1947 def test_thumbnail(t
):
1948 self
.to_screen('[info] Testing thumbnail %s' % t
['id'])
1950 self
.urlopen(HEADRequest(t
['url']))
1951 except network_exceptions
as err
:
1952 self
.to_screen('[info] Unable to connect to thumbnail %s URL "%s" - %s. Skipping...' % (
1953 t
['id'], t
['url'], error_to_compat_str(err
)))
1957 for i
, t
in enumerate(thumbnails
):
1958 if t
.get('id') is None:
1960 if t
.get('width') and t
.get('height'):
1961 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
1962 t
['url'] = sanitize_url(t
['url'])
1963 if self
.params
.get('check_formats'):
1964 info_dict
['thumbnails'] = LazyList(filter(test_thumbnail
, thumbnails
[::-1])).reverse()
1966 def process_video_result(self
, info_dict
, download
=True):
1967 assert info_dict
.get('_type', 'video') == 'video'
1969 if 'id' not in info_dict
:
1970 raise ExtractorError('Missing "id" field in extractor result')
1971 if 'title' not in info_dict
:
1972 raise ExtractorError('Missing "title" field in extractor result')
1974 def report_force_conversion(field
, field_not
, conversion
):
1975 self
.report_warning(
1976 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1977 % (field
, field_not
, conversion
))
1979 def sanitize_string_field(info
, string_field
):
1980 field
= info
.get(string_field
)
1981 if field
is None or isinstance(field
, compat_str
):
1983 report_force_conversion(string_field
, 'a string', 'string')
1984 info
[string_field
] = compat_str(field
)
1986 def sanitize_numeric_fields(info
):
1987 for numeric_field
in self
._NUMERIC
_FIELDS
:
1988 field
= info
.get(numeric_field
)
1989 if field
is None or isinstance(field
, compat_numeric_types
):
1991 report_force_conversion(numeric_field
, 'numeric', 'int')
1992 info
[numeric_field
] = int_or_none(field
)
1994 sanitize_string_field(info_dict
, 'id')
1995 sanitize_numeric_fields(info_dict
)
1997 if 'playlist' not in info_dict
:
1998 # It isn't part of a playlist
1999 info_dict
['playlist'] = None
2000 info_dict
['playlist_index'] = None
2002 self
._sanitize
_thumbnails
(info_dict
)
2004 thumbnail
= info_dict
.get('thumbnail')
2005 thumbnails
= info_dict
.get('thumbnails')
2007 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
2009 info_dict
['thumbnail'] = thumbnails
[-1]['url']
2011 if 'display_id' not in info_dict
and 'id' in info_dict
:
2012 info_dict
['display_id'] = info_dict
['id']
2014 for ts_key
, date_key
in (
2015 ('timestamp', 'upload_date'),
2016 ('release_timestamp', 'release_date'),
2018 if info_dict
.get(date_key
) is None and info_dict
.get(ts_key
) is not None:
2019 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
2020 # see http://bugs.python.org/issue1646728)
2022 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
[ts_key
])
2023 info_dict
[date_key
] = upload_date
.strftime('%Y%m%d')
2024 except (ValueError, OverflowError, OSError):
2027 # Auto generate title fields corresponding to the *_number fields when missing
2028 # in order to always have clean titles. This is very common for TV series.
2029 for field
in ('chapter', 'season', 'episode'):
2030 if info_dict
.get('%s_number' % field
) is not None and not info_dict
.get(field
):
2031 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
['%s_number' % field
])
2033 for cc_kind
in ('subtitles', 'automatic_captions'):
2034 cc
= info_dict
.get(cc_kind
)
2036 for _
, subtitle
in cc
.items():
2037 for subtitle_format
in subtitle
:
2038 if subtitle_format
.get('url'):
2039 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
2040 if subtitle_format
.get('ext') is None:
2041 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
2043 automatic_captions
= info_dict
.get('automatic_captions')
2044 subtitles
= info_dict
.get('subtitles')
2046 info_dict
['requested_subtitles'] = self
.process_subtitles(
2047 info_dict
['id'], subtitles
, automatic_captions
)
2049 # We now pick which formats have to be downloaded
2050 if info_dict
.get('formats') is None:
2051 # There's only one format available
2052 formats
= [info_dict
]
2054 formats
= info_dict
['formats']
2057 if not self
.params
.get('ignore_no_formats_error'):
2058 raise ExtractorError('No video formats found!')
2060 self
.report_warning('No video formats found!')
2062 def is_wellformed(f
):
2065 self
.report_warning(
2066 '"url" field is missing or empty - skipping format, '
2067 'there is an error in extractor')
2069 if isinstance(url
, bytes):
2070 sanitize_string_field(f
, 'url')
2073 # Filter out malformed formats for better extraction robustness
2074 formats
= list(filter(is_wellformed
, formats
))
2078 # We check that all the formats have the format and format_id fields
2079 for i
, format
in enumerate(formats
):
2080 sanitize_string_field(format
, 'format_id')
2081 sanitize_numeric_fields(format
)
2082 format
['url'] = sanitize_url(format
['url'])
2083 if not format
.get('format_id'):
2084 format
['format_id'] = compat_str(i
)
2086 # Sanitize format_id from characters used in format selector expression
2087 format
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', format
['format_id'])
2088 format_id
= format
['format_id']
2089 if format_id
not in formats_dict
:
2090 formats_dict
[format_id
] = []
2091 formats_dict
[format_id
].append(format
)
2093 # Make sure all formats have unique format_id
2094 for format_id
, ambiguous_formats
in formats_dict
.items():
2095 if len(ambiguous_formats
) > 1:
2096 for i
, format
in enumerate(ambiguous_formats
):
2097 format
['format_id'] = '%s-%d' % (format_id
, i
)
2099 for i
, format
in enumerate(formats
):
2100 if format
.get('format') is None:
2101 format
['format'] = '{id} - {res}{note}'.format(
2102 id=format
['format_id'],
2103 res
=self
.format_resolution(format
),
2104 note
=' ({0})'.format(format
['format_note']) if format
.get('format_note') is not None else '',
2106 # Automatically determine file extension if missing
2107 if format
.get('ext') is None:
2108 format
['ext'] = determine_ext(format
['url']).lower()
2109 # Automatically determine protocol if missing (useful for format
2110 # selection purposes)
2111 if format
.get('protocol') is None:
2112 format
['protocol'] = determine_protocol(format
)
2113 # Add HTTP headers, so that external programs can use them from the
2115 full_format_info
= info_dict
.copy()
2116 full_format_info
.update(format
)
2117 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
2118 # Remove private housekeeping stuff
2119 if '__x_forwarded_for_ip' in info_dict
:
2120 del info_dict
['__x_forwarded_for_ip']
2122 # TODO Central sorting goes here
2124 if formats
and formats
[0] is not info_dict
:
2125 # only set the 'formats' fields if the original info_dict list them
2126 # otherwise we end up with a circular reference, the first (and unique)
2127 # element in the 'formats' field in info_dict is info_dict itself,
2128 # which can't be exported to json
2129 info_dict
['formats'] = formats
2131 info_dict
, _
= self
.pre_process(info_dict
)
2133 list_only
= self
.params
.get('list_thumbnails') or self
.params
.get('listformats') or self
.params
.get('listsubtitles')
2135 self
.__forced
_printings
(info_dict
, self
.prepare_filename(info_dict
), incomplete
=True)
2136 if self
.params
.get('list_thumbnails'):
2137 self
.list_thumbnails(info_dict
)
2138 if self
.params
.get('listformats'):
2139 if not info_dict
.get('formats'):
2140 raise ExtractorError('No video formats found', expected
=True)
2141 self
.list_formats(info_dict
)
2142 if self
.params
.get('listsubtitles'):
2143 if 'automatic_captions' in info_dict
:
2144 self
.list_subtitles(
2145 info_dict
['id'], automatic_captions
, 'automatic captions')
2146 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
2149 format_selector
= self
.format_selector
2150 if format_selector
is None:
2151 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
2152 self
.write_debug('Default format spec: %s' % req_format
)
2153 format_selector
= self
.build_format_selector(req_format
)
2155 # While in format selection we may need to have an access to the original
2156 # format set in order to calculate some metrics or do some processing.
2157 # For now we need to be able to guess whether original formats provided
2158 # by extractor are incomplete or not (i.e. whether extractor provides only
2159 # video-only or audio-only formats) for proper formats selection for
2160 # extractors with such incomplete formats (see
2161 # https://github.com/ytdl-org/youtube-dl/pull/5556).
2162 # Since formats may be filtered during format selection and may not match
2163 # the original formats the results may be incorrect. Thus original formats
2164 # or pre-calculated metrics should be passed to format selection routines
2166 # We will pass a context object containing all necessary additional data
2167 # instead of just formats.
2168 # This fixes incorrect format selection issue (see
2169 # https://github.com/ytdl-org/youtube-dl/issues/10083).
2170 incomplete_formats
= (
2171 # All formats are video-only or
2172 all(f
.get('vcodec') != 'none' and f
.get('acodec') == 'none' for f
in formats
)
2173 # all formats are audio-only
2174 or all(f
.get('vcodec') == 'none' and f
.get('acodec') != 'none' for f
in formats
))
2178 'incomplete_formats': incomplete_formats
,
2181 formats_to_download
= list(format_selector(ctx
))
2182 if not formats_to_download
:
2183 if not self
.params
.get('ignore_no_formats_error'):
2184 raise ExtractorError('Requested format is not available', expected
=True)
2186 self
.report_warning('Requested format is not available')
2187 # Process what we can, even without any available formats.
2188 self
.process_info(dict(info_dict
))
2191 '[info] %s: Downloading %d format(s): %s' % (
2192 info_dict
['id'], len(formats_to_download
),
2193 ", ".join([f
['format_id'] for f
in formats_to_download
])))
2194 for fmt
in formats_to_download
:
2195 new_info
= dict(info_dict
)
2196 # Save a reference to the original info_dict so that it can be modified in process_info if needed
2197 new_info
['__original_infodict'] = info_dict
2198 new_info
.update(fmt
)
2199 self
.process_info(new_info
)
2200 # We update the info dict with the best quality format (backwards compatibility)
2201 if formats_to_download
:
2202 info_dict
.update(formats_to_download
[-1])
2205 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
2206 """Select the requested subtitles and their format"""
2208 if normal_subtitles
and self
.params
.get('writesubtitles'):
2209 available_subs
.update(normal_subtitles
)
2210 if automatic_captions
and self
.params
.get('writeautomaticsub'):
2211 for lang
, cap_info
in automatic_captions
.items():
2212 if lang
not in available_subs
:
2213 available_subs
[lang
] = cap_info
2215 if (not self
.params
.get('writesubtitles') and not
2216 self
.params
.get('writeautomaticsub') or not
2220 all_sub_langs
= available_subs
.keys()
2221 if self
.params
.get('allsubtitles', False):
2222 requested_langs
= all_sub_langs
2223 elif self
.params
.get('subtitleslangs', False):
2224 requested_langs
= set()
2225 for lang
in self
.params
.get('subtitleslangs'):
2227 requested_langs
.update(all_sub_langs
)
2229 discard
= lang
[0] == '-'
2232 current_langs
= filter(re
.compile(lang
+ '$').match
, all_sub_langs
)
2234 for lang
in current_langs
:
2235 requested_langs
.discard(lang
)
2237 requested_langs
.update(current_langs
)
2238 elif 'en' in available_subs
:
2239 requested_langs
= ['en']
2241 requested_langs
= [list(all_sub_langs
)[0]]
2242 self
.write_debug('Downloading subtitles: %s' % ', '.join(requested_langs
))
2244 formats_query
= self
.params
.get('subtitlesformat', 'best')
2245 formats_preference
= formats_query
.split('/') if formats_query
else []
2247 for lang
in requested_langs
:
2248 formats
= available_subs
.get(lang
)
2250 self
.report_warning('%s subtitles not available for %s' % (lang
, video_id
))
2252 for ext
in formats_preference
:
2256 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
2262 self
.report_warning(
2263 'No subtitle format found matching "%s" for language %s, '
2264 'using %s' % (formats_query
, lang
, f
['ext']))
2268 def __forced_printings(self
, info_dict
, filename
, incomplete
):
2269 def print_mandatory(field
, actual_field
=None):
2270 if actual_field
is None:
2271 actual_field
= field
2272 if (self
.params
.get('force%s' % field
, False)
2273 and (not incomplete
or info_dict
.get(actual_field
) is not None)):
2274 self
.to_stdout(info_dict
[actual_field
])
2276 def print_optional(field
):
2277 if (self
.params
.get('force%s' % field
, False)
2278 and info_dict
.get(field
) is not None):
2279 self
.to_stdout(info_dict
[field
])
2281 info_dict
= info_dict
.copy()
2282 if filename
is not None:
2283 info_dict
['filename'] = filename
2284 if info_dict
.get('requested_formats') is not None:
2285 # For RTMP URLs, also include the playpath
2286 info_dict
['urls'] = '\n'.join(f
['url'] + f
.get('play_path', '') for f
in info_dict
['requested_formats'])
2287 elif 'url' in info_dict
:
2288 info_dict
['urls'] = info_dict
['url'] + info_dict
.get('play_path', '')
2290 for tmpl
in self
.params
.get('forceprint', []):
2291 if re
.match(r
'\w+$', tmpl
):
2292 tmpl
= '%({})s'.format(tmpl
)
2293 tmpl
, info_copy
= self
.prepare_outtmpl(tmpl
, info_dict
)
2294 self
.to_stdout(tmpl
% info_copy
)
2296 print_mandatory('title')
2297 print_mandatory('id')
2298 print_mandatory('url', 'urls')
2299 print_optional('thumbnail')
2300 print_optional('description')
2301 print_optional('filename')
2302 if self
.params
.get('forceduration', False) and info_dict
.get('duration') is not None:
2303 self
.to_stdout(formatSeconds(info_dict
['duration']))
2304 print_mandatory('format')
2306 if self
.params
.get('forcejson', False):
2307 self
.post_extract(info_dict
)
2308 self
.to_stdout(json
.dumps(info_dict
, default
=repr))
2310 def dl(self
, name
, info
, subtitle
=False, test
=False):
2313 verbose
= self
.params
.get('verbose')
2316 'quiet': not verbose
,
2318 'noprogress': not verbose
,
2320 'skip_unavailable_fragments': False,
2321 'keep_fragments': False,
2323 '_no_ytdl_file': True,
2326 params
= self
.params
2327 fd
= get_suitable_downloader(info
, params
)(self
, params
)
2329 for ph
in self
._progress
_hooks
:
2330 fd
.add_progress_hook(ph
)
2331 urls
= '", "'.join([f
['url'] for f
in info
.get('requested_formats', [])] or [info
['url']])
2332 self
.write_debug('Invoking downloader on "%s"' % urls
)
2333 new_info
= dict(info
)
2334 if new_info
.get('http_headers') is None:
2335 new_info
['http_headers'] = self
._calc
_headers
(new_info
)
2336 return fd
.download(name
, new_info
, subtitle
)
2338 def process_info(self
, info_dict
):
2339 """Process a single resolved IE result."""
2341 assert info_dict
.get('_type', 'video') == 'video'
2343 info_dict
.setdefault('__postprocessors', [])
2345 max_downloads
= self
.params
.get('max_downloads')
2346 if max_downloads
is not None:
2347 if self
._num
_downloads
>= int(max_downloads
):
2348 raise MaxDownloadsReached()
2350 # TODO: backward compatibility, to be removed
2351 info_dict
['fulltitle'] = info_dict
['title']
2353 if 'format' not in info_dict
and 'ext' in info_dict
:
2354 info_dict
['format'] = info_dict
['ext']
2356 if self
._match
_entry
(info_dict
) is not None:
2359 self
.post_extract(info_dict
)
2360 self
._num
_downloads
+= 1
2362 # info_dict['_filename'] needs to be set for backward compatibility
2363 info_dict
['_filename'] = full_filename
= self
.prepare_filename(info_dict
, warn
=True)
2364 temp_filename
= self
.prepare_filename(info_dict
, 'temp')
2368 self
.__forced
_printings
(info_dict
, full_filename
, incomplete
=('format' not in info_dict
))
2370 if self
.params
.get('simulate', False):
2371 if self
.params
.get('force_write_download_archive', False):
2372 self
.record_download_archive(info_dict
)
2374 # Do nothing else if in simulate mode
2377 if full_filename
is None:
2380 if not self
._ensure
_dir
_exists
(encodeFilename(full_filename
)):
2382 if not self
._ensure
_dir
_exists
(encodeFilename(temp_filename
)):
2385 if self
.params
.get('writedescription', False):
2386 descfn
= self
.prepare_filename(info_dict
, 'description')
2387 if not self
._ensure
_dir
_exists
(encodeFilename(descfn
)):
2389 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(descfn
)):
2390 self
.to_screen('[info] Video description is already present')
2391 elif info_dict
.get('description') is None:
2392 self
.report_warning('There\'s no description to write.')
2395 self
.to_screen('[info] Writing video description to: ' + descfn
)
2396 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
2397 descfile
.write(info_dict
['description'])
2398 except (OSError, IOError):
2399 self
.report_error('Cannot write description file ' + descfn
)
2402 if self
.params
.get('writeannotations', False):
2403 annofn
= self
.prepare_filename(info_dict
, 'annotation')
2404 if not self
._ensure
_dir
_exists
(encodeFilename(annofn
)):
2406 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(annofn
)):
2407 self
.to_screen('[info] Video annotations are already present')
2408 elif not info_dict
.get('annotations'):
2409 self
.report_warning('There are no annotations to write.')
2412 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
2413 with io
.open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
2414 annofile
.write(info_dict
['annotations'])
2415 except (KeyError, TypeError):
2416 self
.report_warning('There are no annotations to write.')
2417 except (OSError, IOError):
2418 self
.report_error('Cannot write annotations file: ' + annofn
)
2421 subtitles_are_requested
= any([self
.params
.get('writesubtitles', False),
2422 self
.params
.get('writeautomaticsub')])
2424 if subtitles_are_requested
and info_dict
.get('requested_subtitles'):
2425 # subtitles download errors are already managed as troubles in relevant IE
2426 # that way it will silently go on when used with unsupporting IE
2427 subtitles
= info_dict
['requested_subtitles']
2428 # ie = self.get_info_extractor(info_dict['extractor_key'])
2429 for sub_lang
, sub_info
in subtitles
.items():
2430 sub_format
= sub_info
['ext']
2431 sub_filename
= subtitles_filename(temp_filename
, sub_lang
, sub_format
, info_dict
.get('ext'))
2432 sub_filename_final
= subtitles_filename(
2433 self
.prepare_filename(info_dict
, 'subtitle'), sub_lang
, sub_format
, info_dict
.get('ext'))
2434 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(sub_filename
)):
2435 self
.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang
, sub_format
))
2436 sub_info
['filepath'] = sub_filename
2437 files_to_move
[sub_filename
] = sub_filename_final
2439 self
.to_screen('[info] Writing video subtitles to: ' + sub_filename
)
2440 if sub_info
.get('data') is not None:
2442 # Use newline='' to prevent conversion of newline characters
2443 # See https://github.com/ytdl-org/youtube-dl/issues/10268
2444 with io
.open(encodeFilename(sub_filename
), 'w', encoding
='utf-8', newline
='') as subfile
:
2445 subfile
.write(sub_info
['data'])
2446 sub_info
['filepath'] = sub_filename
2447 files_to_move
[sub_filename
] = sub_filename_final
2448 except (OSError, IOError):
2449 self
.report_error('Cannot write subtitles file ' + sub_filename
)
2453 self
.dl(sub_filename
, sub_info
.copy(), subtitle
=True)
2454 sub_info
['filepath'] = sub_filename
2455 files_to_move
[sub_filename
] = sub_filename_final
2456 except (ExtractorError
, IOError, OSError, ValueError) + network_exceptions
as err
:
2457 self
.report_warning('Unable to download subtitle for "%s": %s' %
2458 (sub_lang
, error_to_compat_str(err
)))
2461 if self
.params
.get('writeinfojson', False):
2462 infofn
= self
.prepare_filename(info_dict
, 'infojson')
2463 if not self
._ensure
_dir
_exists
(encodeFilename(infofn
)):
2465 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(infofn
)):
2466 self
.to_screen('[info] Video metadata is already present')
2468 self
.to_screen('[info] Writing video metadata as JSON to: ' + infofn
)
2470 write_json_file(self
.filter_requested_info(info_dict
, self
.params
.get('clean_infojson', True)), infofn
)
2471 except (OSError, IOError):
2472 self
.report_error('Cannot write video metadata to JSON file ' + infofn
)
2474 info_dict
['__infojson_filename'] = infofn
2476 for thumb_ext
in self
._write
_thumbnails
(info_dict
, temp_filename
):
2477 thumb_filename_temp
= replace_extension(temp_filename
, thumb_ext
, info_dict
.get('ext'))
2478 thumb_filename
= replace_extension(
2479 self
.prepare_filename(info_dict
, 'thumbnail'), thumb_ext
, info_dict
.get('ext'))
2480 files_to_move
[thumb_filename_temp
] = thumb_filename
2482 # Write internet shortcut files
2483 url_link
= webloc_link
= desktop_link
= False
2484 if self
.params
.get('writelink', False):
2485 if sys
.platform
== "darwin": # macOS.
2487 elif sys
.platform
.startswith("linux"):
2489 else: # if sys.platform in ['win32', 'cygwin']:
2491 if self
.params
.get('writeurllink', False):
2493 if self
.params
.get('writewebloclink', False):
2495 if self
.params
.get('writedesktoplink', False):
2498 if url_link
or webloc_link
or desktop_link
:
2499 if 'webpage_url' not in info_dict
:
2500 self
.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information')
2502 ascii_url
= iri_to_uri(info_dict
['webpage_url'])
2504 def _write_link_file(extension
, template
, newline
, embed_filename
):
2505 linkfn
= replace_extension(full_filename
, extension
, info_dict
.get('ext'))
2506 if self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(linkfn
)):
2507 self
.to_screen('[info] Internet shortcut is already present')
2510 self
.to_screen('[info] Writing internet shortcut to: ' + linkfn
)
2511 with io
.open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8', newline
=newline
) as linkfile
:
2512 template_vars
= {'url': ascii_url}
2514 template_vars
['filename'] = linkfn
[:-(len(extension
) + 1)]
2515 linkfile
.write(template
% template_vars
)
2516 except (OSError, IOError):
2517 self
.report_error('Cannot write internet shortcut ' + linkfn
)
2522 if not _write_link_file('url', DOT_URL_LINK_TEMPLATE
, '\r\n', embed_filename
=False):
2525 if not _write_link_file('webloc', DOT_WEBLOC_LINK_TEMPLATE
, '\n', embed_filename
=False):
2528 if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE
, '\n', embed_filename
=True):
2532 info_dict
, files_to_move
= self
.pre_process(info_dict
, 'before_dl', files_to_move
)
2533 except PostProcessingError
as err
:
2534 self
.report_error('Preprocessing: %s' % str(err
))
2537 must_record_download_archive
= False
2538 if self
.params
.get('skip_download', False):
2539 info_dict
['filepath'] = temp_filename
2540 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2541 info_dict
['__files_to_move'] = files_to_move
2542 info_dict
= self
.run_pp(MoveFilesAfterDownloadPP(self
, False), info_dict
)
2547 def existing_file(*filepaths
):
2548 ext
= info_dict
.get('ext')
2549 final_ext
= self
.params
.get('final_ext', ext
)
2551 for file in orderedSet(filepaths
):
2552 if final_ext
!= ext
:
2553 converted
= replace_extension(file, final_ext
, ext
)
2554 if os
.path
.exists(encodeFilename(converted
)):
2555 existing_files
.append(converted
)
2556 if os
.path
.exists(encodeFilename(file)):
2557 existing_files
.append(file)
2559 if not existing_files
or self
.params
.get('overwrites', False):
2560 for file in orderedSet(existing_files
):
2561 self
.report_file_delete(file)
2562 os
.remove(encodeFilename(file))
2565 self
.report_file_already_downloaded(existing_files
[0])
2566 info_dict
['ext'] = os
.path
.splitext(existing_files
[0])[1][1:]
2567 return existing_files
[0]
2570 if info_dict
.get('requested_formats') is not None:
2572 def compatible_formats(formats
):
2573 # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
2574 video_formats
= [format
for format
in formats
if format
.get('vcodec') != 'none']
2575 audio_formats
= [format
for format
in formats
if format
.get('acodec') != 'none']
2576 if len(video_formats
) > 2 or len(audio_formats
) > 2:
2580 exts
= set(format
.get('ext') for format
in formats
)
2582 set(('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma')),
2585 for ext_sets
in COMPATIBLE_EXTS
:
2586 if ext_sets
.issuperset(exts
):
2588 # TODO: Check acodec/vcodec
2591 requested_formats
= info_dict
['requested_formats']
2592 old_ext
= info_dict
['ext']
2593 if self
.params
.get('merge_output_format') is None:
2594 if not compatible_formats(requested_formats
):
2595 info_dict
['ext'] = 'mkv'
2596 self
.report_warning(
2597 'Requested formats are incompatible for merge and will be merged into mkv.')
2598 if (info_dict
['ext'] == 'webm'
2599 and self
.params
.get('writethumbnail', False)
2600 and info_dict
.get('thumbnails')):
2601 info_dict
['ext'] = 'mkv'
2602 self
.report_warning(
2603 'webm doesn\'t support embedding a thumbnail, mkv will be used.')
2605 def correct_ext(filename
):
2606 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
2608 os
.path
.splitext(filename
)[0]
2609 if filename_real_ext
== old_ext
2611 return '%s.%s' % (filename_wo_ext
, info_dict
['ext'])
2613 # Ensure filename always has a correct extension for successful merge
2614 full_filename
= correct_ext(full_filename
)
2615 temp_filename
= correct_ext(temp_filename
)
2616 dl_filename
= existing_file(full_filename
, temp_filename
)
2617 info_dict
['__real_download'] = False
2619 _protocols
= set(determine_protocol(f
) for f
in requested_formats
)
2620 if len(_protocols
) == 1:
2621 info_dict
['protocol'] = _protocols
.pop()
2622 directly_mergable
= (
2623 'no-direct-merge' not in self
.params
.get('compat_opts', [])
2624 and info_dict
.get('protocol') is not None # All requested formats have same protocol
2625 and not self
.params
.get('allow_unplayable_formats')
2626 and get_suitable_downloader(info_dict
, self
.params
).__name
__ == 'FFmpegFD')
2627 if directly_mergable
:
2628 info_dict
['url'] = requested_formats
[0]['url']
2629 # Treat it as a single download
2630 dl_filename
= existing_file(full_filename
, temp_filename
)
2631 if dl_filename
is None:
2632 success
, real_download
= self
.dl(temp_filename
, info_dict
)
2633 info_dict
['__real_download'] = real_download
2636 merger
= FFmpegMergerPP(self
)
2637 if self
.params
.get('allow_unplayable_formats'):
2638 self
.report_warning(
2639 'You have requested merging of multiple formats '
2640 'while also allowing unplayable formats to be downloaded. '
2641 'The formats won\'t be merged to prevent data corruption.')
2642 elif not merger
.available
:
2643 self
.report_warning(
2644 'You have requested merging of multiple formats but ffmpeg is not installed. '
2645 'The formats won\'t be merged.')
2647 if dl_filename
is None:
2648 for f
in requested_formats
:
2649 new_info
= dict(info_dict
)
2650 del new_info
['requested_formats']
2652 fname
= prepend_extension(
2653 self
.prepare_filename(new_info
, 'temp'),
2654 'f%s' % f
['format_id'], new_info
['ext'])
2655 if not self
._ensure
_dir
_exists
(fname
):
2657 downloaded
.append(fname
)
2658 partial_success
, real_download
= self
.dl(fname
, new_info
)
2659 info_dict
['__real_download'] = info_dict
['__real_download'] or real_download
2660 success
= success
and partial_success
2661 if merger
.available
and not self
.params
.get('allow_unplayable_formats'):
2662 info_dict
['__postprocessors'].append(merger
)
2663 info_dict
['__files_to_merge'] = downloaded
2664 # Even if there were no downloads, it is being merged only now
2665 info_dict
['__real_download'] = True
2667 for file in downloaded
:
2668 files_to_move
[file] = None
2670 # Just a single file
2671 dl_filename
= existing_file(full_filename
, temp_filename
)
2672 if dl_filename
is None:
2673 success
, real_download
= self
.dl(temp_filename
, info_dict
)
2674 info_dict
['__real_download'] = real_download
2676 dl_filename
= dl_filename
or temp_filename
2677 info_dict
['__finaldir'] = os
.path
.dirname(os
.path
.abspath(encodeFilename(full_filename
)))
2679 except network_exceptions
as err
:
2680 self
.report_error('unable to download video data: %s' % error_to_compat_str(err
))
2682 except (OSError, IOError) as err
:
2683 raise UnavailableVideoError(err
)
2684 except (ContentTooShortError
, ) as err
:
2685 self
.report_error('content too short (expected %s bytes and served %s)' % (err
.expected
, err
.downloaded
))
2688 if success
and full_filename
!= '-':
2692 fixup_policy
= self
.params
.get('fixup')
2693 vid
= info_dict
['id']
2695 if fixup_policy
in ('ignore', 'never'):
2697 elif fixup_policy
== 'warn':
2699 elif fixup_policy
!= 'force':
2700 assert fixup_policy
in ('detect_or_warn', None)
2701 if not info_dict
.get('__real_download'):
2704 def ffmpeg_fixup(cndn
, msg
, cls
):
2708 self
.report_warning(f
'{vid}: {msg}')
2712 info_dict
['__postprocessors'].append(pp
)
2714 self
.report_warning(f
'{vid}: {msg}. Install ffmpeg to fix this automatically')
2716 stretched_ratio
= info_dict
.get('stretched_ratio')
2718 stretched_ratio
not in (1, None),
2719 f
'Non-uniform pixel ratio {stretched_ratio}',
2720 FFmpegFixupStretchedPP
)
2723 (info_dict
.get('requested_formats') is None
2724 and info_dict
.get('container') == 'm4a_dash'
2725 and info_dict
.get('ext') == 'm4a'),
2726 'writing DASH m4a. Only some players support this container',
2729 downloader
= (get_suitable_downloader(info_dict
, self
.params
).__name
__
2730 if 'protocol' in info_dict
else None)
2731 ffmpeg_fixup(downloader
== 'HlsFD', 'malformed AAC bitstream detected', FFmpegFixupM3u8PP
)
2732 ffmpeg_fixup(downloader
== 'WebSocketFragmentFD', 'malformed timestamps detected', FFmpegFixupTimestampPP
)
2733 ffmpeg_fixup(downloader
== 'WebSocketFragmentFD', 'malformed duration detected', FFmpegFixupDurationPP
)
2737 info_dict
= self
.post_process(dl_filename
, info_dict
, files_to_move
)
2738 except PostProcessingError
as err
:
2739 self
.report_error('Postprocessing: %s' % str(err
))
2742 for ph
in self
._post
_hooks
:
2743 ph(info_dict
['filepath'])
2744 except Exception as err
:
2745 self
.report_error('post hooks: %s' % str(err
))
2747 must_record_download_archive
= True
2749 if must_record_download_archive
or self
.params
.get('force_write_download_archive', False):
2750 self
.record_download_archive(info_dict
)
2751 max_downloads
= self
.params
.get('max_downloads')
2752 if max_downloads
is not None and self
._num
_downloads
>= int(max_downloads
):
2753 raise MaxDownloadsReached()
2755 def download(self
, url_list
):
2756 """Download a given list of URLs."""
2757 outtmpl
= self
.outtmpl_dict
['default']
2758 if (len(url_list
) > 1
2760 and '%' not in outtmpl
2761 and self
.params
.get('max_downloads') != 1):
2762 raise SameFileError(outtmpl
)
2764 for url
in url_list
:
2766 # It also downloads the videos
2767 res
= self
.extract_info(
2768 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
2769 except UnavailableVideoError
:
2770 self
.report_error('unable to download video')
2771 except MaxDownloadsReached
:
2772 self
.to_screen('[info] Maximum number of downloaded files reached')
2774 except ExistingVideoReached
:
2775 self
.to_screen('[info] Encountered a file that is already in the archive, stopping due to --break-on-existing')
2777 except RejectedVideoReached
:
2778 self
.to_screen('[info] Encountered a file that did not match filter, stopping due to --break-on-reject')
2781 if self
.params
.get('dump_single_json', False):
2782 self
.post_extract(res
)
2783 self
.to_stdout(json
.dumps(res
, default
=repr))
2785 return self
._download
_retcode
2787 def download_with_info_file(self
, info_filename
):
2788 with contextlib
.closing(fileinput
.FileInput(
2789 [info_filename
], mode
='r',
2790 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
2791 # FileInput doesn't have a read method, we can't call json.load
2792 info
= self
.filter_requested_info(json
.loads('\n'.join(f
)), self
.params
.get('clean_infojson', True))
2794 self
.process_ie_result(info
, download
=True)
2795 except (DownloadError
, EntryNotInPlaylist
, ThrottledDownload
):
2796 webpage_url
= info
.get('webpage_url')
2797 if webpage_url
is not None:
2798 self
.report_warning('The info failed to download, trying with "%s"' % webpage_url
)
2799 return self
.download([webpage_url
])
2802 return self
._download
_retcode
2805 def filter_requested_info(info_dict
, actually_filter
=True):
2806 remove_keys
= ['__original_infodict'] # Always remove this since this may contain a copy of the entire dict
2807 keep_keys
= ['_type'], # Always keep this to facilitate load-info-json
2809 remove_keys
+= ('requested_formats', 'requested_subtitles', 'requested_entries', 'filepath', 'entries', 'original_url')
2810 empty_values
= (None, {}, [], set(), tuple())
2811 reject
= lambda k
, v
: k
not in keep_keys
and (
2812 k
.startswith('_') or k
in remove_keys
or v
in empty_values
)
2814 info_dict
['epoch'] = int(time
.time())
2815 reject
= lambda k
, v
: k
in remove_keys
2816 filter_fn
= lambda obj
: (
2817 list(map(filter_fn
, obj
)) if isinstance(obj
, (LazyList
, list, tuple, set))
2818 else obj
if not isinstance(obj
, dict)
2819 else dict((k
, filter_fn(v
)) for k
, v
in obj
.items() if not reject(k
, v
)))
2820 return filter_fn(info_dict
)
2822 def run_pp(self
, pp
, infodict
):
2823 files_to_delete
= []
2824 if '__files_to_move' not in infodict
:
2825 infodict
['__files_to_move'] = {}
2826 files_to_delete
, infodict
= pp
.run(infodict
)
2827 if not files_to_delete
:
2830 if self
.params
.get('keepvideo', False):
2831 for f
in files_to_delete
:
2832 infodict
['__files_to_move'].setdefault(f
, '')
2834 for old_filename
in set(files_to_delete
):
2835 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
2837 os
.remove(encodeFilename(old_filename
))
2838 except (IOError, OSError):
2839 self
.report_warning('Unable to remove downloaded original file')
2840 if old_filename
in infodict
['__files_to_move']:
2841 del infodict
['__files_to_move'][old_filename
]
2845 def post_extract(info_dict
):
2846 def actual_post_extract(info_dict
):
2847 if info_dict
.get('_type') in ('playlist', 'multi_video'):
2848 for video_dict
in info_dict
.get('entries', {}):
2849 actual_post_extract(video_dict
or {})
2852 post_extractor
= info_dict
.get('__post_extractor') or (lambda: {})
2853 extra
= post_extractor().items()
2854 info_dict
.update(extra
)
2855 info_dict
.pop('__post_extractor', None)
2857 original_infodict
= info_dict
.get('__original_infodict') or {}
2858 original_infodict
.update(extra
)
2859 original_infodict
.pop('__post_extractor', None)
2861 actual_post_extract(info_dict
or {})
2863 def pre_process(self
, ie_info
, key
='pre_process', files_to_move
=None):
2864 info
= dict(ie_info
)
2865 info
['__files_to_move'] = files_to_move
or {}
2866 for pp
in self
._pps
[key
]:
2867 info
= self
.run_pp(pp
, info
)
2868 return info
, info
.pop('__files_to_move', None)
2870 def post_process(self
, filename
, ie_info
, files_to_move
=None):
2871 """Run all the postprocessors on the given file."""
2872 info
= dict(ie_info
)
2873 info
['filepath'] = filename
2874 info
['__files_to_move'] = files_to_move
or {}
2876 for pp
in ie_info
.get('__postprocessors', []) + self
._pps
['post_process']:
2877 info
= self
.run_pp(pp
, info
)
2878 info
= self
.run_pp(MoveFilesAfterDownloadPP(self
), info
)
2879 del info
['__files_to_move']
2880 for pp
in self
._pps
['after_move']:
2881 info
= self
.run_pp(pp
, info
)
2884 def _make_archive_id(self
, info_dict
):
2885 video_id
= info_dict
.get('id')
2888 # Future-proof against any change in case
2889 # and backwards compatibility with prior versions
2890 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
2891 if extractor
is None:
2892 url
= str_or_none(info_dict
.get('url'))
2895 # Try to find matching extractor for the URL and take its ie_key
2896 for ie
in self
._ies
:
2897 if ie
.suitable(url
):
2898 extractor
= ie
.ie_key()
2902 return '%s %s' % (extractor
.lower(), video_id
)
2904 def in_download_archive(self
, info_dict
):
2905 fn
= self
.params
.get('download_archive')
2909 vid_id
= self
._make
_archive
_id
(info_dict
)
2911 return False # Incomplete video information
2913 return vid_id
in self
.archive
2915 def record_download_archive(self
, info_dict
):
2916 fn
= self
.params
.get('download_archive')
2919 vid_id
= self
._make
_archive
_id
(info_dict
)
2921 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
2922 archive_file
.write(vid_id
+ '\n')
2923 self
.archive
.add(vid_id
)
2926 def format_resolution(format
, default
='unknown'):
2927 if format
.get('vcodec') == 'none':
2928 if format
.get('acodec') == 'none':
2931 if format
.get('resolution') is not None:
2932 return format
['resolution']
2933 if format
.get('width') and format
.get('height'):
2934 res
= '%dx%d' % (format
['width'], format
['height'])
2935 elif format
.get('height'):
2936 res
= '%sp' % format
['height']
2937 elif format
.get('width'):
2938 res
= '%dx?' % format
['width']
2943 def _format_note(self
, fdict
):
2945 if fdict
.get('ext') in ['f4f', 'f4m']:
2946 res
+= '(unsupported) '
2947 if fdict
.get('language'):
2950 res
+= '[%s] ' % fdict
['language']
2951 if fdict
.get('format_note') is not None:
2952 res
+= fdict
['format_note'] + ' '
2953 if fdict
.get('tbr') is not None:
2954 res
+= '%4dk ' % fdict
['tbr']
2955 if fdict
.get('container') is not None:
2958 res
+= '%s container' % fdict
['container']
2959 if (fdict
.get('vcodec') is not None
2960 and fdict
.get('vcodec') != 'none'):
2963 res
+= fdict
['vcodec']
2964 if fdict
.get('vbr') is not None:
2966 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
2968 if fdict
.get('vbr') is not None:
2969 res
+= '%4dk' % fdict
['vbr']
2970 if fdict
.get('fps') is not None:
2973 res
+= '%sfps' % fdict
['fps']
2974 if fdict
.get('acodec') is not None:
2977 if fdict
['acodec'] == 'none':
2980 res
+= '%-5s' % fdict
['acodec']
2981 elif fdict
.get('abr') is not None:
2985 if fdict
.get('abr') is not None:
2986 res
+= '@%3dk' % fdict
['abr']
2987 if fdict
.get('asr') is not None:
2988 res
+= ' (%5dHz)' % fdict
['asr']
2989 if fdict
.get('filesize') is not None:
2992 res
+= format_bytes(fdict
['filesize'])
2993 elif fdict
.get('filesize_approx') is not None:
2996 res
+= '~' + format_bytes(fdict
['filesize_approx'])
2999 def _format_note_table(self
, f
):
3000 def join_fields(*vargs
):
3001 return ', '.join((val
for val
in vargs
if val
!= ''))
3004 'UNSUPPORTED' if f
.get('ext') in ('f4f', 'f4m') else '',
3005 format_field(f
, 'language', '[%s]'),
3006 format_field(f
, 'format_note'),
3007 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
3008 format_field(f
, 'asr', '%5dHz'))
3010 def list_formats(self
, info_dict
):
3011 formats
= info_dict
.get('formats', [info_dict
])
3013 'list-formats' not in self
.params
.get('compat_opts', [])
3014 and self
.params
.get('listformats_table', True) is not False)
3018 format_field(f
, 'format_id'),
3019 format_field(f
, 'ext'),
3020 self
.format_resolution(f
),
3021 format_field(f
, 'fps', '%d'),
3023 format_field(f
, 'filesize', ' %s', func
=format_bytes
) + format_field(f
, 'filesize_approx', '~%s', func
=format_bytes
),
3024 format_field(f
, 'tbr', '%4dk'),
3025 shorten_protocol_name(f
.get('protocol', '').replace("native", "n")),
3027 format_field(f
, 'vcodec', default
='unknown').replace('none', ''),
3028 format_field(f
, 'vbr', '%4dk'),
3029 format_field(f
, 'acodec', default
='unknown').replace('none', ''),
3030 format_field(f
, 'abr', '%3dk'),
3031 format_field(f
, 'asr', '%5dHz'),
3032 self
._format
_note
_table
(f
)]
3034 if f
.get('preference') is None or f
['preference'] >= -1000]
3035 header_line
= ['ID', 'EXT', 'RESOLUTION', 'FPS', '|', ' FILESIZE', ' TBR', 'PROTO',
3036 '|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'NOTE']
3040 format_field(f
, 'format_id'),
3041 format_field(f
, 'ext'),
3042 self
.format_resolution(f
),
3043 self
._format
_note
(f
)]
3045 if f
.get('preference') is None or f
['preference'] >= -1000]
3046 header_line
= ['format code', 'extension', 'resolution', 'note']
3049 '[info] Available formats for %s:' % info_dict
['id'])
3050 self
.to_stdout(render_table(
3051 header_line
, table
, delim
=new_format
, extraGap
=(0 if new_format
else 1), hideEmpty
=new_format
))
3053 def list_thumbnails(self
, info_dict
):
3054 thumbnails
= list(info_dict
.get('thumbnails'))
3056 self
.to_screen('[info] No thumbnails present for %s' % info_dict
['id'])
3060 '[info] Thumbnails for %s:' % info_dict
['id'])
3061 self
.to_stdout(render_table(
3062 ['ID', 'width', 'height', 'URL'],
3063 [[t
['id'], t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
]))
3065 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
3067 self
.to_screen('%s has no %s' % (video_id
, name
))
3070 'Available %s for %s:' % (name
, video_id
))
3072 def _row(lang
, formats
):
3073 exts
, names
= zip(*((f
['ext'], f
.get('name') or 'unknown') for f
in reversed(formats
)))
3074 if len(set(names
)) == 1:
3075 names
= [] if names
[0] == 'unknown' else names
[:1]
3076 return [lang
, ', '.join(names
), ', '.join(exts
)]
3078 self
.to_stdout(render_table(
3079 ['Language', 'Name', 'Formats'],
3080 [_row(lang
, formats
) for lang
, formats
in subtitles
.items()],
3083 def urlopen(self
, req
):
3084 """ Start an HTTP download """
3085 if isinstance(req
, compat_basestring
):
3086 req
= sanitized_Request(req
)
3087 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
3089 def print_debug_header(self
):
3090 if not self
.params
.get('verbose'):
3093 if type('') is not compat_str
:
3094 # Python 2.6 on SLES11 SP1 (https://github.com/ytdl-org/youtube-dl/issues/3326)
3095 self
.report_warning(
3096 'Your Python is broken! Update to a newer and supported version')
3098 stdout_encoding
= getattr(
3099 sys
.stdout
, 'encoding', 'missing (%s)' % type(sys
.stdout
).__name
__)
3101 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
3102 locale
.getpreferredencoding(),
3103 sys
.getfilesystemencoding(),
3105 self
.get_encoding()))
3106 write_string(encoding_str
, encoding
=None)
3109 '(exe)' if hasattr(sys
, 'frozen')
3110 else '(zip)' if isinstance(globals().get('__loader__'), zipimporter
)
3111 else '(source)' if os
.path
.basename(sys
.argv
[0]) == '__main__.py'
3113 self
._write
_string
('[debug] yt-dlp version %s %s\n' % (__version__
, source
))
3115 self
._write
_string
('[debug] Lazy loading extractors enabled\n')
3118 '[debug] Plugin Extractors: %s\n' % [ie
.ie_key() for ie
in _PLUGIN_CLASSES
])
3119 if self
.params
.get('compat_opts'):
3121 '[debug] Compatibility options: %s\n' % ', '.join(self
.params
.get('compat_opts')))
3123 sp
= subprocess
.Popen(
3124 ['git', 'rev-parse', '--short', 'HEAD'],
3125 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
3126 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
3127 out
, err
= process_communicate_or_kill(sp
)
3128 out
= out
.decode().strip()
3129 if re
.match('[0-9a-f]+', out
):
3130 self
._write
_string
('[debug] Git HEAD: %s\n' % out
)
3137 def python_implementation():
3138 impl_name
= platform
.python_implementation()
3139 if impl_name
== 'PyPy' and hasattr(sys
, 'pypy_version_info'):
3140 return impl_name
+ ' version %d.%d.%d' % sys
.pypy_version_info
[:3]
3143 self
._write
_string
('[debug] Python version %s (%s %s) - %s\n' % (
3144 platform
.python_version(),
3145 python_implementation(),
3146 platform
.architecture()[0],
3149 exe_versions
= FFmpegPostProcessor
.get_versions(self
)
3150 exe_versions
['rtmpdump'] = rtmpdump_version()
3151 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
3152 exe_str
= ', '.join(
3154 for exe
, v
in sorted(exe_versions
.items())
3159 self
._write
_string
('[debug] exe versions: %s\n' % exe_str
)
3162 for handler
in self
._opener
.handlers
:
3163 if hasattr(handler
, 'proxies'):
3164 proxy_map
.update(handler
.proxies
)
3165 self
._write
_string
('[debug] Proxy map: ' + compat_str(proxy_map
) + '\n')
3167 if self
.params
.get('call_home', False):
3168 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
3169 self
._write
_string
('[debug] Public IP address: %s\n' % ipaddr
)
3171 latest_version
= self
.urlopen(
3172 'https://yt-dl.org/latest/version').read().decode('utf-8')
3173 if version_tuple(latest_version
) > version_tuple(__version__
):
3174 self
.report_warning(
3175 'You are using an outdated version (newest version: %s)! '
3176 'See https://yt-dl.org/update if you need help updating.' %
3179 def _setup_opener(self
):
3180 timeout_val
= self
.params
.get('socket_timeout')
3181 self
._socket
_timeout
= 600 if timeout_val
is None else float(timeout_val
)
3183 opts_cookiefile
= self
.params
.get('cookiefile')
3184 opts_proxy
= self
.params
.get('proxy')
3186 if opts_cookiefile
is None:
3187 self
.cookiejar
= compat_cookiejar
.CookieJar()
3189 opts_cookiefile
= expand_path(opts_cookiefile
)
3190 self
.cookiejar
= YoutubeDLCookieJar(opts_cookiefile
)
3191 if os
.access(opts_cookiefile
, os
.R_OK
):
3192 self
.cookiejar
.load(ignore_discard
=True, ignore_expires
=True)
3194 cookie_processor
= YoutubeDLCookieProcessor(self
.cookiejar
)
3195 if opts_proxy
is not None:
3196 if opts_proxy
== '':
3199 proxies
= {'http': opts_proxy, 'https': opts_proxy}
3201 proxies
= compat_urllib_request
.getproxies()
3202 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
3203 if 'http' in proxies
and 'https' not in proxies
:
3204 proxies
['https'] = proxies
['http']
3205 proxy_handler
= PerRequestProxyHandler(proxies
)
3207 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
3208 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
3209 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
3210 redirect_handler
= YoutubeDLRedirectHandler()
3211 data_handler
= compat_urllib_request_DataHandler()
3213 # When passing our own FileHandler instance, build_opener won't add the
3214 # default FileHandler and allows us to disable the file protocol, which
3215 # can be used for malicious purposes (see
3216 # https://github.com/ytdl-org/youtube-dl/issues/8227)
3217 file_handler
= compat_urllib_request
.FileHandler()
3219 def file_open(*args
, **kwargs
):
3220 raise compat_urllib_error
.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
3221 file_handler
.file_open
= file_open
3223 opener
= compat_urllib_request
.build_opener(
3224 proxy_handler
, https_handler
, cookie_processor
, ydlh
, redirect_handler
, data_handler
, file_handler
)
3226 # Delete the default user-agent header, which would otherwise apply in
3227 # cases where our custom HTTP handler doesn't come into play
3228 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
3229 opener
.addheaders
= []
3230 self
._opener
= opener
3232 def encode(self
, s
):
3233 if isinstance(s
, bytes):
3234 return s
# Already encoded
3237 return s
.encode(self
.get_encoding())
3238 except UnicodeEncodeError as err
:
3239 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
3242 def get_encoding(self
):
3243 encoding
= self
.params
.get('encoding')
3244 if encoding
is None:
3245 encoding
= preferredencoding()
3248 def _write_thumbnails(self
, info_dict
, filename
): # return the extensions
3249 write_all
= self
.params
.get('write_all_thumbnails', False)
3251 if write_all
or self
.params
.get('writethumbnail', False):
3252 thumbnails
= info_dict
.get('thumbnails') or []
3253 multiple
= write_all
and len(thumbnails
) > 1
3256 for t
in thumbnails
[::-1]:
3257 thumb_ext
= determine_ext(t
['url'], 'jpg')
3258 suffix
= '%s.' % t
['id'] if multiple
else ''
3259 thumb_display_id
= '%s ' % t
['id'] if multiple
else ''
3260 thumb_filename
= replace_extension(filename
, suffix
+ thumb_ext
, info_dict
.get('ext'))
3262 if not self
.params
.get('overwrites', True) and os
.path
.exists(encodeFilename(thumb_filename
)):
3263 ret
.append(suffix
+ thumb_ext
)
3264 t
['filepath'] = thumb_filename
3265 self
.to_screen('[%s] %s: Thumbnail %sis already present' %
3266 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
3268 self
.to_screen('[%s] %s: Downloading thumbnail %s ...' %
3269 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
3271 uf
= self
.urlopen(t
['url'])
3272 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
3273 shutil
.copyfileobj(uf
, thumbf
)
3274 ret
.append(suffix
+ thumb_ext
)
3275 self
.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
3276 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
, thumb_filename
))
3277 t
['filepath'] = thumb_filename
3278 except network_exceptions
as err
:
3279 self
.report_warning('Unable to download thumbnail "%s": %s' %
3280 (t
['url'], error_to_compat_str(err
)))
3281 if ret
and not write_all
: