4 from __future__
import absolute_import
, unicode_literals
29 from string
import ascii_letters
34 compat_get_terminal_size
,
40 compat_tokenize_tokenize
,
42 compat_urllib_request
,
43 compat_urllib_request_DataHandler
,
54 DOT_DESKTOP_LINK_TEMPLATE
,
55 DOT_URL_LINK_TEMPLATE
,
56 DOT_WEBLOC_LINK_TEMPLATE
,
76 PerRequestProxyHandler
,
81 register_socks_protocols
,
93 UnavailableVideoError
,
99 YoutubeDLCookieProcessor
,
101 YoutubeDLRedirectHandler
,
103 from .cache
import Cache
104 from .extractor
import get_info_extractor
, gen_extractor_classes
, _LAZY_LOADER
105 from .extractor
.openload
import PhantomJSwrapper
106 from .downloader
import get_suitable_downloader
107 from .downloader
.rtmp
import rtmpdump_version
108 from .postprocessor
import (
111 FFmpegFixupStretchedPP
,
114 FFmpegSubtitlesConvertorPP
,
117 from .version
import __version__
119 if compat_os_name
== 'nt':
123 class YoutubeDL(object):
126 YoutubeDL objects are the ones responsible of downloading the
127 actual video file and writing it to disk if the user has requested
128 it, among some other tasks. In most cases there should be one per
129 program. As, given a video URL, the downloader doesn't know how to
130 extract all the needed information, task that InfoExtractors do, it
131 has to pass the URL to one of them.
133 For this, YoutubeDL objects have a method that allows
134 InfoExtractors to be registered in a given order. When it is passed
135 a URL, the YoutubeDL object handles it to the first InfoExtractor it
136 finds that reports being able to handle it. The InfoExtractor extracts
137 all the information about the video or videos the URL refers to, and
138 YoutubeDL process the extracted information, possibly using a File
139 Downloader to download the video.
141 YoutubeDL objects accept a lot of parameters. In order not to saturate
142 the object constructor with arguments, it receives a dictionary of
143 options instead. These options are available through the params
144 attribute for the InfoExtractors to use. The YoutubeDL also
145 registers itself as the downloader in charge for the InfoExtractors
146 that are added to it, so this is a "mutual registration".
150 username: Username for authentication purposes.
151 password: Password for authentication purposes.
152 videopassword: Password for accessing a video.
153 ap_mso: Adobe Pass multiple-system operator identifier.
154 ap_username: Multiple-system operator account username.
155 ap_password: Multiple-system operator account password.
156 usenetrc: Use netrc for authentication instead.
157 verbose: Print additional info to stdout.
158 quiet: Do not print messages to stdout.
159 no_warnings: Do not print out anything for warnings.
160 forceurl: Force printing final URL.
161 forcetitle: Force printing title.
162 forceid: Force printing ID.
163 forcethumbnail: Force printing thumbnail URL.
164 forcedescription: Force printing description.
165 forcefilename: Force printing final filename.
166 forceduration: Force printing duration.
167 forcejson: Force printing info_dict as JSON.
168 dump_single_json: Force printing the info_dict of the whole playlist
169 (or video) as a single JSON line.
170 force_write_download_archive: Force writing download archive regardless of
171 'skip_download' or 'simulate'.
172 simulate: Do not download the video files.
173 format: Video format code. see "FORMAT SELECTION" for more details.
174 format_sort: How to sort the video formats. see "Sorting Formats" for more details.
175 format_sort_force: Force the given format_sort. see "Sorting Formats" for more details.
176 allow_multiple_video_streams: Allow multiple video streams to be merged into a single file
177 allow_multiple_audio_streams: Allow multiple audio streams to be merged into a single file
178 outtmpl: Template for output names.
179 restrictfilenames: Do not allow "&" and spaces in file names.
180 trim_file_name: Limit length of filename (extension excluded).
181 ignoreerrors: Do not stop on download errors. (Default False when running youtube-dlc, but True when directly accessing YoutubeDL class)
182 force_generic_extractor: Force downloader to use the generic extractor
183 nooverwrites: Prevent overwriting files.
184 playliststart: Playlist item to start at.
185 playlistend: Playlist item to end at.
186 playlist_items: Specific indices of playlist to download.
187 playlistreverse: Download playlist items in reverse order.
188 playlistrandom: Download playlist items in random order.
189 matchtitle: Download only matching titles.
190 rejecttitle: Reject downloads for matching titles.
191 logger: Log messages to a logging.Logger instance.
192 logtostderr: Log messages to stderr instead of stdout.
193 writedescription: Write the video description to a .description file
194 writeinfojson: Write the video description to a .info.json file
195 writeannotations: Write the video annotations to a .annotations.xml file
196 writethumbnail: Write the thumbnail image to a file
197 write_all_thumbnails: Write all thumbnail formats to files
198 writelink: Write an internet shortcut file, depending on the
199 current platform (.url/.webloc/.desktop)
200 writeurllink: Write a Windows internet shortcut file (.url)
201 writewebloclink: Write a macOS internet shortcut file (.webloc)
202 writedesktoplink: Write a Linux internet shortcut file (.desktop)
203 writesubtitles: Write the video subtitles to a file
204 writeautomaticsub: Write the automatically generated subtitles to a file
205 allsubtitles: Downloads all the subtitles of the video
206 (requires writesubtitles or writeautomaticsub)
207 listsubtitles: Lists all available subtitles for the video
208 subtitlesformat: The format code for subtitles
209 subtitleslangs: List of languages of the subtitles to download
210 keepvideo: Keep the video file after post-processing
211 daterange: A DateRange object, download only if the upload_date is in the range.
212 skip_download: Skip the actual download of the video file
213 cachedir: Location of the cache files in the filesystem.
214 False to disable filesystem cache.
215 noplaylist: Download single video instead of a playlist if in doubt.
216 age_limit: An integer representing the user's age in years.
217 Unsuitable videos for the given age are skipped.
218 min_views: An integer representing the minimum view count the video
219 must have in order to not be skipped.
220 Videos without view count information are always
221 downloaded. None for no limit.
222 max_views: An integer representing the maximum view count.
223 Videos that are more popular than that are not
225 Videos without view count information are always
226 downloaded. None for no limit.
227 download_archive: File name of a file where all downloads are recorded.
228 Videos already present in the file are not downloaded
230 break_on_existing: Stop the download process after attempting to download a file that's
232 cookiefile: File name where cookies should be read from and dumped to.
233 nocheckcertificate:Do not verify SSL certificates
234 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
235 At the moment, this is only supported by YouTube.
236 proxy: URL of the proxy server to use
237 geo_verification_proxy: URL of the proxy to use for IP address verification
238 on geo-restricted sites.
239 socket_timeout: Time to wait for unresponsive hosts, in seconds
240 bidi_workaround: Work around buggy terminals without bidirectional text
241 support, using fridibi
242 debug_printtraffic:Print out sent and received HTTP traffic
243 include_ads: Download ads as well
244 default_search: Prepend this string if an input url is not valid.
245 'auto' for elaborate guessing
246 encoding: Use this encoding instead of the system-specified.
247 extract_flat: Do not resolve URLs, return the immediate result.
248 Pass in 'in_playlist' to only show this behavior for
250 postprocessors: A list of dictionaries, each with an entry
251 * key: The name of the postprocessor. See
252 youtube_dlc/postprocessor/__init__.py for a list.
253 as well as any further keyword arguments for the
255 post_hooks: A list of functions that get called as the final step
256 for each video file, after all postprocessors have been
257 called. The filename will be passed as the only argument.
258 progress_hooks: A list of functions that get called on download
259 progress, with a dictionary with the entries
260 * status: One of "downloading", "error", or "finished".
261 Check this first and ignore unknown values.
263 If status is one of "downloading", or "finished", the
264 following properties may also be present:
265 * filename: The final filename (always present)
266 * tmpfilename: The filename we're currently writing to
267 * downloaded_bytes: Bytes on disk
268 * total_bytes: Size of the whole file, None if unknown
269 * total_bytes_estimate: Guess of the eventual file size,
271 * elapsed: The number of seconds since download started.
272 * eta: The estimated time in seconds, None if unknown
273 * speed: The download speed in bytes/second, None if
275 * fragment_index: The counter of the currently
276 downloaded video fragment.
277 * fragment_count: The number of fragments (= individual
278 files that will be merged)
280 Progress hooks are guaranteed to be called at least once
281 (with status "finished") if the download is successful.
282 merge_output_format: Extension to use when merging formats.
283 fixup: Automatically correct known faults of the file.
285 - "never": do nothing
286 - "warn": only emit a warning
287 - "detect_or_warn": check whether we can do anything
288 about it, warn otherwise (default)
289 source_address: Client-side IP address to bind to.
290 call_home: Boolean, true iff we are allowed to contact the
291 youtube-dlc servers for debugging.
292 sleep_interval: Number of seconds to sleep before each download when
293 used alone or a lower bound of a range for randomized
294 sleep before each download (minimum possible number
295 of seconds to sleep) when used along with
297 max_sleep_interval:Upper bound of a range for randomized sleep before each
298 download (maximum possible number of seconds to sleep).
299 Must only be used along with sleep_interval.
300 Actual sleep time will be a random float from range
301 [sleep_interval; max_sleep_interval].
302 listformats: Print an overview of available video formats and exit.
303 list_thumbnails: Print a table of all thumbnails and exit.
304 match_filter: A function that gets called with the info_dict of
306 If it returns a message, the video is ignored.
307 If it returns None, the video is downloaded.
308 match_filter_func in utils.py is one example for this.
309 no_color: Do not emit color codes in output.
310 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
313 Two-letter ISO 3166-2 country code that will be used for
314 explicit geographic restriction bypassing via faking
315 X-Forwarded-For HTTP header
317 IP range in CIDR notation that will be used similarly to
320 The following options determine which downloader is picked:
321 external_downloader: Executable of the external downloader to call.
322 None or unset for standard (built-in) downloader.
323 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
324 if True, otherwise use ffmpeg/avconv if False, otherwise
325 use downloader suggested by extractor if None.
327 The following parameters are not used by YoutubeDL itself, they are used by
328 the downloader (see youtube_dlc/downloader/common.py):
329 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
330 noresizebuffer, retries, continuedl, noprogress, consoletitle,
331 xattr_set_filesize, external_downloader_args, hls_use_mpegts,
334 The following options are used by the post processors:
335 prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
336 otherwise prefer ffmpeg.
337 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
338 to the binary or its containing directory.
339 postprocessor_args: A dictionary of postprocessor names (in lower case) and a list
340 of additional command-line arguments for the postprocessor.
341 Use 'default' as the name for arguments to passed to all PP.
343 The following options are used by the Youtube extractor:
344 youtube_include_dash_manifest: If True (default), DASH manifests and related
345 data will be downloaded and processed by extractor.
346 You can reduce network I/O by disabling it if you don't
350 _NUMERIC_FIELDS
= set((
351 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
352 'timestamp', 'upload_year', 'upload_month', 'upload_day',
353 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
354 'average_rating', 'comment_count', 'age_limit',
355 'start_time', 'end_time',
356 'chapter_number', 'season_number', 'episode_number',
357 'track_number', 'disc_number', 'release_year',
364 _download_retcode
= None
365 _num_downloads
= None
368 def __init__(self
, params
=None, auto_init
=True):
369 """Create a FileDownloader object with the given options."""
373 self
._ies
_instances
= {}
375 self
._post
_hooks
= []
376 self
._progress
_hooks
= []
377 self
._download
_retcode
= 0
378 self
._num
_downloads
= 0
379 self
._screen
_file
= [sys
.stdout
, sys
.stderr
][params
.get('logtostderr', False)]
380 self
._err
_file
= sys
.stderr
383 'nocheckcertificate': False,
385 self
.params
.update(params
)
386 self
.cache
= Cache(self
)
389 """Preload the archive, if any is specified"""
390 def preload_download_archive(self
):
391 fn
= self
.params
.get('download_archive')
395 with locked_file(fn
, 'r', encoding
='utf-8') as archive_file
:
396 for line
in archive_file
:
397 self
.archive
.add(line
.strip())
398 except IOError as ioe
:
399 if ioe
.errno
!= errno
.ENOENT
:
404 def check_deprecated(param
, option
, suggestion
):
405 if self
.params
.get(param
) is not None:
407 '%s is deprecated. Use %s instead.' % (option
, suggestion
))
411 if self
.params
.get('verbose'):
412 self
.to_stdout('[debug] Loading archive file %r' % self
.params
.get('download_archive'))
414 preload_download_archive(self
)
416 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
417 if self
.params
.get('geo_verification_proxy') is None:
418 self
.params
['geo_verification_proxy'] = self
.params
['cn_verification_proxy']
420 check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
421 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
422 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
424 if params
.get('bidi_workaround', False):
427 master
, slave
= pty
.openpty()
428 width
= compat_get_terminal_size().columns
432 width_args
= ['-w', str(width
)]
434 stdin
=subprocess
.PIPE
,
436 stderr
=self
._err
_file
)
438 self
._output
_process
= subprocess
.Popen(
439 ['bidiv'] + width_args
, **sp_kwargs
442 self
._output
_process
= subprocess
.Popen(
443 ['fribidi', '-c', 'UTF-8'] + width_args
, **sp_kwargs
)
444 self
._output
_channel
= os
.fdopen(master
, 'rb')
445 except OSError as ose
:
446 if ose
.errno
== errno
.ENOENT
:
447 self
.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
451 if (sys
.platform
!= 'win32'
452 and sys
.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
453 and not params
.get('restrictfilenames', False)):
454 # Unicode filesystem API will throw errors (#1474, #13027)
456 'Assuming --restrict-filenames since file system encoding '
457 'cannot encode all characters. '
458 'Set the LC_ALL environment variable to fix this.')
459 self
.params
['restrictfilenames'] = True
461 if isinstance(params
.get('outtmpl'), bytes):
463 'Parameter outtmpl is bytes, but should be a unicode string. '
464 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
469 self
.print_debug_header()
470 self
.add_default_info_extractors()
472 for pp_def_raw
in self
.params
.get('postprocessors', []):
473 pp_class
= get_postprocessor(pp_def_raw
['key'])
474 pp_def
= dict(pp_def_raw
)
476 pp
= pp_class(self
, **compat_kwargs(pp_def
))
477 self
.add_post_processor(pp
)
479 for ph
in self
.params
.get('post_hooks', []):
480 self
.add_post_hook(ph
)
482 for ph
in self
.params
.get('progress_hooks', []):
483 self
.add_progress_hook(ph
)
485 register_socks_protocols()
487 def warn_if_short_id(self
, argv
):
488 # short YouTube ID starting with dash?
490 i
for i
, a
in enumerate(argv
)
491 if re
.match(r
'^-[0-9A-Za-z_-]{10}$', a
)]
495 + [a
for i
, a
in enumerate(argv
) if i
not in idxs
]
496 + ['--'] + [argv
[i
] for i
in idxs
]
499 'Long argument string detected. '
500 'Use -- to separate parameters and URLs, like this:\n%s\n' %
501 args_to_str(correct_argv
))
503 def add_info_extractor(self
, ie
):
504 """Add an InfoExtractor object to the end of the list."""
506 if not isinstance(ie
, type):
507 self
._ies
_instances
[ie
.ie_key()] = ie
508 ie
.set_downloader(self
)
510 def get_info_extractor(self
, ie_key
):
512 Get an instance of an IE with name ie_key, it will try to get one from
513 the _ies list, if there's no instance it will create a new one and add
514 it to the extractor list.
516 ie
= self
._ies
_instances
.get(ie_key
)
518 ie
= get_info_extractor(ie_key
)()
519 self
.add_info_extractor(ie
)
522 def add_default_info_extractors(self
):
524 Add the InfoExtractors returned by gen_extractors to the end of the list
526 for ie
in gen_extractor_classes():
527 self
.add_info_extractor(ie
)
529 def add_post_processor(self
, pp
):
530 """Add a PostProcessor object to the end of the chain."""
532 pp
.set_downloader(self
)
534 def add_post_hook(self
, ph
):
535 """Add the post hook"""
536 self
._post
_hooks
.append(ph
)
538 def add_progress_hook(self
, ph
):
539 """Add the progress hook (currently only for the file downloader)"""
540 self
._progress
_hooks
.append(ph
)
542 def _bidi_workaround(self
, message
):
543 if not hasattr(self
, '_output_channel'):
546 assert hasattr(self
, '_output_process')
547 assert isinstance(message
, compat_str
)
548 line_count
= message
.count('\n') + 1
549 self
._output
_process
.stdin
.write((message
+ '\n').encode('utf-8'))
550 self
._output
_process
.stdin
.flush()
551 res
= ''.join(self
._output
_channel
.readline().decode('utf-8')
552 for _
in range(line_count
))
553 return res
[:-len('\n')]
555 def to_screen(self
, message
, skip_eol
=False):
556 """Print message to stdout if not in quiet mode."""
557 return self
.to_stdout(message
, skip_eol
, check_quiet
=True)
559 def _write_string(self
, s
, out
=None):
560 write_string(s
, out
=out
, encoding
=self
.params
.get('encoding'))
562 def to_stdout(self
, message
, skip_eol
=False, check_quiet
=False):
563 """Print message to stdout if not in quiet mode."""
564 if self
.params
.get('logger'):
565 self
.params
['logger'].debug(message
)
566 elif not check_quiet
or not self
.params
.get('quiet', False):
567 message
= self
._bidi
_workaround
(message
)
568 terminator
= ['\n', ''][skip_eol
]
569 output
= message
+ terminator
571 self
._write
_string
(output
, self
._screen
_file
)
573 def to_stderr(self
, message
):
574 """Print message to stderr."""
575 assert isinstance(message
, compat_str
)
576 if self
.params
.get('logger'):
577 self
.params
['logger'].error(message
)
579 message
= self
._bidi
_workaround
(message
)
580 output
= message
+ '\n'
581 self
._write
_string
(output
, self
._err
_file
)
583 def to_console_title(self
, message
):
584 if not self
.params
.get('consoletitle', False):
586 if compat_os_name
== 'nt':
587 if ctypes
.windll
.kernel32
.GetConsoleWindow():
588 # c_wchar_p() might not be necessary if `message` is
589 # already of type unicode()
590 ctypes
.windll
.kernel32
.SetConsoleTitleW(ctypes
.c_wchar_p(message
))
591 elif 'TERM' in os
.environ
:
592 self
._write
_string
('\033]0;%s\007' % message
, self
._screen
_file
)
594 def save_console_title(self
):
595 if not self
.params
.get('consoletitle', False):
597 if self
.params
.get('simulate', False):
599 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
600 # Save the title on stack
601 self
._write
_string
('\033[22;0t', self
._screen
_file
)
603 def restore_console_title(self
):
604 if not self
.params
.get('consoletitle', False):
606 if self
.params
.get('simulate', False):
608 if compat_os_name
!= 'nt' and 'TERM' in os
.environ
:
609 # Restore the title from stack
610 self
._write
_string
('\033[23;0t', self
._screen
_file
)
613 self
.save_console_title()
616 def __exit__(self
, *args
):
617 self
.restore_console_title()
619 if self
.params
.get('cookiefile') is not None:
620 self
.cookiejar
.save(ignore_discard
=True, ignore_expires
=True)
622 def trouble(self
, message
=None, tb
=None):
623 """Determine action to take when a download problem appears.
625 Depending on if the downloader has been configured to ignore
626 download errors or not, this method may throw an exception or
627 not when errors are found, after printing the message.
629 tb, if given, is additional traceback information.
631 if message
is not None:
632 self
.to_stderr(message
)
633 if self
.params
.get('verbose'):
635 if sys
.exc_info()[0]: # if .trouble has been called from an except block
637 if hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
638 tb
+= ''.join(traceback
.format_exception(*sys
.exc_info()[1].exc_info
))
639 tb
+= encode_compat_str(traceback
.format_exc())
641 tb_data
= traceback
.format_list(traceback
.extract_stack())
642 tb
= ''.join(tb_data
)
644 if not self
.params
.get('ignoreerrors', False):
645 if sys
.exc_info()[0] and hasattr(sys
.exc_info()[1], 'exc_info') and sys
.exc_info()[1].exc_info
[0]:
646 exc_info
= sys
.exc_info()[1].exc_info
648 exc_info
= sys
.exc_info()
649 raise DownloadError(message
, exc_info
)
650 self
._download
_retcode
= 1
652 def report_warning(self
, message
):
654 Print the message to stderr, it will be prefixed with 'WARNING:'
655 If stderr is a tty file the 'WARNING:' will be colored
657 if self
.params
.get('logger') is not None:
658 self
.params
['logger'].warning(message
)
660 if self
.params
.get('no_warnings'):
662 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
663 _msg_header
= '\033[0;33mWARNING:\033[0m'
665 _msg_header
= 'WARNING:'
666 warning_message
= '%s %s' % (_msg_header
, message
)
667 self
.to_stderr(warning_message
)
669 def report_error(self
, message
, tb
=None):
671 Do the same as trouble, but prefixes the message with 'ERROR:', colored
672 in red if stderr is a tty file.
674 if not self
.params
.get('no_color') and self
._err
_file
.isatty() and compat_os_name
!= 'nt':
675 _msg_header
= '\033[0;31mERROR:\033[0m'
677 _msg_header
= 'ERROR:'
678 error_message
= '%s %s' % (_msg_header
, message
)
679 self
.trouble(error_message
, tb
)
681 def report_file_already_downloaded(self
, file_name
):
682 """Report file has already been fully downloaded."""
684 self
.to_screen('[download] %s has already been downloaded' % file_name
)
685 except UnicodeEncodeError:
686 self
.to_screen('[download] The file has already been downloaded')
688 def prepare_filename(self
, info_dict
):
689 """Generate the output filename."""
691 template_dict
= dict(info_dict
)
693 template_dict
['epoch'] = int(time
.time())
694 autonumber_size
= self
.params
.get('autonumber_size')
695 if autonumber_size
is None:
697 template_dict
['autonumber'] = self
.params
.get('autonumber_start', 1) - 1 + self
._num
_downloads
698 if template_dict
.get('resolution') is None:
699 if template_dict
.get('width') and template_dict
.get('height'):
700 template_dict
['resolution'] = '%dx%d' % (template_dict
['width'], template_dict
['height'])
701 elif template_dict
.get('height'):
702 template_dict
['resolution'] = '%sp' % template_dict
['height']
703 elif template_dict
.get('width'):
704 template_dict
['resolution'] = '%dx?' % template_dict
['width']
706 sanitize
= lambda k
, v
: sanitize_filename(
708 restricted
=self
.params
.get('restrictfilenames'),
709 is_id
=(k
== 'id' or k
.endswith('_id')))
710 template_dict
= dict((k
, v
if isinstance(v
, compat_numeric_types
) else sanitize(k
, v
))
711 for k
, v
in template_dict
.items()
712 if v
is not None and not isinstance(v
, (list, tuple, dict)))
713 template_dict
= collections
.defaultdict(lambda: 'NA', template_dict
)
715 outtmpl
= self
.params
.get('outtmpl', DEFAULT_OUTTMPL
)
717 # For fields playlist_index and autonumber convert all occurrences
718 # of %(field)s to %(field)0Nd for backward compatibility
719 field_size_compat_map
= {
720 'playlist_index': len(str(template_dict
['n_entries'])),
721 'autonumber': autonumber_size
,
723 FIELD_SIZE_COMPAT_RE
= r
'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
724 mobj
= re
.search(FIELD_SIZE_COMPAT_RE
, outtmpl
)
727 FIELD_SIZE_COMPAT_RE
,
728 r
'%%(\1)0%dd' % field_size_compat_map
[mobj
.group('field')],
731 # Missing numeric fields used together with integer presentation types
732 # in format specification will break the argument substitution since
733 # string 'NA' is returned for missing fields. We will patch output
734 # template for missing fields to meet string presentation type.
735 for numeric_field
in self
._NUMERIC
_FIELDS
:
736 if numeric_field
not in template_dict
:
737 # As of [1] format syntax is:
738 # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
739 # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
743 \({0}\) # mapping key
744 (?:[#0\-+ ]+)? # conversion flags (optional)
745 (?:\d+)? # minimum field width (optional)
746 (?:\.\d+)? # precision (optional)
747 [hlL]? # length modifier (optional)
748 [diouxXeEfFgGcrs%] # conversion type
751 FORMAT_RE
.format(numeric_field
),
752 r
'%({0})s'.format(numeric_field
), outtmpl
)
754 # expand_path translates '%%' into '%' and '$$' into '$'
755 # correspondingly that is not what we want since we need to keep
756 # '%%' intact for template dict substitution step. Working around
757 # with boundary-alike separator hack.
758 sep
= ''.join([random
.choice(ascii_letters
) for _
in range(32)])
759 outtmpl
= outtmpl
.replace('%%', '%{0}%'.format(sep
)).replace('$$', '${0}$'.format(sep
))
761 # outtmpl should be expand_path'ed before template dict substitution
762 # because meta fields may contain env variables we don't want to
763 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
764 # title "Hello $PATH", we don't want `$PATH` to be expanded.
765 filename
= expand_path(outtmpl
).replace(sep
, '') % template_dict
767 # https://github.com/blackjack4494/youtube-dlc/issues/85
768 trim_file_name
= self
.params
.get('trim_file_name', False)
770 fn_groups
= filename
.rsplit('.')
773 if len(fn_groups
) > 2:
774 sub_ext
= fn_groups
[-2]
775 filename
= '.'.join(filter(None, [fn_groups
[0][:trim_file_name
], sub_ext
, ext
]))
777 # Temporary fix for #4787
778 # 'Treat' all problem characters by passing filename through preferredencoding
779 # to workaround encoding issues with subprocess on python2 @ Windows
780 if sys
.version_info
< (3, 0) and sys
.platform
== 'win32':
781 filename
= encodeFilename(filename
, True).decode(preferredencoding())
782 return sanitize_path(filename
)
783 except ValueError as err
:
784 self
.report_error('Error in output template: ' + str(err
) + ' (encoding: ' + repr(preferredencoding()) + ')')
787 def _match_entry(self
, info_dict
, incomplete
):
788 """ Returns None if the file should be downloaded """
790 video_title
= info_dict
.get('title', info_dict
.get('id', 'video'))
791 if 'title' in info_dict
:
792 # This can happen when we're just evaluating the playlist
793 title
= info_dict
['title']
794 matchtitle
= self
.params
.get('matchtitle', False)
796 if not re
.search(matchtitle
, title
, re
.IGNORECASE
):
797 return '"' + title
+ '" title did not match pattern "' + matchtitle
+ '"'
798 rejecttitle
= self
.params
.get('rejecttitle', False)
800 if re
.search(rejecttitle
, title
, re
.IGNORECASE
):
801 return '"' + title
+ '" title matched reject pattern "' + rejecttitle
+ '"'
802 date
= info_dict
.get('upload_date')
804 dateRange
= self
.params
.get('daterange', DateRange())
805 if date
not in dateRange
:
806 return '%s upload date is not in range %s' % (date_from_str(date
).isoformat(), dateRange
)
807 view_count
= info_dict
.get('view_count')
808 if view_count
is not None:
809 min_views
= self
.params
.get('min_views')
810 if min_views
is not None and view_count
< min_views
:
811 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title
, view_count
, min_views
)
812 max_views
= self
.params
.get('max_views')
813 if max_views
is not None and view_count
> max_views
:
814 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title
, view_count
, max_views
)
815 if age_restricted(info_dict
.get('age_limit'), self
.params
.get('age_limit')):
816 return 'Skipping "%s" because it is age restricted' % video_title
817 if self
.in_download_archive(info_dict
):
818 return '%s has already been recorded in archive' % video_title
821 match_filter
= self
.params
.get('match_filter')
822 if match_filter
is not None:
823 ret
= match_filter(info_dict
)
830 def add_extra_info(info_dict
, extra_info
):
831 '''Set the keys from extra_info in info dict if they are missing'''
832 for key
, value
in extra_info
.items():
833 info_dict
.setdefault(key
, value
)
835 def extract_info(self
, url
, download
=True, ie_key
=None, info_dict
=None, extra_info
={},
836 process
=True, force_generic_extractor
=False):
838 Returns a list with a dictionary for each video we find.
839 If 'download', also downloads the videos.
840 extra_info is a dict containing the extra values to add to each result
843 if not ie_key
and force_generic_extractor
:
847 ies
= [self
.get_info_extractor(ie_key
)]
852 if not ie
.suitable(url
):
856 ie
= self
.get_info_extractor(ie_key
)
858 self
.report_warning('The program functionality for this site has been marked as broken, '
859 'and will probably not work.')
862 temp_id
= ie
.extract_id(url
) if callable(getattr(ie
, 'extract_id', None)) else ie
._match
_id
(url
)
863 except (AssertionError, IndexError, AttributeError):
865 if temp_id
is not None and self
.in_download_archive({'id': temp_id, 'ie_key': ie_key}
):
866 self
.to_screen("[%s] %s: has already been recorded in archive" % (
870 return self
.__extract
_info
(url
, ie
, download
, extra_info
, process
, info_dict
)
873 self
.report_error('no suitable InfoExtractor for URL %s' % url
)
875 def __handle_extraction_exceptions(func
):
876 def wrapper(self
, *args
, **kwargs
):
878 return func(self
, *args
, **kwargs
)
879 except GeoRestrictedError
as e
:
882 msg
+= '\nThis video is available in %s.' % ', '.join(
883 map(ISO3166Utils
.short2full
, e
.countries
))
884 msg
+= '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
885 self
.report_error(msg
)
886 except ExtractorError
as e
: # An error we somewhat expected
887 self
.report_error(compat_str(e
), e
.format_traceback())
888 except MaxDownloadsReached
:
890 except Exception as e
:
891 if self
.params
.get('ignoreerrors', False):
892 self
.report_error(error_to_compat_str(e
), tb
=encode_compat_str(traceback
.format_exc()))
897 @__handle_extraction_exceptions
898 def __extract_info(self
, url
, ie
, download
, extra_info
, process
, info_dict
):
899 ie_result
= ie
.extract(url
)
900 if ie_result
is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
902 if isinstance(ie_result
, list):
903 # Backwards compatibility: old IE result format
905 '_type': 'compat_list',
906 'entries': ie_result
,
909 if info_dict
.get('id'):
910 ie_result
['id'] = info_dict
['id']
911 if info_dict
.get('title'):
912 ie_result
['title'] = info_dict
['title']
913 self
.add_default_extra_info(ie_result
, ie
, url
)
915 return self
.process_ie_result(ie_result
, download
, extra_info
)
919 def add_default_extra_info(self
, ie_result
, ie
, url
):
920 self
.add_extra_info(ie_result
, {
921 'extractor': ie
.IE_NAME
,
924 formatSeconds(ie_result
['duration'], '-')
925 if ie_result
.get('duration', None) is not None
927 'webpage_url_basename': url_basename(url
),
928 'extractor_key': ie
.ie_key(),
931 def process_ie_result(self
, ie_result
, download
=True, extra_info
={}):
933 Take the result of the ie(may be modified) and resolve all unresolved
934 references (URLs, playlist items).
936 It will also download the videos if 'download'.
937 Returns the resolved ie_result.
939 result_type
= ie_result
.get('_type', 'video')
941 if result_type
in ('url', 'url_transparent'):
942 ie_result
['url'] = sanitize_url(ie_result
['url'])
943 extract_flat
= self
.params
.get('extract_flat', False)
944 if ((extract_flat
== 'in_playlist' and 'playlist' in extra_info
)
945 or extract_flat
is True):
946 self
.__forced
_printings
(
947 ie_result
, self
.prepare_filename(ie_result
),
951 if result_type
== 'video':
952 self
.add_extra_info(ie_result
, extra_info
)
953 return self
.process_video_result(ie_result
, download
=download
)
954 elif result_type
== 'url':
955 # We have to add extra_info to the results because it may be
956 # contained in a playlist
957 return self
.extract_info(ie_result
['url'],
958 download
, info_dict
=ie_result
,
959 ie_key
=ie_result
.get('ie_key'),
960 extra_info
=extra_info
)
961 elif result_type
== 'url_transparent':
962 # Use the information from the embedding page
963 info
= self
.extract_info(
964 ie_result
['url'], ie_key
=ie_result
.get('ie_key'),
965 extra_info
=extra_info
, download
=False, process
=False)
967 # extract_info may return None when ignoreerrors is enabled and
968 # extraction failed with an error, don't crash and return early
973 force_properties
= dict(
974 (k
, v
) for k
, v
in ie_result
.items() if v
is not None)
975 for f
in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
976 if f
in force_properties
:
977 del force_properties
[f
]
978 new_result
= info
.copy()
979 new_result
.update(force_properties
)
981 # Extracted info may not be a video result (i.e.
982 # info.get('_type', 'video') != video) but rather an url or
983 # url_transparent. In such cases outer metadata (from ie_result)
984 # should be propagated to inner one (info). For this to happen
985 # _type of info should be overridden with url_transparent. This
986 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
987 if new_result
.get('_type') == 'url':
988 new_result
['_type'] = 'url_transparent'
990 return self
.process_ie_result(
991 new_result
, download
=download
, extra_info
=extra_info
)
992 elif result_type
in ('playlist', 'multi_video'):
993 # We process each entry in the playlist
994 playlist
= ie_result
.get('title') or ie_result
.get('id')
995 self
.to_screen('[download] Downloading playlist: %s' % playlist
)
997 playlist_results
= []
999 playliststart
= self
.params
.get('playliststart', 1) - 1
1000 playlistend
= self
.params
.get('playlistend')
1001 # For backwards compatibility, interpret -1 as whole list
1002 if playlistend
== -1:
1005 playlistitems_str
= self
.params
.get('playlist_items')
1006 playlistitems
= None
1007 if playlistitems_str
is not None:
1008 def iter_playlistitems(format
):
1009 for string_segment
in format
.split(','):
1010 if '-' in string_segment
:
1011 start
, end
= string_segment
.split('-')
1012 for item
in range(int(start
), int(end
) + 1):
1015 yield int(string_segment
)
1016 playlistitems
= orderedSet(iter_playlistitems(playlistitems_str
))
1018 ie_entries
= ie_result
['entries']
1020 def make_playlistitems_entries(list_ie_entries
):
1021 num_entries
= len(list_ie_entries
)
1023 list_ie_entries
[i
- 1] for i
in playlistitems
1024 if -num_entries
<= i
- 1 < num_entries
]
1026 def report_download(num_entries
):
1028 '[%s] playlist %s: Downloading %d videos' %
1029 (ie_result
['extractor'], playlist
, num_entries
))
1031 if isinstance(ie_entries
, list):
1032 n_all_entries
= len(ie_entries
)
1034 entries
= make_playlistitems_entries(ie_entries
)
1036 entries
= ie_entries
[playliststart
:playlistend
]
1037 n_entries
= len(entries
)
1039 '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
1040 (ie_result
['extractor'], playlist
, n_all_entries
, n_entries
))
1041 elif isinstance(ie_entries
, PagedList
):
1044 for item
in playlistitems
:
1045 entries
.extend(ie_entries
.getslice(
1049 entries
= ie_entries
.getslice(
1050 playliststart
, playlistend
)
1051 n_entries
= len(entries
)
1052 report_download(n_entries
)
1055 entries
= make_playlistitems_entries(list(itertools
.islice(
1056 ie_entries
, 0, max(playlistitems
))))
1058 entries
= list(itertools
.islice(
1059 ie_entries
, playliststart
, playlistend
))
1060 n_entries
= len(entries
)
1061 report_download(n_entries
)
1063 if self
.params
.get('playlistreverse', False):
1064 entries
= entries
[::-1]
1066 if self
.params
.get('playlistrandom', False):
1067 random
.shuffle(entries
)
1069 x_forwarded_for
= ie_result
.get('__x_forwarded_for_ip')
1071 for i
, entry
in enumerate(entries
, 1):
1072 self
.to_screen('[download] Downloading video %s of %s' % (i
, n_entries
))
1073 # This __x_forwarded_for_ip thing is a bit ugly but requires
1076 entry
['__x_forwarded_for_ip'] = x_forwarded_for
1078 'n_entries': n_entries
,
1079 'playlist': playlist
,
1080 'playlist_id': ie_result
.get('id'),
1081 'playlist_title': ie_result
.get('title'),
1082 'playlist_uploader': ie_result
.get('uploader'),
1083 'playlist_uploader_id': ie_result
.get('uploader_id'),
1084 'playlist_index': playlistitems
[i
- 1] if playlistitems
else i
+ playliststart
,
1085 'extractor': ie_result
['extractor'],
1086 'webpage_url': ie_result
['webpage_url'],
1087 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1088 'extractor_key': ie_result
['extractor_key'],
1091 reason
= self
._match
_entry
(entry
, incomplete
=True)
1092 if reason
is not None:
1093 if reason
.endswith('has already been recorded in the archive') and self
.params
.get('break_on_existing'):
1094 print('[download] tried downloading a file that\'s already in the archive, stopping since --break-on-existing is set.')
1097 self
.to_screen('[download] ' + reason
)
1100 entry_result
= self
.__process
_iterable
_entry
(entry
, download
, extra
)
1101 # TODO: skip failed (empty) entries?
1102 playlist_results
.append(entry_result
)
1103 ie_result
['entries'] = playlist_results
1104 self
.to_screen('[download] Finished downloading playlist: %s' % playlist
)
1106 elif result_type
== 'compat_list':
1107 self
.report_warning(
1108 'Extractor %s returned a compat_list result. '
1109 'It needs to be updated.' % ie_result
.get('extractor'))
1112 self
.add_extra_info(
1115 'extractor': ie_result
['extractor'],
1116 'webpage_url': ie_result
['webpage_url'],
1117 'webpage_url_basename': url_basename(ie_result
['webpage_url']),
1118 'extractor_key': ie_result
['extractor_key'],
1122 ie_result
['entries'] = [
1123 self
.process_ie_result(_fixup(r
), download
, extra_info
)
1124 for r
in ie_result
['entries']
1128 raise Exception('Invalid result type: %s' % result_type
)
1130 @__handle_extraction_exceptions
1131 def __process_iterable_entry(self
, entry
, download
, extra_info
):
1132 return self
.process_ie_result(
1133 entry
, download
=download
, extra_info
=extra_info
)
1135 def _build_format_filter(self
, filter_spec
):
1136 " Returns a function to filter the formats according to the filter_spec "
1146 operator_rex
= re
.compile(r
'''(?x)\s*
1147 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
1148 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1149 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1151 ''' % '|'.join(map(re
.escape
, OPERATORS
.keys())))
1152 m
= operator_rex
.search(filter_spec
)
1155 comparison_value
= int(m
.group('value'))
1157 comparison_value
= parse_filesize(m
.group('value'))
1158 if comparison_value
is None:
1159 comparison_value
= parse_filesize(m
.group('value') + 'B')
1160 if comparison_value
is None:
1162 'Invalid value %r in format specification %r' % (
1163 m
.group('value'), filter_spec
))
1164 op
= OPERATORS
[m
.group('op')]
1169 '^=': lambda attr
, value
: attr
.startswith(value
),
1170 '$=': lambda attr
, value
: attr
.endswith(value
),
1171 '*=': lambda attr
, value
: value
in attr
,
1173 str_operator_rex
= re
.compile(r
'''(?x)
1174 \s*(?P<key>[a-zA-Z0-9._-]+)
1175 \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?
1176 \s*(?P<value>[a-zA-Z0-9._-]+)
1178 ''' % '|'.join(map(re
.escape
, STR_OPERATORS
.keys())))
1179 m
= str_operator_rex
.search(filter_spec
)
1181 comparison_value
= m
.group('value')
1182 str_op
= STR_OPERATORS
[m
.group('op')]
1183 if m
.group('negation'):
1184 op
= lambda attr
, value
: not str_op(attr
, value
)
1189 raise ValueError('Invalid filter specification %r' % filter_spec
)
1192 actual_value
= f
.get(m
.group('key'))
1193 if actual_value
is None:
1194 return m
.group('none_inclusive')
1195 return op(actual_value
, comparison_value
)
1198 def _default_format_spec(self
, info_dict
, download
=True):
1201 merger
= FFmpegMergerPP(self
)
1202 return merger
.available
and merger
.can_merge()
1205 not self
.params
.get('simulate', False)
1209 or info_dict
.get('is_live', False)
1210 or self
.params
.get('outtmpl', DEFAULT_OUTTMPL
) == '-'))
1213 'best/bestvideo+bestaudio'
1215 else 'bestvideo*+bestaudio/best'
1216 if not self
.params
.get('allow_multiple_audio_streams', False)
1217 else 'bestvideo+bestaudio/best')
1219 def build_format_selector(self
, format_spec
):
1220 def syntax_error(note
, start
):
1222 'Invalid format specification: '
1223 '{0}\n\t{1}\n\t{2}^'.format(note
, format_spec
, ' ' * start
[1]))
1224 return SyntaxError(message
)
1226 PICKFIRST
= 'PICKFIRST'
1230 FormatSelector
= collections
.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1232 allow_multiple_streams
= {'audio': self
.params
.get('allow_multiple_audio_streams', False),
1233 'video': self
.params
.get('allow_multiple_video_streams', False)}
1235 def _parse_filter(tokens
):
1237 for type, string
, start
, _
, _
in tokens
:
1238 if type == tokenize
.OP
and string
== ']':
1239 return ''.join(filter_parts
)
1241 filter_parts
.append(string
)
1243 def _remove_unused_ops(tokens
):
1244 # Remove operators that we don't use and join them with the surrounding strings
1245 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1246 ALLOWED_OPS
= ('/', '+', ',', '(', ')')
1247 last_string
, last_start
, last_end
, last_line
= None, None, None, None
1248 for type, string
, start
, end
, line
in tokens
:
1249 if type == tokenize
.OP
and string
== '[':
1251 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1253 yield type, string
, start
, end
, line
1254 # everything inside brackets will be handled by _parse_filter
1255 for type, string
, start
, end
, line
in tokens
:
1256 yield type, string
, start
, end
, line
1257 if type == tokenize
.OP
and string
== ']':
1259 elif type == tokenize
.OP
and string
in ALLOWED_OPS
:
1261 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1263 yield type, string
, start
, end
, line
1264 elif type in [tokenize
.NAME
, tokenize
.NUMBER
, tokenize
.OP
]:
1266 last_string
= string
1270 last_string
+= string
1272 yield tokenize
.NAME
, last_string
, last_start
, last_end
, last_line
1274 def _parse_format_selection(tokens
, inside_merge
=False, inside_choice
=False, inside_group
=False):
1276 current_selector
= None
1277 for type, string
, start
, _
, _
in tokens
:
1278 # ENCODING is only defined in python 3.x
1279 if type == getattr(tokenize
, 'ENCODING', None):
1281 elif type in [tokenize
.NAME
, tokenize
.NUMBER
]:
1282 current_selector
= FormatSelector(SINGLE
, string
, [])
1283 elif type == tokenize
.OP
:
1285 if not inside_group
:
1286 # ')' will be handled by the parentheses group
1287 tokens
.restore_last_token()
1289 elif inside_merge
and string
in ['/', ',']:
1290 tokens
.restore_last_token()
1292 elif inside_choice
and string
== ',':
1293 tokens
.restore_last_token()
1296 if not current_selector
:
1297 raise syntax_error('"," must follow a format selector', start
)
1298 selectors
.append(current_selector
)
1299 current_selector
= None
1301 if not current_selector
:
1302 raise syntax_error('"/" must follow a format selector', start
)
1303 first_choice
= current_selector
1304 second_choice
= _parse_format_selection(tokens
, inside_choice
=True)
1305 current_selector
= FormatSelector(PICKFIRST
, (first_choice
, second_choice
), [])
1307 if not current_selector
:
1308 current_selector
= FormatSelector(SINGLE
, 'best', [])
1309 format_filter
= _parse_filter(tokens
)
1310 current_selector
.filters
.append(format_filter
)
1312 if current_selector
:
1313 raise syntax_error('Unexpected "("', start
)
1314 group
= _parse_format_selection(tokens
, inside_group
=True)
1315 current_selector
= FormatSelector(GROUP
, group
, [])
1317 if not current_selector
:
1318 raise syntax_error('Unexpected "+"', start
)
1319 selector_1
= current_selector
1320 selector_2
= _parse_format_selection(tokens
, inside_merge
=True)
1322 raise syntax_error('Expected a selector', start
)
1323 current_selector
= FormatSelector(MERGE
, (selector_1
, selector_2
), [])
1325 raise syntax_error('Operator not recognized: "{0}"'.format(string
), start
)
1326 elif type == tokenize
.ENDMARKER
:
1328 if current_selector
:
1329 selectors
.append(current_selector
)
1332 def _build_selector_function(selector
):
1333 if isinstance(selector
, list): # ,
1334 fs
= [_build_selector_function(s
) for s
in selector
]
1336 def selector_function(ctx
):
1338 for format
in f(ctx
):
1340 return selector_function
1342 elif selector
.type == GROUP
: # ()
1343 selector_function
= _build_selector_function(selector
.selector
)
1345 elif selector
.type == PICKFIRST
: # /
1346 fs
= [_build_selector_function(s
) for s
in selector
.selector
]
1348 def selector_function(ctx
):
1350 picked_formats
= list(f(ctx
))
1352 return picked_formats
1355 elif selector
.type == SINGLE
: # atom
1356 format_spec
= selector
.selector
if selector
.selector
is not None else 'best'
1358 if format_spec
== 'all':
1359 def selector_function(ctx
):
1360 formats
= list(ctx
['formats'])
1366 format_fallback
= False
1367 format_spec_obj
= re
.match(r
'(best|worst|b|w)(video|audio|v|a)?(\*)?$', format_spec
)
1368 if format_spec_obj
is not None:
1369 format_idx
= 0 if format_spec_obj
.group(1)[0] == 'w' else -1
1370 format_type
= format_spec_obj
.group(2)[0] if format_spec_obj
.group(2) else False
1371 not_format_type
= 'v' if format_type
== 'a' else 'a'
1372 format_modified
= format_spec_obj
.group(3) is not None
1374 format_fallback
= not format_type
and not format_modified
# for b, w
1375 filter_f
= ((lambda f
: f
.get(format_type
+ 'codec') != 'none')
1376 if format_type
and format_modified
# bv*, ba*, wv*, wa*
1377 else (lambda f
: f
.get(not_format_type
+ 'codec') == 'none')
1378 if format_type
# bv, ba, wv, wa
1379 else (lambda f
: f
.get('vcodec') != 'none' and f
.get('acodec') != 'none')
1380 if not format_modified
# b, w
1384 filter_f
= ((lambda f
: f
.get('ext') == format_spec
)
1385 if format_spec
in ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav'] # extension
1386 else (lambda f
: f
.get('format_id') == format_spec
)) # id
1388 def selector_function(ctx
):
1389 formats
= list(ctx
['formats'])
1392 matches
= list(filter(filter_f
, formats
)) if filter_f
is not None else formats
1394 yield matches
[format_idx
]
1395 elif format_fallback
== 'force' or (format_fallback
and ctx
['incomplete_formats']):
1396 # for extractors with incomplete formats (audio only (soundcloud)
1397 # or video only (imgur)) best/worst will fallback to
1398 # best/worst {video,audio}-only format
1399 yield formats
[format_idx
]
1401 elif selector
.type == MERGE
: # +
1402 def _merge(formats_pair
):
1403 format_1
, format_2
= formats_pair
1406 formats_info
.extend(format_1
.get('requested_formats', (format_1
,)))
1407 formats_info
.extend(format_2
.get('requested_formats', (format_2
,)))
1409 if not allow_multiple_streams
['video'] or not allow_multiple_streams
['audio']:
1410 get_no_more
= {"video": False, "audio": False}
1411 for (i
, fmt_info
) in enumerate(formats_info
):
1412 for aud_vid
in ["audio", "video"]:
1413 if not allow_multiple_streams
[aud_vid
] and fmt_info
.get(aud_vid
[0] + 'codec') != 'none':
1414 if get_no_more
[aud_vid
]:
1416 get_no_more
[aud_vid
] = True
1418 if len(formats_info
) == 1:
1419 return formats_info
[0]
1421 video_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('vcodec') != 'none']
1422 audio_fmts
= [fmt_info
for fmt_info
in formats_info
if fmt_info
.get('acodec') != 'none']
1424 the_only_video
= video_fmts
[0] if len(video_fmts
) == 1 else None
1425 the_only_audio
= audio_fmts
[0] if len(audio_fmts
) == 1 else None
1427 output_ext
= self
.params
.get('merge_output_format')
1430 output_ext
= the_only_video
['ext']
1431 elif the_only_audio
and not video_fmts
:
1432 output_ext
= the_only_audio
['ext']
1437 'requested_formats': formats_info
,
1438 'format': '+'.join(fmt_info
.get('format') for fmt_info
in formats_info
),
1439 'format_id': '+'.join(fmt_info
.get('format_id') for fmt_info
in formats_info
),
1445 'width': the_only_video
.get('width'),
1446 'height': the_only_video
.get('height'),
1447 'resolution': the_only_video
.get('resolution'),
1448 'fps': the_only_video
.get('fps'),
1449 'vcodec': the_only_video
.get('vcodec'),
1450 'vbr': the_only_video
.get('vbr'),
1451 'stretched_ratio': the_only_video
.get('stretched_ratio'),
1456 'acodec': the_only_audio
.get('acodec'),
1457 'abr': the_only_audio
.get('abr'),
1462 selector_1
, selector_2
= map(_build_selector_function
, selector
.selector
)
1464 def selector_function(ctx
):
1465 for pair
in itertools
.product(
1466 selector_1(copy
.deepcopy(ctx
)), selector_2(copy
.deepcopy(ctx
))):
1469 filters
= [self
._build
_format
_filter
(f
) for f
in selector
.filters
]
1471 def final_selector(ctx
):
1472 ctx_copy
= copy
.deepcopy(ctx
)
1473 for _filter
in filters
:
1474 ctx_copy
['formats'] = list(filter(_filter
, ctx_copy
['formats']))
1475 return selector_function(ctx_copy
)
1476 return final_selector
1478 stream
= io
.BytesIO(format_spec
.encode('utf-8'))
1480 tokens
= list(_remove_unused_ops(compat_tokenize_tokenize(stream
.readline
)))
1481 except tokenize
.TokenError
:
1482 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec
)))
1484 class TokenIterator(object):
1485 def __init__(self
, tokens
):
1486 self
.tokens
= tokens
1493 if self
.counter
>= len(self
.tokens
):
1494 raise StopIteration()
1495 value
= self
.tokens
[self
.counter
]
1501 def restore_last_token(self
):
1504 parsed_selector
= _parse_format_selection(iter(TokenIterator(tokens
)))
1505 return _build_selector_function(parsed_selector
)
1507 def _calc_headers(self
, info_dict
):
1508 res
= std_headers
.copy()
1510 add_headers
= info_dict
.get('http_headers')
1512 res
.update(add_headers
)
1514 cookies
= self
._calc
_cookies
(info_dict
)
1516 res
['Cookie'] = cookies
1518 if 'X-Forwarded-For' not in res
:
1519 x_forwarded_for_ip
= info_dict
.get('__x_forwarded_for_ip')
1520 if x_forwarded_for_ip
:
1521 res
['X-Forwarded-For'] = x_forwarded_for_ip
1525 def _calc_cookies(self
, info_dict
):
1526 pr
= sanitized_Request(info_dict
['url'])
1527 self
.cookiejar
.add_cookie_header(pr
)
1528 return pr
.get_header('Cookie')
1530 def process_video_result(self
, info_dict
, download
=True):
1531 assert info_dict
.get('_type', 'video') == 'video'
1533 if 'id' not in info_dict
:
1534 raise ExtractorError('Missing "id" field in extractor result')
1535 if 'title' not in info_dict
:
1536 raise ExtractorError('Missing "title" field in extractor result')
1538 def report_force_conversion(field
, field_not
, conversion
):
1539 self
.report_warning(
1540 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1541 % (field
, field_not
, conversion
))
1543 def sanitize_string_field(info
, string_field
):
1544 field
= info
.get(string_field
)
1545 if field
is None or isinstance(field
, compat_str
):
1547 report_force_conversion(string_field
, 'a string', 'string')
1548 info
[string_field
] = compat_str(field
)
1550 def sanitize_numeric_fields(info
):
1551 for numeric_field
in self
._NUMERIC
_FIELDS
:
1552 field
= info
.get(numeric_field
)
1553 if field
is None or isinstance(field
, compat_numeric_types
):
1555 report_force_conversion(numeric_field
, 'numeric', 'int')
1556 info
[numeric_field
] = int_or_none(field
)
1558 sanitize_string_field(info_dict
, 'id')
1559 sanitize_numeric_fields(info_dict
)
1561 if 'playlist' not in info_dict
:
1562 # It isn't part of a playlist
1563 info_dict
['playlist'] = None
1564 info_dict
['playlist_index'] = None
1566 thumbnails
= info_dict
.get('thumbnails')
1567 if thumbnails
is None:
1568 thumbnail
= info_dict
.get('thumbnail')
1570 info_dict
['thumbnails'] = thumbnails
= [{'url': thumbnail}
]
1572 thumbnails
.sort(key
=lambda t
: (
1573 t
.get('preference') if t
.get('preference') is not None else -1,
1574 t
.get('width') if t
.get('width') is not None else -1,
1575 t
.get('height') if t
.get('height') is not None else -1,
1576 t
.get('id') if t
.get('id') is not None else '', t
.get('url')))
1577 for i
, t
in enumerate(thumbnails
):
1578 t
['url'] = sanitize_url(t
['url'])
1579 if t
.get('width') and t
.get('height'):
1580 t
['resolution'] = '%dx%d' % (t
['width'], t
['height'])
1581 if t
.get('id') is None:
1584 if self
.params
.get('list_thumbnails'):
1585 self
.list_thumbnails(info_dict
)
1588 thumbnail
= info_dict
.get('thumbnail')
1590 info_dict
['thumbnail'] = sanitize_url(thumbnail
)
1592 info_dict
['thumbnail'] = thumbnails
[-1]['url']
1594 if 'display_id' not in info_dict
and 'id' in info_dict
:
1595 info_dict
['display_id'] = info_dict
['id']
1597 if info_dict
.get('upload_date') is None and info_dict
.get('timestamp') is not None:
1598 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1599 # see http://bugs.python.org/issue1646728)
1601 upload_date
= datetime
.datetime
.utcfromtimestamp(info_dict
['timestamp'])
1602 info_dict
['upload_date'] = upload_date
.strftime('%Y%m%d')
1603 except (ValueError, OverflowError, OSError):
1606 # Auto generate title fields corresponding to the *_number fields when missing
1607 # in order to always have clean titles. This is very common for TV series.
1608 for field
in ('chapter', 'season', 'episode'):
1609 if info_dict
.get('%s_number' % field
) is not None and not info_dict
.get(field
):
1610 info_dict
[field
] = '%s %d' % (field
.capitalize(), info_dict
['%s_number' % field
])
1612 for cc_kind
in ('subtitles', 'automatic_captions'):
1613 cc
= info_dict
.get(cc_kind
)
1615 for _
, subtitle
in cc
.items():
1616 for subtitle_format
in subtitle
:
1617 if subtitle_format
.get('url'):
1618 subtitle_format
['url'] = sanitize_url(subtitle_format
['url'])
1619 if subtitle_format
.get('ext') is None:
1620 subtitle_format
['ext'] = determine_ext(subtitle_format
['url']).lower()
1622 automatic_captions
= info_dict
.get('automatic_captions')
1623 subtitles
= info_dict
.get('subtitles')
1625 if self
.params
.get('listsubtitles', False):
1626 if 'automatic_captions' in info_dict
:
1627 self
.list_subtitles(
1628 info_dict
['id'], automatic_captions
, 'automatic captions')
1629 self
.list_subtitles(info_dict
['id'], subtitles
, 'subtitles')
1632 info_dict
['requested_subtitles'] = self
.process_subtitles(
1633 info_dict
['id'], subtitles
, automatic_captions
)
1635 # We now pick which formats have to be downloaded
1636 if info_dict
.get('formats') is None:
1637 # There's only one format available
1638 formats
= [info_dict
]
1640 formats
= info_dict
['formats']
1643 raise ExtractorError('No video formats found!')
1645 def is_wellformed(f
):
1648 self
.report_warning(
1649 '"url" field is missing or empty - skipping format, '
1650 'there is an error in extractor')
1652 if isinstance(url
, bytes):
1653 sanitize_string_field(f
, 'url')
1656 # Filter out malformed formats for better extraction robustness
1657 formats
= list(filter(is_wellformed
, formats
))
1661 # We check that all the formats have the format and format_id fields
1662 for i
, format
in enumerate(formats
):
1663 sanitize_string_field(format
, 'format_id')
1664 sanitize_numeric_fields(format
)
1665 format
['url'] = sanitize_url(format
['url'])
1666 if not format
.get('format_id'):
1667 format
['format_id'] = compat_str(i
)
1669 # Sanitize format_id from characters used in format selector expression
1670 format
['format_id'] = re
.sub(r
'[\s,/+\[\]()]', '_', format
['format_id'])
1671 format_id
= format
['format_id']
1672 if format_id
not in formats_dict
:
1673 formats_dict
[format_id
] = []
1674 formats_dict
[format_id
].append(format
)
1676 # Make sure all formats have unique format_id
1677 for format_id
, ambiguous_formats
in formats_dict
.items():
1678 if len(ambiguous_formats
) > 1:
1679 for i
, format
in enumerate(ambiguous_formats
):
1680 format
['format_id'] = '%s-%d' % (format_id
, i
)
1682 for i
, format
in enumerate(formats
):
1683 if format
.get('format') is None:
1684 format
['format'] = '{id} - {res}{note}'.format(
1685 id=format
['format_id'],
1686 res
=self
.format_resolution(format
),
1687 note
=' ({0})'.format(format
['format_note']) if format
.get('format_note') is not None else '',
1689 # Automatically determine file extension if missing
1690 if format
.get('ext') is None:
1691 format
['ext'] = determine_ext(format
['url']).lower()
1692 # Automatically determine protocol if missing (useful for format
1693 # selection purposes)
1694 if format
.get('protocol') is None:
1695 format
['protocol'] = determine_protocol(format
)
1696 # Add HTTP headers, so that external programs can use them from the
1698 full_format_info
= info_dict
.copy()
1699 full_format_info
.update(format
)
1700 format
['http_headers'] = self
._calc
_headers
(full_format_info
)
1701 # Remove private housekeeping stuff
1702 if '__x_forwarded_for_ip' in info_dict
:
1703 del info_dict
['__x_forwarded_for_ip']
1705 # TODO Central sorting goes here
1707 if formats
[0] is not info_dict
:
1708 # only set the 'formats' fields if the original info_dict list them
1709 # otherwise we end up with a circular reference, the first (and unique)
1710 # element in the 'formats' field in info_dict is info_dict itself,
1711 # which can't be exported to json
1712 info_dict
['formats'] = formats
1713 if self
.params
.get('listformats'):
1714 self
.list_formats(info_dict
)
1717 req_format
= self
.params
.get('format')
1718 if req_format
is None:
1719 req_format
= self
._default
_format
_spec
(info_dict
, download
=download
)
1720 if self
.params
.get('verbose'):
1721 self
._write
_string
('[debug] Default format spec: %s\n' % req_format
)
1723 format_selector
= self
.build_format_selector(req_format
)
1725 # While in format selection we may need to have an access to the original
1726 # format set in order to calculate some metrics or do some processing.
1727 # For now we need to be able to guess whether original formats provided
1728 # by extractor are incomplete or not (i.e. whether extractor provides only
1729 # video-only or audio-only formats) for proper formats selection for
1730 # extractors with such incomplete formats (see
1731 # https://github.com/ytdl-org/youtube-dl/pull/5556).
1732 # Since formats may be filtered during format selection and may not match
1733 # the original formats the results may be incorrect. Thus original formats
1734 # or pre-calculated metrics should be passed to format selection routines
1736 # We will pass a context object containing all necessary additional data
1737 # instead of just formats.
1738 # This fixes incorrect format selection issue (see
1739 # https://github.com/ytdl-org/youtube-dl/issues/10083).
1740 incomplete_formats
= (
1741 # All formats are video-only or
1742 all(f
.get('vcodec') != 'none' and f
.get('acodec') == 'none' for f
in formats
)
1743 # all formats are audio-only
1744 or all(f
.get('vcodec') == 'none' and f
.get('acodec') != 'none' for f
in formats
))
1748 'incomplete_formats': incomplete_formats
,
1751 formats_to_download
= list(format_selector(ctx
))
1752 if not formats_to_download
:
1753 raise ExtractorError('requested format not available',
1757 self
.to_screen('[info] Downloading format(s) %s' % ", ".join([f
['format_id'] for f
in formats_to_download
]))
1758 if len(formats_to_download
) > 1:
1759 self
.to_screen('[info] %s: downloading video in %s formats' % (info_dict
['id'], len(formats_to_download
)))
1760 for format
in formats_to_download
:
1761 new_info
= dict(info_dict
)
1762 new_info
.update(format
)
1763 self
.process_info(new_info
)
1764 # We update the info dict with the best quality format (backwards compatibility)
1765 info_dict
.update(formats_to_download
[-1])
1768 def process_subtitles(self
, video_id
, normal_subtitles
, automatic_captions
):
1769 """Select the requested subtitles and their format"""
1771 if normal_subtitles
and self
.params
.get('writesubtitles'):
1772 available_subs
.update(normal_subtitles
)
1773 if automatic_captions
and self
.params
.get('writeautomaticsub'):
1774 for lang
, cap_info
in automatic_captions
.items():
1775 if lang
not in available_subs
:
1776 available_subs
[lang
] = cap_info
1778 if (not self
.params
.get('writesubtitles') and not
1779 self
.params
.get('writeautomaticsub') or not
1783 if self
.params
.get('allsubtitles', False):
1784 requested_langs
= available_subs
.keys()
1786 if self
.params
.get('subtitleslangs', False):
1787 requested_langs
= self
.params
.get('subtitleslangs')
1788 elif 'en' in available_subs
:
1789 requested_langs
= ['en']
1791 requested_langs
= [list(available_subs
.keys())[0]]
1793 formats_query
= self
.params
.get('subtitlesformat', 'best')
1794 formats_preference
= formats_query
.split('/') if formats_query
else []
1796 for lang
in requested_langs
:
1797 formats
= available_subs
.get(lang
)
1799 self
.report_warning('%s subtitles not available for %s' % (lang
, video_id
))
1801 for ext
in formats_preference
:
1805 matches
= list(filter(lambda f
: f
['ext'] == ext
, formats
))
1811 self
.report_warning(
1812 'No subtitle format found matching "%s" for language %s, '
1813 'using %s' % (formats_query
, lang
, f
['ext']))
1817 def __forced_printings(self
, info_dict
, filename
, incomplete
):
1818 def print_mandatory(field
):
1819 if (self
.params
.get('force%s' % field
, False)
1820 and (not incomplete
or info_dict
.get(field
) is not None)):
1821 self
.to_stdout(info_dict
[field
])
1823 def print_optional(field
):
1824 if (self
.params
.get('force%s' % field
, False)
1825 and info_dict
.get(field
) is not None):
1826 self
.to_stdout(info_dict
[field
])
1828 print_mandatory('title')
1829 print_mandatory('id')
1830 if self
.params
.get('forceurl', False) and not incomplete
:
1831 if info_dict
.get('requested_formats') is not None:
1832 for f
in info_dict
['requested_formats']:
1833 self
.to_stdout(f
['url'] + f
.get('play_path', ''))
1835 # For RTMP URLs, also include the playpath
1836 self
.to_stdout(info_dict
['url'] + info_dict
.get('play_path', ''))
1837 print_optional('thumbnail')
1838 print_optional('description')
1839 if self
.params
.get('forcefilename', False) and filename
is not None:
1840 self
.to_stdout(filename
)
1841 if self
.params
.get('forceduration', False) and info_dict
.get('duration') is not None:
1842 self
.to_stdout(formatSeconds(info_dict
['duration']))
1843 print_mandatory('format')
1844 if self
.params
.get('forcejson', False):
1845 self
.to_stdout(json
.dumps(info_dict
))
1847 def process_info(self
, info_dict
):
1848 """Process a single resolved IE result."""
1850 assert info_dict
.get('_type', 'video') == 'video'
1852 max_downloads
= self
.params
.get('max_downloads')
1853 if max_downloads
is not None:
1854 if self
._num
_downloads
>= int(max_downloads
):
1855 raise MaxDownloadsReached()
1857 # TODO: backward compatibility, to be removed
1858 info_dict
['fulltitle'] = info_dict
['title']
1860 if 'format' not in info_dict
:
1861 info_dict
['format'] = info_dict
['ext']
1863 reason
= self
._match
_entry
(info_dict
, incomplete
=False)
1864 if reason
is not None:
1865 self
.to_screen('[download] ' + reason
)
1868 self
._num
_downloads
+= 1
1870 info_dict
['_filename'] = filename
= self
.prepare_filename(info_dict
)
1873 self
.__forced
_printings
(info_dict
, filename
, incomplete
=False)
1875 if self
.params
.get('simulate', False):
1876 if self
.params
.get('force_write_download_archive', False):
1877 self
.record_download_archive(info_dict
)
1879 # Do nothing else if in simulate mode
1882 if filename
is None:
1885 def ensure_dir_exists(path
):
1887 dn
= os
.path
.dirname(path
)
1888 if dn
and not os
.path
.exists(dn
):
1891 except (OSError, IOError) as err
:
1892 self
.report_error('unable to create directory ' + error_to_compat_str(err
))
1895 if not ensure_dir_exists(sanitize_path(encodeFilename(filename
))):
1898 if self
.params
.get('writedescription', False):
1899 descfn
= replace_extension(filename
, 'description', info_dict
.get('ext'))
1900 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(descfn
)):
1901 self
.to_screen('[info] Video description is already present')
1902 elif info_dict
.get('description') is None:
1903 self
.report_warning('There\'s no description to write.')
1906 self
.to_screen('[info] Writing video description to: ' + descfn
)
1907 with io
.open(encodeFilename(descfn
), 'w', encoding
='utf-8') as descfile
:
1908 descfile
.write(info_dict
['description'])
1909 except (OSError, IOError):
1910 self
.report_error('Cannot write description file ' + descfn
)
1913 if self
.params
.get('writeannotations', False):
1914 annofn
= replace_extension(filename
, 'annotations.xml', info_dict
.get('ext'))
1915 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(annofn
)):
1916 self
.to_screen('[info] Video annotations are already present')
1917 elif not info_dict
.get('annotations'):
1918 self
.report_warning('There are no annotations to write.')
1921 self
.to_screen('[info] Writing video annotations to: ' + annofn
)
1922 with io
.open(encodeFilename(annofn
), 'w', encoding
='utf-8') as annofile
:
1923 annofile
.write(info_dict
['annotations'])
1924 except (KeyError, TypeError):
1925 self
.report_warning('There are no annotations to write.')
1926 except (OSError, IOError):
1927 self
.report_error('Cannot write annotations file: ' + annofn
)
1930 def dl(name
, info
, subtitle
=False):
1931 fd
= get_suitable_downloader(info
, self
.params
)(self
, self
.params
)
1932 for ph
in self
._progress
_hooks
:
1933 fd
.add_progress_hook(ph
)
1934 if self
.params
.get('verbose'):
1935 self
.to_screen('[debug] Invoking downloader on %r' % info
.get('url'))
1936 return fd
.download(name
, info
, subtitle
)
1938 subtitles_are_requested
= any([self
.params
.get('writesubtitles', False),
1939 self
.params
.get('writeautomaticsub')])
1941 if subtitles_are_requested
and info_dict
.get('requested_subtitles'):
1942 # subtitles download errors are already managed as troubles in relevant IE
1943 # that way it will silently go on when used with unsupporting IE
1944 subtitles
= info_dict
['requested_subtitles']
1945 # ie = self.get_info_extractor(info_dict['extractor_key'])
1946 for sub_lang
, sub_info
in subtitles
.items():
1947 sub_format
= sub_info
['ext']
1948 sub_filename
= subtitles_filename(filename
, sub_lang
, sub_format
, info_dict
.get('ext'))
1949 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(sub_filename
)):
1950 self
.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang
, sub_format
))
1952 self
.to_screen('[info] Writing video subtitles to: ' + sub_filename
)
1953 if sub_info
.get('data') is not None:
1955 # Use newline='' to prevent conversion of newline characters
1956 # See https://github.com/ytdl-org/youtube-dl/issues/10268
1957 with io
.open(encodeFilename(sub_filename
), 'w', encoding
='utf-8', newline
='') as subfile
:
1958 subfile
.write(sub_info
['data'])
1959 except (OSError, IOError):
1960 self
.report_error('Cannot write subtitles file ' + sub_filename
)
1964 dl(sub_filename
, sub_info
, subtitle
=True)
1966 if self.params.get('sleep_interval_subtitles', False):
1967 dl(sub_filename, sub_info)
1969 sub_data = ie._request_webpage(
1970 sub_info['url'], info_dict['id'], note=False).read()
1971 with io.open(encodeFilename(sub_filename), 'wb') as subfile:
1972 subfile.write(sub_data)
1974 except (ExtractorError
, IOError, OSError, ValueError, compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
1975 self
.report_warning('Unable to download subtitle for "%s": %s' %
1976 (sub_lang
, error_to_compat_str(err
)))
1979 if self
.params
.get('skip_download', False):
1980 if self
.params
.get('convertsubtitles', False):
1981 subconv
= FFmpegSubtitlesConvertorPP(self
, format
=self
.params
.get('convertsubtitles'))
1982 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
1984 os
.path
.splitext(filename
)[0]
1985 if filename_real_ext
== info_dict
['ext']
1987 afilename
= '%s.%s' % (filename_wo_ext
, self
.params
.get('convertsubtitles'))
1988 if subconv
.available
:
1989 info_dict
.setdefault('__postprocessors', [])
1990 # info_dict['__postprocessors'].append(subconv)
1991 if os
.path
.exists(encodeFilename(afilename
)):
1993 '[download] %s has already been downloaded and '
1994 'converted' % afilename
)
1997 self
.post_process(filename
, info_dict
)
1998 except (PostProcessingError
) as err
:
1999 self
.report_error('postprocessing: %s' % str(err
))
2002 if self
.params
.get('writeinfojson', False):
2003 infofn
= replace_extension(filename
, 'info.json', info_dict
.get('ext'))
2004 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(infofn
)):
2005 self
.to_screen('[info] Video description metadata is already present')
2007 self
.to_screen('[info] Writing video description metadata as JSON to: ' + infofn
)
2009 write_json_file(self
.filter_requested_info(info_dict
), infofn
)
2010 except (OSError, IOError):
2011 self
.report_error('Cannot write metadata to JSON file ' + infofn
)
2014 self
._write
_thumbnails
(info_dict
, filename
)
2016 # Write internet shortcut files
2017 url_link
= webloc_link
= desktop_link
= False
2018 if self
.params
.get('writelink', False):
2019 if sys
.platform
== "darwin": # macOS.
2021 elif sys
.platform
.startswith("linux"):
2023 else: # if sys.platform in ['win32', 'cygwin']:
2025 if self
.params
.get('writeurllink', False):
2027 if self
.params
.get('writewebloclink', False):
2029 if self
.params
.get('writedesktoplink', False):
2032 if url_link
or webloc_link
or desktop_link
:
2033 if 'webpage_url' not in info_dict
:
2034 self
.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information')
2036 ascii_url
= iri_to_uri(info_dict
['webpage_url'])
2038 def _write_link_file(extension
, template
, newline
, embed_filename
):
2039 linkfn
= replace_extension(filename
, extension
, info_dict
.get('ext'))
2040 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(linkfn
)):
2041 self
.to_screen('[info] Internet shortcut is already present')
2044 self
.to_screen('[info] Writing internet shortcut to: ' + linkfn
)
2045 with io
.open(encodeFilename(to_high_limit_path(linkfn
)), 'w', encoding
='utf-8', newline
=newline
) as linkfile
:
2046 template_vars
= {'url': ascii_url}
2048 template_vars
['filename'] = linkfn
[:-(len(extension
) + 1)]
2049 linkfile
.write(template
% template_vars
)
2050 except (OSError, IOError):
2051 self
.report_error('Cannot write internet shortcut ' + linkfn
)
2056 if not _write_link_file('url', DOT_URL_LINK_TEMPLATE
, '\r\n', embed_filename
=False):
2059 if not _write_link_file('webloc', DOT_WEBLOC_LINK_TEMPLATE
, '\n', embed_filename
=False):
2062 if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE
, '\n', embed_filename
=True):
2066 must_record_download_archive
= False
2067 if not self
.params
.get('skip_download', False):
2069 if info_dict
.get('requested_formats') is not None:
2072 merger
= FFmpegMergerPP(self
)
2073 if not merger
.available
:
2075 self
.report_warning('You have requested multiple '
2076 'formats but ffmpeg or avconv are not installed.'
2077 ' The formats won\'t be merged.')
2079 postprocessors
= [merger
]
2081 def compatible_formats(formats
):
2082 # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
2083 video_formats
= [format
for format
in formats
if format
.get('vcodec') != 'none']
2084 audio_formats
= [format
for format
in formats
if format
.get('acodec') != 'none']
2085 if len(video_formats
) > 2 or len(audio_formats
) > 2:
2089 exts
= set(format
.get('ext') for format
in formats
)
2091 set(('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma')),
2094 for ext_sets
in COMPATIBLE_EXTS
:
2095 if ext_sets
.issuperset(exts
):
2097 # TODO: Check acodec/vcodec
2100 filename_real_ext
= os
.path
.splitext(filename
)[1][1:]
2102 os
.path
.splitext(filename
)[0]
2103 if filename_real_ext
== info_dict
['ext']
2105 requested_formats
= info_dict
['requested_formats']
2106 if self
.params
.get('merge_output_format') is None and not compatible_formats(requested_formats
):
2107 info_dict
['ext'] = 'mkv'
2108 self
.report_warning(
2109 'Requested formats are incompatible for merge and will be merged into mkv.')
2110 # Ensure filename always has a correct extension for successful merge
2111 filename
= '%s.%s' % (filename_wo_ext
, info_dict
['ext'])
2112 if os
.path
.exists(encodeFilename(filename
)):
2114 '[download] %s has already been downloaded and '
2115 'merged' % filename
)
2117 for f
in requested_formats
:
2118 new_info
= dict(info_dict
)
2120 fname
= prepend_extension(
2121 self
.prepare_filename(new_info
),
2122 'f%s' % f
['format_id'], new_info
['ext'])
2123 if not ensure_dir_exists(fname
):
2125 downloaded
.append(fname
)
2126 partial_success
, real_download
= dl(fname
, new_info
)
2127 success
= success
and partial_success
2128 info_dict
['__postprocessors'] = postprocessors
2129 info_dict
['__files_to_merge'] = downloaded
2130 # Even if there were no downloads, it is being merged only now
2131 info_dict
['__real_download'] = True
2133 # Just a single file
2134 success
, real_download
= dl(filename
, info_dict
)
2135 info_dict
['__real_download'] = real_download
2136 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
2137 self
.report_error('unable to download video data: %s' % error_to_compat_str(err
))
2139 except (OSError, IOError) as err
:
2140 raise UnavailableVideoError(err
)
2141 except (ContentTooShortError
, ) as err
:
2142 self
.report_error('content too short (expected %s bytes and served %s)' % (err
.expected
, err
.downloaded
))
2145 if success
and filename
!= '-':
2147 fixup_policy
= self
.params
.get('fixup')
2148 if fixup_policy
is None:
2149 fixup_policy
= 'detect_or_warn'
2151 INSTALL_FFMPEG_MESSAGE
= 'Install ffmpeg or avconv to fix this automatically.'
2153 stretched_ratio
= info_dict
.get('stretched_ratio')
2154 if stretched_ratio
is not None and stretched_ratio
!= 1:
2155 if fixup_policy
== 'warn':
2156 self
.report_warning('%s: Non-uniform pixel ratio (%s)' % (
2157 info_dict
['id'], stretched_ratio
))
2158 elif fixup_policy
== 'detect_or_warn':
2159 stretched_pp
= FFmpegFixupStretchedPP(self
)
2160 if stretched_pp
.available
:
2161 info_dict
.setdefault('__postprocessors', [])
2162 info_dict
['__postprocessors'].append(stretched_pp
)
2164 self
.report_warning(
2165 '%s: Non-uniform pixel ratio (%s). %s'
2166 % (info_dict
['id'], stretched_ratio
, INSTALL_FFMPEG_MESSAGE
))
2168 assert fixup_policy
in ('ignore', 'never')
2170 if (info_dict
.get('requested_formats') is None
2171 and info_dict
.get('container') == 'm4a_dash'):
2172 if fixup_policy
== 'warn':
2173 self
.report_warning(
2174 '%s: writing DASH m4a. '
2175 'Only some players support this container.'
2177 elif fixup_policy
== 'detect_or_warn':
2178 fixup_pp
= FFmpegFixupM4aPP(self
)
2179 if fixup_pp
.available
:
2180 info_dict
.setdefault('__postprocessors', [])
2181 info_dict
['__postprocessors'].append(fixup_pp
)
2183 self
.report_warning(
2184 '%s: writing DASH m4a. '
2185 'Only some players support this container. %s'
2186 % (info_dict
['id'], INSTALL_FFMPEG_MESSAGE
))
2188 assert fixup_policy
in ('ignore', 'never')
2190 if (info_dict
.get('protocol') == 'm3u8_native'
2191 or info_dict
.get('protocol') == 'm3u8'
2192 and self
.params
.get('hls_prefer_native')):
2193 if fixup_policy
== 'warn':
2194 self
.report_warning('%s: malformed AAC bitstream detected.' % (
2196 elif fixup_policy
== 'detect_or_warn':
2197 fixup_pp
= FFmpegFixupM3u8PP(self
)
2198 if fixup_pp
.available
:
2199 info_dict
.setdefault('__postprocessors', [])
2200 info_dict
['__postprocessors'].append(fixup_pp
)
2202 self
.report_warning(
2203 '%s: malformed AAC bitstream detected. %s'
2204 % (info_dict
['id'], INSTALL_FFMPEG_MESSAGE
))
2206 assert fixup_policy
in ('ignore', 'never')
2209 self
.post_process(filename
, info_dict
)
2210 except (PostProcessingError
) as err
:
2211 self
.report_error('postprocessing: %s' % str(err
))
2214 for ph
in self
._post
_hooks
:
2216 except Exception as err
:
2217 self
.report_error('post hooks: %s' % str(err
))
2219 must_record_download_archive
= True
2221 if must_record_download_archive
or self
.params
.get('force_write_download_archive', False):
2222 self
.record_download_archive(info_dict
)
2223 max_downloads
= self
.params
.get('max_downloads')
2224 if max_downloads
is not None and self
._num
_downloads
>= int(max_downloads
):
2225 raise MaxDownloadsReached()
2227 def download(self
, url_list
):
2228 """Download a given list of URLs."""
2229 outtmpl
= self
.params
.get('outtmpl', DEFAULT_OUTTMPL
)
2230 if (len(url_list
) > 1
2232 and '%' not in outtmpl
2233 and self
.params
.get('max_downloads') != 1):
2234 raise SameFileError(outtmpl
)
2236 for url
in url_list
:
2238 # It also downloads the videos
2239 res
= self
.extract_info(
2240 url
, force_generic_extractor
=self
.params
.get('force_generic_extractor', False))
2241 except UnavailableVideoError
:
2242 self
.report_error('unable to download video')
2243 except MaxDownloadsReached
:
2244 self
.to_screen('[info] Maximum number of downloaded files reached.')
2247 if self
.params
.get('dump_single_json', False):
2248 self
.to_stdout(json
.dumps(res
))
2250 return self
._download
_retcode
2252 def download_with_info_file(self
, info_filename
):
2253 with contextlib
.closing(fileinput
.FileInput(
2254 [info_filename
], mode
='r',
2255 openhook
=fileinput
.hook_encoded('utf-8'))) as f
:
2256 # FileInput doesn't have a read method, we can't call json.load
2257 info
= self
.filter_requested_info(json
.loads('\n'.join(f
)))
2259 self
.process_ie_result(info
, download
=True)
2260 except DownloadError
:
2261 webpage_url
= info
.get('webpage_url')
2262 if webpage_url
is not None:
2263 self
.report_warning('The info failed to download, trying with "%s"' % webpage_url
)
2264 return self
.download([webpage_url
])
2267 return self
._download
_retcode
2270 def filter_requested_info(info_dict
):
2272 (k
, v
) for k
, v
in info_dict
.items()
2273 if k
not in ['requested_formats', 'requested_subtitles'])
2275 def post_process(self
, filename
, ie_info
):
2276 """Run all the postprocessors on the given file."""
2277 info
= dict(ie_info
)
2278 info
['filepath'] = filename
2280 if ie_info
.get('__postprocessors') is not None:
2281 pps_chain
.extend(ie_info
['__postprocessors'])
2282 pps_chain
.extend(self
._pps
)
2283 for pp
in pps_chain
:
2284 files_to_delete
= []
2286 files_to_delete
, info
= pp
.run(info
)
2287 except PostProcessingError
as e
:
2288 self
.report_error(e
.msg
)
2289 if files_to_delete
and not self
.params
.get('keepvideo', False):
2290 for old_filename
in set(files_to_delete
):
2291 self
.to_screen('Deleting original file %s (pass -k to keep)' % old_filename
)
2293 os
.remove(encodeFilename(old_filename
))
2294 except (IOError, OSError):
2295 self
.report_warning('Unable to remove downloaded original file')
2297 def _make_archive_id(self
, info_dict
):
2298 video_id
= info_dict
.get('id')
2301 # Future-proof against any change in case
2302 # and backwards compatibility with prior versions
2303 extractor
= info_dict
.get('extractor_key') or info_dict
.get('ie_key') # key in a playlist
2304 if extractor
is None:
2305 url
= str_or_none(info_dict
.get('url'))
2308 # Try to find matching extractor for the URL and take its ie_key
2309 for ie
in self
._ies
:
2310 if ie
.suitable(url
):
2311 extractor
= ie
.ie_key()
2315 return extractor
.lower() + ' ' + video_id
2317 def in_download_archive(self
, info_dict
):
2318 fn
= self
.params
.get('download_archive')
2322 vid_id
= self
._make
_archive
_id
(info_dict
)
2324 return False # Incomplete video information
2326 return vid_id
in self
.archive
2328 def record_download_archive(self
, info_dict
):
2329 fn
= self
.params
.get('download_archive')
2332 vid_id
= self
._make
_archive
_id
(info_dict
)
2334 with locked_file(fn
, 'a', encoding
='utf-8') as archive_file
:
2335 archive_file
.write(vid_id
+ '\n')
2336 self
.archive
.add(vid_id
)
2339 def format_resolution(format
, default
='unknown'):
2340 if format
.get('vcodec') == 'none':
2342 if format
.get('resolution') is not None:
2343 return format
['resolution']
2344 if format
.get('height') is not None:
2345 if format
.get('width') is not None:
2346 res
= '%sx%s' % (format
['width'], format
['height'])
2348 res
= '%sp' % format
['height']
2349 elif format
.get('width') is not None:
2350 res
= '%dx?' % format
['width']
2355 def _format_note(self
, fdict
):
2357 if fdict
.get('ext') in ['f4f', 'f4m']:
2358 res
+= '(unsupported) '
2359 if fdict
.get('language'):
2362 res
+= '[%s] ' % fdict
['language']
2363 if fdict
.get('format_note') is not None:
2364 res
+= fdict
['format_note'] + ' '
2365 if fdict
.get('tbr') is not None:
2366 res
+= '%4dk ' % fdict
['tbr']
2367 if fdict
.get('container') is not None:
2370 res
+= '%s container' % fdict
['container']
2371 if (fdict
.get('vcodec') is not None
2372 and fdict
.get('vcodec') != 'none'):
2375 res
+= fdict
['vcodec']
2376 if fdict
.get('vbr') is not None:
2378 elif fdict
.get('vbr') is not None and fdict
.get('abr') is not None:
2380 if fdict
.get('vbr') is not None:
2381 res
+= '%4dk' % fdict
['vbr']
2382 if fdict
.get('fps') is not None:
2385 res
+= '%sfps' % fdict
['fps']
2386 if fdict
.get('acodec') is not None:
2389 if fdict
['acodec'] == 'none':
2392 res
+= '%-5s' % fdict
['acodec']
2393 elif fdict
.get('abr') is not None:
2397 if fdict
.get('abr') is not None:
2398 res
+= '@%3dk' % fdict
['abr']
2399 if fdict
.get('asr') is not None:
2400 res
+= ' (%5dHz)' % fdict
['asr']
2401 if fdict
.get('filesize') is not None:
2404 res
+= format_bytes(fdict
['filesize'])
2405 elif fdict
.get('filesize_approx') is not None:
2408 res
+= '~' + format_bytes(fdict
['filesize_approx'])
2411 def _format_note_table(self
, f
):
2412 def join_fields(*vargs
):
2413 return ', '.join((val
for val
in vargs
if val
!= ''))
2416 'UNSUPPORTED' if f
.get('ext') in ('f4f', 'f4m') else '',
2417 format_field(f
, 'language', '[%s]'),
2418 format_field(f
, 'format_note'),
2419 format_field(f
, 'container', ignore
=(None, f
.get('ext'))),
2420 format_field(f
, 'asr', '%5dHz'))
2422 def list_formats(self
, info_dict
):
2423 formats
= info_dict
.get('formats', [info_dict
])
2424 new_format
= self
.params
.get('listformats_table', False)
2428 format_field(f
, 'format_id'),
2429 format_field(f
, 'ext'),
2430 self
.format_resolution(f
),
2431 format_field(f
, 'fps', '%d'),
2433 format_field(f
, 'filesize', ' %s', func
=format_bytes
) + format_field(f
, 'filesize_approx', '~%s', func
=format_bytes
),
2434 format_field(f
, 'tbr', '%4dk'),
2435 f
.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n"),
2437 format_field(f
, 'vcodec', default
='unknown').replace('none', ''),
2438 format_field(f
, 'vbr', '%4dk'),
2439 format_field(f
, 'acodec', default
='unknown').replace('none', ''),
2440 format_field(f
, 'abr', '%3dk'),
2441 format_field(f
, 'asr', '%5dHz'),
2442 self
._format
_note
_table
(f
)]
2444 if f
.get('preference') is None or f
['preference'] >= -1000]
2445 header_line
= ['ID', 'EXT', 'RESOLUTION', 'FPS', '|', ' FILESIZE', ' TBR', 'PROTO',
2446 '|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'NOTE']
2450 format_field(f
, 'format_id'),
2451 format_field(f
, 'ext'),
2452 self
.format_resolution(f
),
2453 self
._format
_note
(f
)]
2455 if f
.get('preference') is None or f
['preference'] >= -1000]
2456 header_line
= ['format code', 'extension', 'resolution', 'note']
2458 # if len(formats) > 1:
2459 # table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2461 '[info] Available formats for %s:\n%s' % (info_dict
['id'], render_table(
2465 extraGap
=(0 if new_format
else 1),
2466 hideEmpty
=new_format
)))
2468 def list_thumbnails(self
, info_dict
):
2469 thumbnails
= info_dict
.get('thumbnails')
2471 self
.to_screen('[info] No thumbnails present for %s' % info_dict
['id'])
2475 '[info] Thumbnails for %s:' % info_dict
['id'])
2476 self
.to_screen(render_table(
2477 ['ID', 'width', 'height', 'URL'],
2478 [[t
['id'], t
.get('width', 'unknown'), t
.get('height', 'unknown'), t
['url']] for t
in thumbnails
]))
2480 def list_subtitles(self
, video_id
, subtitles
, name
='subtitles'):
2482 self
.to_screen('%s has no %s' % (video_id
, name
))
2485 'Available %s for %s:' % (name
, video_id
))
2486 self
.to_screen(render_table(
2487 ['Language', 'formats'],
2488 [[lang
, ', '.join(f
['ext'] for f
in reversed(formats
))]
2489 for lang
, formats
in subtitles
.items()]))
2491 def urlopen(self
, req
):
2492 """ Start an HTTP download """
2493 if isinstance(req
, compat_basestring
):
2494 req
= sanitized_Request(req
)
2495 return self
._opener
.open(req
, timeout
=self
._socket
_timeout
)
2497 def print_debug_header(self
):
2498 if not self
.params
.get('verbose'):
2501 if type('') is not compat_str
:
2502 # Python 2.6 on SLES11 SP1 (https://github.com/ytdl-org/youtube-dl/issues/3326)
2503 self
.report_warning(
2504 'Your Python is broken! Update to a newer and supported version')
2506 stdout_encoding
= getattr(
2507 sys
.stdout
, 'encoding', 'missing (%s)' % type(sys
.stdout
).__name
__)
2509 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2510 locale
.getpreferredencoding(),
2511 sys
.getfilesystemencoding(),
2513 self
.get_encoding()))
2514 write_string(encoding_str
, encoding
=None)
2516 self
._write
_string
('[debug] youtube-dlc version ' + __version__
+ '\n')
2518 self
._write
_string
('[debug] Lazy loading extractors enabled' + '\n')
2520 sp
= subprocess
.Popen(
2521 ['git', 'rev-parse', '--short', 'HEAD'],
2522 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
,
2523 cwd
=os
.path
.dirname(os
.path
.abspath(__file__
)))
2524 out
, err
= sp
.communicate()
2525 out
= out
.decode().strip()
2526 if re
.match('[0-9a-f]+', out
):
2527 self
._write
_string
('[debug] Git HEAD: ' + out
+ '\n')
2534 def python_implementation():
2535 impl_name
= platform
.python_implementation()
2536 if impl_name
== 'PyPy' and hasattr(sys
, 'pypy_version_info'):
2537 return impl_name
+ ' version %d.%d.%d' % sys
.pypy_version_info
[:3]
2540 self
._write
_string
('[debug] Python version %s (%s) - %s\n' % (
2541 platform
.python_version(), python_implementation(),
2544 exe_versions
= FFmpegPostProcessor
.get_versions(self
)
2545 exe_versions
['rtmpdump'] = rtmpdump_version()
2546 exe_versions
['phantomjs'] = PhantomJSwrapper
._version
()
2547 exe_str
= ', '.join(
2549 for exe
, v
in sorted(exe_versions
.items())
2554 self
._write
_string
('[debug] exe versions: %s\n' % exe_str
)
2557 for handler
in self
._opener
.handlers
:
2558 if hasattr(handler
, 'proxies'):
2559 proxy_map
.update(handler
.proxies
)
2560 self
._write
_string
('[debug] Proxy map: ' + compat_str(proxy_map
) + '\n')
2562 if self
.params
.get('call_home', False):
2563 ipaddr
= self
.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2564 self
._write
_string
('[debug] Public IP address: %s\n' % ipaddr
)
2565 latest_version
= self
.urlopen(
2566 'https://yt-dl.org/latest/version').read().decode('utf-8')
2567 if version_tuple(latest_version
) > version_tuple(__version__
):
2568 self
.report_warning(
2569 'You are using an outdated version (newest version: %s)! '
2570 'See https://yt-dl.org/update if you need help updating.' %
2573 def _setup_opener(self
):
2574 timeout_val
= self
.params
.get('socket_timeout')
2575 self
._socket
_timeout
= 600 if timeout_val
is None else float(timeout_val
)
2577 opts_cookiefile
= self
.params
.get('cookiefile')
2578 opts_proxy
= self
.params
.get('proxy')
2580 if opts_cookiefile
is None:
2581 self
.cookiejar
= compat_cookiejar
.CookieJar()
2583 opts_cookiefile
= expand_path(opts_cookiefile
)
2584 self
.cookiejar
= YoutubeDLCookieJar(opts_cookiefile
)
2585 if os
.access(opts_cookiefile
, os
.R_OK
):
2586 self
.cookiejar
.load(ignore_discard
=True, ignore_expires
=True)
2588 cookie_processor
= YoutubeDLCookieProcessor(self
.cookiejar
)
2589 if opts_proxy
is not None:
2590 if opts_proxy
== '':
2593 proxies
= {'http': opts_proxy, 'https': opts_proxy}
2595 proxies
= compat_urllib_request
.getproxies()
2596 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
2597 if 'http' in proxies
and 'https' not in proxies
:
2598 proxies
['https'] = proxies
['http']
2599 proxy_handler
= PerRequestProxyHandler(proxies
)
2601 debuglevel
= 1 if self
.params
.get('debug_printtraffic') else 0
2602 https_handler
= make_HTTPS_handler(self
.params
, debuglevel
=debuglevel
)
2603 ydlh
= YoutubeDLHandler(self
.params
, debuglevel
=debuglevel
)
2604 redirect_handler
= YoutubeDLRedirectHandler()
2605 data_handler
= compat_urllib_request_DataHandler()
2607 # When passing our own FileHandler instance, build_opener won't add the
2608 # default FileHandler and allows us to disable the file protocol, which
2609 # can be used for malicious purposes (see
2610 # https://github.com/ytdl-org/youtube-dl/issues/8227)
2611 file_handler
= compat_urllib_request
.FileHandler()
2613 def file_open(*args
, **kwargs
):
2614 raise compat_urllib_error
.URLError('file:// scheme is explicitly disabled in youtube-dlc for security reasons')
2615 file_handler
.file_open
= file_open
2617 opener
= compat_urllib_request
.build_opener(
2618 proxy_handler
, https_handler
, cookie_processor
, ydlh
, redirect_handler
, data_handler
, file_handler
)
2620 # Delete the default user-agent header, which would otherwise apply in
2621 # cases where our custom HTTP handler doesn't come into play
2622 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
2623 opener
.addheaders
= []
2624 self
._opener
= opener
2626 def encode(self
, s
):
2627 if isinstance(s
, bytes):
2628 return s
# Already encoded
2631 return s
.encode(self
.get_encoding())
2632 except UnicodeEncodeError as err
:
2633 err
.reason
= err
.reason
+ '. Check your system encoding configuration or use the --encoding option.'
2636 def get_encoding(self
):
2637 encoding
= self
.params
.get('encoding')
2638 if encoding
is None:
2639 encoding
= preferredencoding()
2642 def _write_thumbnails(self
, info_dict
, filename
):
2643 if self
.params
.get('writethumbnail', False):
2644 thumbnails
= info_dict
.get('thumbnails')
2646 thumbnails
= [thumbnails
[-1]]
2647 elif self
.params
.get('write_all_thumbnails', False):
2648 thumbnails
= info_dict
.get('thumbnails')
2653 # No thumbnails present, so return immediately
2656 for t
in thumbnails
:
2657 thumb_ext
= determine_ext(t
['url'], 'jpg')
2658 suffix
= '_%s' % t
['id'] if len(thumbnails
) > 1 else ''
2659 thumb_display_id
= '%s ' % t
['id'] if len(thumbnails
) > 1 else ''
2660 t
['filename'] = thumb_filename
= replace_extension(filename
+ suffix
, thumb_ext
, info_dict
.get('ext'))
2662 if self
.params
.get('nooverwrites', False) and os
.path
.exists(encodeFilename(thumb_filename
)):
2663 self
.to_screen('[%s] %s: Thumbnail %sis already present' %
2664 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
2666 self
.to_screen('[%s] %s: Downloading thumbnail %s...' %
2667 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
))
2669 uf
= self
.urlopen(t
['url'])
2670 with open(encodeFilename(thumb_filename
), 'wb') as thumbf
:
2671 shutil
.copyfileobj(uf
, thumbf
)
2672 self
.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2673 (info_dict
['extractor'], info_dict
['id'], thumb_display_id
, thumb_filename
))
2674 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
2675 self
.report_warning('Unable to download thumbnail "%s": %s' %
2676 (t
['url'], error_to_compat_str(err
)))