import base64
import datetime
import hashlib
+import itertools
import json
import netrc
import os
* width Width of the video, if known
* height Height of the video, if known
* resolution Textual description of width and height
+ * dynamic_range The dynamic range of the video. One of:
+ "SDR" (None), "HDR10", "HDR10+, "HDR12", "HLG, "DV"
* tbr Average bitrate of audio and video in KBit/s
* abr Average audio bitrate in KBit/s
* acodec Name of the audio codec in use
* "resolution" (optional, string "{width}x{height}",
deprecated)
* "filesize" (optional, int)
- * "_test_url" (optional, bool) - If true, test the URL
thumbnail: Full URL to a video thumbnail image.
description: Full video description.
uploader: Full name of the video uploader.
_LOGIN_HINTS = {
'any': 'Use --cookies, --username and --password or --netrc to provide account credentials',
'cookies': (
- 'Use --cookies for the authentication. '
- 'See https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl for how to pass cookies'),
+ 'Use --cookies-from-browser or --cookies for the authentication. '
+ 'See https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl for how to manually pass cookies'),
'password': 'Use --username and --password or --netrc to provide account credentials',
}
def __init__(self, downloader=None):
- """Constructor. Receives an optional downloader."""
+ """Constructor. Receives an optional downloader (a YoutubeDL instance).
+ If a downloader is not passed during initialization,
+ it must be set using "set_downloader()" before "extract()" is called"""
self._ready = False
self._x_forwarded_for_ip = None
self._printed_messages = set()
See _download_webpage docstring for arguments specification.
"""
if not self._downloader._first_webpage_request:
- sleep_interval = float_or_none(self.get_param('sleep_interval_requests')) or 0
+ sleep_interval = self.get_param('sleep_interval_requests') or 0
if sleep_interval > 0:
self.to_screen('Sleeping %s seconds ...' % sleep_interval)
time.sleep(sleep_interval)
# Methods for following #608
@staticmethod
- def url_result(url, ie=None, video_id=None, video_title=None):
+ def url_result(url, ie=None, video_id=None, video_title=None, **kwargs):
"""Returns a URL that points to a page that should be processed"""
# TODO: ie should be the class used for getting the info
video_info = {'_type': 'url',
'url': url,
'ie_key': ie}
+ video_info.update(kwargs)
if video_id is not None:
video_info['id'] = video_id
if video_title is not None:
if mobj:
break
- if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
- _name = '\033[0;34m%s\033[0m' % name
- else:
- _name = name
+ _name = self._downloader._color_text(name, 'blue')
if mobj:
if group is None:
regex = r' *((?P<reverse>\+)?(?P<field>[a-zA-Z0-9_]+)((?P<separator>[~:])(?P<limit>.*?))?)? *$'
default = ('hidden', 'aud_or_vid', 'hasvid', 'ie_pref', 'lang', 'quality',
- 'res', 'fps', 'codec:vp9.2', 'size', 'br', 'asr',
+ 'res', 'fps', 'hdr:12', 'codec:vp9.2', 'size', 'br', 'asr',
'proto', 'ext', 'hasaud', 'source', 'format_id') # These must not be aliases
ytdl_default = ('hasaud', 'lang', 'quality', 'tbr', 'filesize', 'vbr',
'height', 'width', 'proto', 'vext', 'abr', 'aext',
'order': ['av0?1', 'vp0?9.2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']},
'acodec': {'type': 'ordered', 'regex': True,
'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e?a?c-?3', 'dts', '', None, 'none']},
+ 'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range',
+ 'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]},
'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol',
'order': ['(ht|f)tps', '(ht|f)tp$', 'm3u8.+', '.*dash', 'ws|websocket', '', 'mms|rtsp', 'none', 'f4']},
'vext': {'type': 'ordered', 'field': 'video_ext',
has_multiple_limits = has_limit and has_multiple_fields and not self._get_field_setting(field, 'same_limit')
fields = self._get_field_setting(field, 'field') if has_multiple_fields else (field,)
- limits = limit_text.split(":") if has_multiple_limits else (limit_text,) if has_limit else tuple()
+ limits = limit_text.split(':') if has_multiple_limits else (limit_text,) if has_limit else tuple()
limit_count = len(limits)
for (i, f) in enumerate(fields):
add_item(f, reverse, closest,
if format.get('vbr') is not None and format.get('abr') is not None:
format['tbr'] = format.get('vbr', 0) + format.get('abr', 0)
else:
- if format.get('vcodec') != "none" and format.get('vbr') is None:
+ if format.get('vcodec') != 'none' and format.get('vbr') is None:
format['vbr'] = format.get('tbr') - format.get('abr', 0)
- if format.get('acodec') != "none" and format.get('abr') is None:
+ if format.get('acodec') != 'none' and format.get('abr') is None:
format['abr'] = format.get('tbr') - format.get('vbr', 0)
return tuple(self._calculate_field_preference(format, field) for field in self._order)
'format_note': 'Quality selection URL',
}
+ def _report_ignoring_subs(self, name):
+ self.report_warning(bug_reports_message(
+ f'Ignoring subtitle tracks found in the {name} manifest; '
+ 'if any subtitle tracks are missing,'
+ ), only_once=True)
+
def _extract_m3u8_formats(self, *args, **kwargs):
fmts, subs = self._extract_m3u8_formats_and_subtitles(*args, **kwargs)
if subs:
- self.report_warning(bug_reports_message(
- "Ignoring subtitle tracks found in the HLS manifest; "
- "if any subtitle tracks are missing,"
- ), only_once=True)
+ self._report_ignoring_subs('HLS')
return fmts
def _extract_m3u8_formats_and_subtitles(
if '#EXT-X-FAXS-CM:' in m3u8_doc: # Adobe Flash Access
return formats, subtitles
- has_drm = re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc)
+ has_drm = re.search(r'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://', m3u8_doc)
def format_url(url):
return url if re.match(r'^https?://', url) else compat_urlparse.urljoin(m3u8_url, url)
def _extract_smil_formats(self, *args, **kwargs):
fmts, subs = self._extract_smil_formats_and_subtitles(*args, **kwargs)
if subs:
- self.report_warning(bug_reports_message(
- "Ignoring subtitle tracks found in the SMIL manifest; "
- "if any subtitle tracks are missing,"
- ), only_once=True)
+ self._report_ignoring_subs('SMIL')
return fmts
def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
def _extract_mpd_formats(self, *args, **kwargs):
fmts, subs = self._extract_mpd_formats_and_subtitles(*args, **kwargs)
if subs:
- self.report_warning(bug_reports_message(
- "Ignoring subtitle tracks found in the DASH manifest; "
- "if any subtitle tracks are missing,"
- ), only_once=True)
+ self._report_ignoring_subs('DASH')
return fmts
def _extract_mpd_formats_and_subtitles(
def _parse_mpd_formats(self, *args, **kwargs):
fmts, subs = self._parse_mpd_formats_and_subtitles(*args, **kwargs)
if subs:
- self.report_warning(bug_reports_message(
- "Ignoring subtitle tracks found in the DASH manifest; "
- "if any subtitle tracks are missing,"
- ), only_once=True)
+ self._report_ignoring_subs('DASH')
return fmts
def _parse_mpd_formats_and_subtitles(
content_type = mime_type
elif codecs.split('.')[0] == 'stpp':
content_type = 'text'
+ elif mimetype2ext(mime_type) in ('tt', 'dfxp', 'ttml', 'xml', 'json'):
+ content_type = 'text'
else:
self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
continue
def _extract_ism_formats(self, *args, **kwargs):
fmts, subs = self._extract_ism_formats_and_subtitles(*args, **kwargs)
if subs:
- self.report_warning(bug_reports_message(
- "Ignoring subtitle tracks found in the ISM manifest; "
- "if any subtitle tracks are missing,"
- ))
+ self._report_ignoring_subs('ISM')
return fmts
def _extract_ism_formats_and_subtitles(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
def _extract_akamai_formats(self, *args, **kwargs):
fmts, subs = self._extract_akamai_formats_and_subtitles(*args, **kwargs)
if subs:
- self.report_warning(bug_reports_message(
- "Ignoring subtitle tracks found in the manifests; "
- "if any subtitle tracks are missing,"
- ))
+ self._report_ignoring_subs('akamai')
return fmts
def _extract_akamai_formats_and_subtitles(self, manifest_url, video_id, hosts={}):
def _get_subtitles(self, *args, **kwargs):
raise NotImplementedError('This method must be implemented by subclasses')
+ def extract_comments(self, *args, **kwargs):
+ if not self.get_param('getcomments'):
+ return None
+ generator = self._get_comments(*args, **kwargs)
+
+ def extractor():
+ comments = []
+ try:
+ while True:
+ comments.append(next(generator))
+ except KeyboardInterrupt:
+ interrupted = True
+ self.to_screen('Interrupted by user')
+ except StopIteration:
+ interrupted = False
+ comment_count = len(comments)
+ self.to_screen(f'Extracted {comment_count} comments')
+ return {
+ 'comments': comments,
+ 'comment_count': None if interrupted else comment_count
+ }
+ return extractor
+
+ def _get_comments(self, *args, **kwargs):
+ raise NotImplementedError('This method must be implemented by subclasses')
+
@staticmethod
def _merge_subtitle_items(subtitle_list1, subtitle_list2):
""" Merge subtitle items for one language. Items with duplicated URLs
return self._get_n_results(query, n)
def _get_n_results(self, query, n):
- """Get a specified number of results for a query"""
+ """Get a specified number of results for a query.
+ Either this function or _search_results must be overridden by subclasses """
+ return self.playlist_result(
+ itertools.islice(self._search_results(query), 0, None if n == float('inf') else n),
+ query, query)
+
+ def _search_results(self, query):
+ """Returns an iterator of search results"""
raise NotImplementedError('This method must be implemented by subclasses')
@property
projects / yt-dlp.git / blobdiff