]>
jfr.im git - yt-dlp.git/blob - youtube_dl/extractor/common.py
7 import xml
.etree
.ElementTree
23 class InfoExtractor(object):
24 """Information Extractor class.
26 Information extractors are the classes that, given a URL, extract
27 information about the video (or videos) the URL refers to. This
28 information includes the real video URL, the video title, author and
29 others. The information is stored in a dictionary which is then
30 passed to the FileDownloader. The FileDownloader processes this
31 information possibly downloading the video to the file system, among
32 other possible outcomes.
34 The dictionaries must include the following fields:
37 title: Video title, unescaped.
39 ext: Video filename extension.
41 Instead of url and ext, formats can also specified.
43 The following fields are optional:
45 format: The video format, defaults to ext (used for --get-format)
46 thumbnails: A list of dictionaries (with the entries "resolution" and
47 "url") for the varying thumbnails
48 thumbnail: Full URL to a video thumbnail image.
49 description: One-line video description.
50 uploader: Full name of the video uploader.
51 upload_date: Video upload date (YYYYMMDD).
52 uploader_id: Nickname or id of the video uploader.
53 location: Physical location of the video.
54 player_url: SWF Player URL (used for rtmpdump).
55 subtitles: The subtitle file contents as a dictionary in the format
56 {language: subtitles}.
57 duration: Length of the video in seconds, as an integer.
58 view_count: How many users have watched the video on the platform.
59 like_count: Number of positive ratings of the video
60 dislike_count: Number of negative ratings of the video
61 comment_count: Number of comments on the video
62 urlhandle: [internal] The urlHandle to be used to download the file,
63 like returned by urllib.request.urlopen
64 age_limit: Age restriction for the video, as an integer (years)
65 formats: A list of dictionaries for each format available, it must
66 be ordered from worst to best quality. Potential fields:
67 * url Mandatory. The URL of the video file
68 * ext Will be calculated from url if missing
69 * format A human-readable description of the format
70 ("mp4 container with h264/opus").
71 Calculated from the format_id, width, height.
72 and format_note fields if missing.
73 * format_id A short description of the format
74 ("mp4_h264_opus" or "19")
75 * format_note Additional info about the format
76 ("3D" or "DASH video")
77 * width Width of the video, if known
78 * height Height of the video, if known
79 * abr Average audio bitrate in KBit/s
80 * acodec Name of the audio codec in use
81 * vbr Average video bitrate in KBit/s
82 * vcodec Name of the video codec in use
83 * filesize The number of bytes, if known in advance
84 webpage_url: The url to the video webpage, if given to youtube-dl it
85 should allow to get the same result again. (It will be set
86 by YoutubeDL if it's missing)
88 Unless mentioned otherwise, the fields should be Unicode strings.
90 Subclasses of this one should re-define the _real_initialize() and
91 _real_extract() methods and define a _VALID_URL regexp.
92 Probably, they should also be added to the list of extractors.
94 _real_extract() must return a *list* of information dictionaries as
97 Finally, the _WORKING attribute should be set to False for broken IEs
98 in order to warn the users and skip the tests.
105 def __init__(self
, downloader
=None):
106 """Constructor. Receives an optional downloader."""
108 self
.set_downloader(downloader
)
111 def suitable(cls
, url
):
112 """Receives a URL and returns True if suitable for this IE."""
114 # This does not use has/getattr intentionally - we want to know whether
115 # we have cached the regexp for *this* class, whereas getattr would also
116 # match the superclass
117 if '_VALID_URL_RE' not in cls
.__dict
__:
118 cls
._VALID
_URL
_RE
= re
.compile(cls
._VALID
_URL
)
119 return cls
._VALID
_URL
_RE
.match(url
) is not None
123 """Getter method for _WORKING."""
126 def initialize(self
):
127 """Initializes an instance (authentication, etc)."""
129 self
._real
_initialize
()
132 def extract(self
, url
):
133 """Extracts URL information and returns it in list of dicts."""
135 return self
._real
_extract
(url
)
137 def set_downloader(self
, downloader
):
138 """Sets the downloader for this IE."""
139 self
._downloader
= downloader
141 def _real_initialize(self
):
142 """Real initialization process. Redefine in subclasses."""
145 def _real_extract(self
, url
):
146 """Real extraction process. Redefine in subclasses."""
151 """A string for getting the InfoExtractor with get_info_extractor"""
152 return cls
.__name
__[:-2]
156 return type(self
).__name
__[:-2]
158 def _request_webpage(self
, url_or_request
, video_id
, note
=None, errnote
=None, fatal
=True):
159 """ Returns the response handle """
161 self
.report_download_webpage(video_id
)
162 elif note
is not False:
164 self
.to_screen(u
'%s' % (note
,))
166 self
.to_screen(u
'%s: %s' % (video_id
, note
))
168 return self
._downloader
.urlopen(url_or_request
)
169 except (compat_urllib_error
.URLError
, compat_http_client
.HTTPException
, socket
.error
) as err
:
171 errnote
= u
'Unable to download webpage'
172 errmsg
= u
'%s: %s' % (errnote
, compat_str(err
))
174 raise ExtractorError(errmsg
, sys
.exc_info()[2], cause
=err
)
176 self
._downloader
.report_warning(errmsg
)
179 def _download_webpage_handle(self
, url_or_request
, video_id
, note
=None, errnote
=None, fatal
=True):
180 """ Returns a tuple (page content as string, URL handle) """
182 # Strip hashes from the URL (#1038)
183 if isinstance(url_or_request
, (compat_str
, str)):
184 url_or_request
= url_or_request
.partition('#')[0]
186 urlh
= self
._request
_webpage
(url_or_request
, video_id
, note
, errnote
, fatal
)
190 content_type
= urlh
.headers
.get('Content-Type', '')
191 webpage_bytes
= urlh
.read()
192 m
= re
.match(r
'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type
)
194 encoding
= m
.group(1)
196 m
= re
.search(br
'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
197 webpage_bytes[:1024])
199 encoding = m.group(1).decode('ascii')
202 if self._downloader.params.get('dump_intermediate_pages', False):
204 url = url_or_request.get_full_url()
205 except AttributeError:
207 self.to_screen(u'Dumping request to ' + url)
208 dump = base64.b64encode(webpage_bytes).decode('ascii')
209 self._downloader.to_screen(dump)
210 if self._downloader.params.get('write_pages', False):
212 url = url_or_request.get_full_url()
213 except AttributeError:
215 raw_filename = ('%s_%s.dump' % (video_id, url))
216 filename = sanitize_filename(raw_filename, restricted=True)
217 self.to_screen(u'Saving request to ' + filename)
218 with open(filename, 'wb') as outf:
219 outf.write(webpage_bytes)
221 content = webpage_bytes.decode(encoding, 'replace')
222 return (content, urlh)
224 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
225 """ Returns the data of the page as a string """
226 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
233 def _download_xml(self, url_or_request, video_id,
234 note=u'Downloading XML', errnote=u'Unable to download XML',
235 transform_source=None):
236 """Return the xml as an xml.etree.ElementTree.Element"""
237 xml_string = self._download_webpage(url_or_request, video_id, note, errnote)
239 xml_string = transform_source(xml_string)
240 return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
242 def to_screen(self, msg):
243 """Print msg to screen, prefixing it with '[ie_name]'"""
244 self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg))
246 def report_extraction(self, id_or_name):
247 """Report information extraction."""
248 self.to_screen(u'%s: Extracting information' % id_or_name)
250 def report_download_webpage(self, video_id):
251 """Report webpage download."""
252 self.to_screen(u'%s: Downloading webpage' % video_id)
254 def report_age_confirmation(self):
255 """Report attempt to confirm age."""
256 self.to_screen(u'Confirming age')
258 def report_login(self):
259 """Report attempt to log in."""
260 self.to_screen(u'Logging in')
262 #Methods for following #608
263 def url_result(self, url, ie=None, video_id=None):
264 """Returns a url that points to a page that should be processed"""
265 #TODO: ie should be the class used for getting the info
266 video_info = {'_type': 'url',
269 if video_id is not None:
270 video_info['id'] = video_id
272 def playlist_result(self, entries, playlist_id=None, playlist_title=None):
273 """Returns a playlist"""
274 video_info = {'_type': 'playlist',
277 video_info['id'] = playlist_id
279 video_info['title'] = playlist_title
282 def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):
284 Perform a regex search on the given string, using a single or a list of
285 patterns returning the first matching group.
286 In case of failure return a default value or raise a WARNING or a
287 RegexNotFoundError, depending on fatal, specifying the field name.
289 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
290 mobj = re.search(pattern, string, flags)
293 mobj = re.search(p, string, flags)
296 if sys.stderr.isatty() and os.name != 'nt':
297 _name = u'\033[0;34m%s\033[0m' % name
302 # return the first matching group
303 return next(g for g in mobj.groups() if g is not None)
304 elif default is not None:
307 raise RegexNotFoundError(u'Unable to extract %s' % _name)
309 self._downloader.report_warning(u'unable to extract %s; '
310 u'please report this issue on http://yt-dl.org/bug' % _name)
313 def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):
315 Like _search_regex, but strips HTML tags and unescapes entities.
317 res = self._search_regex(pattern, string, name, default, fatal, flags)
319 return clean_html(res).strip()
323 def _get_login_info(self):
325 Get the the login info as (username, password)
326 It will look in the netrc file using the _NETRC_MACHINE value
327 If there's no info available, return (None, None)
329 if self._downloader is None:
334 downloader_params = self._downloader.params
336 # Attempt to use provided username and password or .netrc data
337 if downloader_params.get('username', None) is not None:
338 username = downloader_params['username']
339 password = downloader_params['password']
340 elif downloader_params.get('usenetrc', False):
342 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
347 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
348 except (IOError, netrc.NetrcParseError) as err:
349 self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
351 return (username, password)
353 # Helper functions for extracting OpenGraph info
355 def _og_regexes(prop):
356 content_re = r'content=(?:"([^
>]+?
)"|\'(.+?)\')'
357 property_re = r'property=[\'"]og
:%s[\'"]' % re.escape(prop)
358 template = r'<meta[^>]+?%s[^>]+?%s'
360 template % (property_re, content_re),
361 template % (content_re, property_re),
364 def _og_search_property(self, prop, html, name=None, **kargs):
366 name = 'OpenGraph %s' % prop
367 escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
370 return unescapeHTML(escaped)
372 def _og_search_thumbnail(self, html, **kargs):
373 return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)
375 def _og_search_description(self, html, **kargs):
376 return self._og_search_property('description', html, fatal=False, **kargs)
378 def _og_search_title(self, html, **kargs):
379 return self._og_search_property('title', html, **kargs)
381 def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
382 regexes = self._og_regexes('video')
383 if secure: regexes = self._og_regexes('video:secure_url') + regexes
384 return self._html_search_regex(regexes, html, name, **kargs)
386 def _html_search_meta(self, name, html, display_name=None):
387 if display_name is None:
389 return self._html_search_regex(
391 (?=[^>]+(?:itemprop|name|property)=["\']%s["\'])
392 [^>]+content=["\']([^
"\']+)["\']''' % re.escape(name),
393 html, display_name, fatal=False)
395 def _dc_search_uploader(self, html):
396 return self._html_search_meta('dc.creator', html, 'uploader')
398 def _rta_search(self, html):
399 # See http://www.rtalabel.org/index.php?content=howtofaq#single
400 if re.search(r'(?ix)<meta\s+name="rating"\s+'
401 r' content="RTA-5042-1996-1400-1577-RTA"',
406 def _media_rating_search(self, html):
407 # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
408 rating = self._html_search_meta('rating', html)
420 return RATING_TABLE.get(rating.lower(), None)
424 class SearchInfoExtractor(InfoExtractor):
426 Base class for paged search queries extractors.
427 They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
428 Instances should define _SEARCH_KEY and _MAX_RESULTS.
432 def _make_valid_url(cls):
433 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
436 def suitable(cls, url):
437 return re.match(cls._make_valid_url(), url) is not None
439 def _real_extract(self, query):
440 mobj = re.match(self._make_valid_url(), query)
442 raise ExtractorError(u'Invalid search query "%s"' % query)
444 prefix = mobj.group('prefix')
445 query = mobj.group('query')
447 return self._get_n_results(query, 1)
448 elif prefix == 'all':
449 return self._get_n_results(query, self._MAX_RESULTS)
453 raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query))
454 elif n > self._MAX_RESULTS:
455 self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
456 n = self._MAX_RESULTS
457 return self._get_n_results(query, n)
459 def _get_n_results(self, query, n):
460 """Get a specified number of results for a query"""
461 raise NotImplementedError("This method must be implemented by subclasses")
464 def SEARCH_KEY(self):
465 return self._SEARCH_KEY