youtube_dl/extractor/common.py

   1 import base64
   2 import hashlib
   3 import json
   4 import os
   5 import re
   6 import socket
   7 import sys
   8 import netrc
   9 import xml.etree.ElementTree
  10
  11 from ..utils import (
  12     compat_http_client,
  13     compat_urllib_error,
  14     compat_urllib_parse_urlparse,
  15     compat_str,
  16
  17     clean_html,
  18     compiled_regex_type,
  19     ExtractorError,
  20     RegexNotFoundError,
  21     sanitize_filename,
  22     unescapeHTML,
  23 )
  24 _NO_DEFAULT = object()
  25
  26
  27 class InfoExtractor(object):
  28     """Information Extractor class.
  29
  30     Information extractors are the classes that, given a URL, extract
  31     information about the video (or videos) the URL refers to. This
  32     information includes the real video URL, the video title, author and
  33     others. The information is stored in a dictionary which is then
  34     passed to the FileDownloader. The FileDownloader processes this
  35     information possibly downloading the video to the file system, among
  36     other possible outcomes.
  37
  38     The dictionaries must include the following fields:
  39
  40     id:             Video identifier.
  41     title:          Video title, unescaped.
  42
  43     Additionally, it must contain either a formats entry or a url one:
  44
  45     formats:        A list of dictionaries for each format available, ordered
  46                     from worst to best quality.
  47
  48                     Potential fields:
  49                     * url        Mandatory. The URL of the video file
  50                     * ext        Will be calculated from url if missing
  51                     * format     A human-readable description of the format
  52                                  ("mp4 container with h264/opus").
  53                                  Calculated from the format_id, width, height.
  54                                  and format_note fields if missing.
  55                     * format_id  A short description of the format
  56                                  ("mp4_h264_opus" or "19").
  57                                 Technically optional, but strongly recommended.
  58                     * format_note Additional info about the format
  59                                  ("3D" or "DASH video")
  60                     * width      Width of the video, if known
  61                     * height     Height of the video, if known
  62                     * resolution Textual description of width and height
  63                     * tbr        Average bitrate of audio and video in KBit/s
  64                     * abr        Average audio bitrate in KBit/s
  65                     * acodec     Name of the audio codec in use
  66                     * asr        Audio sampling rate in Hertz
  67                     * vbr        Average video bitrate in KBit/s
  68                     * vcodec     Name of the video codec in use
  69                     * container  Name of the container format
  70                     * filesize   The number of bytes, if known in advance
  71                     * player_url SWF Player URL (used for rtmpdump).
  72                     * protocol   The protocol that will be used for the actual
  73                                  download, lower-case.
  74                                  "http", "https", "rtsp", "rtmp", "m3u8" or so.
  75                     * preference Order number of this format. If this field is
  76                                  present and not None, the formats get sorted
  77                                  by this field.
  78                                  -1 for default (order by other properties),
  79                                  -2 or smaller for less than default.
  80                     * quality    Order number of the video quality of this
  81                                  format, irrespective of the file format.
  82                                  -1 for default (order by other properties),
  83                                  -2 or smaller for less than default.
  84     url:            Final video URL.
  85     ext:            Video filename extension.
  86     format:         The video format, defaults to ext (used for --get-format)
  87     player_url:     SWF Player URL (used for rtmpdump).
  88
  89     The following fields are optional:
  90
  91     thumbnails:     A list of dictionaries (with the entries "resolution" and
  92                     "url") for the varying thumbnails
  93     thumbnail:      Full URL to a video thumbnail image.
  94     description:    One-line video description.
  95     uploader:       Full name of the video uploader.
  96     upload_date:    Video upload date (YYYYMMDD).
  97     uploader_id:    Nickname or id of the video uploader.
  98     location:       Physical location of the video.
  99     subtitles:      The subtitle file contents as a dictionary in the format
 100                     {language: subtitles}.
 101     duration:       Length of the video in seconds, as an integer.
 102     view_count:     How many users have watched the video on the platform.
 103     like_count:     Number of positive ratings of the video
 104     dislike_count:  Number of negative ratings of the video
 105     comment_count:  Number of comments on the video
 106     age_limit:      Age restriction for the video, as an integer (years)
 107     webpage_url:    The url to the video webpage, if given to youtube-dl it
 108                     should allow to get the same result again. (It will be set
 109                     by YoutubeDL if it's missing)
 110
 111     Unless mentioned otherwise, the fields should be Unicode strings.
 112
 113     Subclasses of this one should re-define the _real_initialize() and
 114     _real_extract() methods and define a _VALID_URL regexp.
 115     Probably, they should also be added to the list of extractors.
 116
 117     _real_extract() must return a *list* of information dictionaries as
 118     described above.
 119
 120     Finally, the _WORKING attribute should be set to False for broken IEs
 121     in order to warn the users and skip the tests.
 122     """
 123
 124     _ready = False
 125     _downloader = None
 126     _WORKING = True
 127
 128     def __init__(self, downloader=None):
 129         """Constructor. Receives an optional downloader."""
 130         self._ready = False
 131         self.set_downloader(downloader)
 132
 133     @classmethod
 134     def suitable(cls, url):
 135         """Receives a URL and returns True if suitable for this IE."""
 136
 137         # This does not use has/getattr intentionally - we want to know whether
 138         # we have cached the regexp for *this* class, whereas getattr would also
 139         # match the superclass
 140         if '_VALID_URL_RE' not in cls.__dict__:
 141             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 142         return cls._VALID_URL_RE.match(url) is not None
 143
 144     @classmethod
 145     def working(cls):
 146         """Getter method for _WORKING."""
 147         return cls._WORKING
 148
 149     def initialize(self):
 150         """Initializes an instance (authentication, etc)."""
 151         if not self._ready:
 152             self._real_initialize()
 153             self._ready = True
 154
 155     def extract(self, url):
 156         """Extracts URL information and returns it in list of dicts."""
 157         self.initialize()
 158         return self._real_extract(url)
 159
 160     def set_downloader(self, downloader):
 161         """Sets the downloader for this IE."""
 162         self._downloader = downloader
 163
 164     def _real_initialize(self):
 165         """Real initialization process. Redefine in subclasses."""
 166         pass
 167
 168     def _real_extract(self, url):
 169         """Real extraction process. Redefine in subclasses."""
 170         pass
 171
 172     @classmethod
 173     def ie_key(cls):
 174         """A string for getting the InfoExtractor with get_info_extractor"""
 175         return cls.__name__[:-2]
 176
 177     @property
 178     def IE_NAME(self):
 179         return type(self).__name__[:-2]
 180
 181     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 182         """ Returns the response handle """
 183         if note is None:
 184             self.report_download_webpage(video_id)
 185         elif note is not False:
 186             if video_id is None:
 187                 self.to_screen(u'%s' % (note,))
 188             else:
 189                 self.to_screen(u'%s: %s' % (video_id, note))
 190         try:
 191             return self._downloader.urlopen(url_or_request)
 192         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 193             if errnote is False:
 194                 return False
 195             if errnote is None:
 196                 errnote = u'Unable to download webpage'
 197             errmsg = u'%s: %s' % (errnote, compat_str(err))
 198             if fatal:
 199                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 200             else:
 201                 self._downloader.report_warning(errmsg)
 202                 return False
 203
 204     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 205         """ Returns a tuple (page content as string, URL handle) """
 206
 207         # Strip hashes from the URL (#1038)
 208         if isinstance(url_or_request, (compat_str, str)):
 209             url_or_request = url_or_request.partition('#')[0]
 210
 211         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
 212         if urlh is False:
 213             assert not fatal
 214             return False
 215         content_type = urlh.headers.get('Content-Type', '')
 216         webpage_bytes = urlh.read()
 217         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 218         if m:
 219             encoding = m.group(1)
 220         else:
 221             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 222                           webpage_bytes[:1024])
 223             if m:
 224                 encoding = m.group(1).decode('ascii')
 225             elif webpage_bytes.startswith(b'\xff\xfe'):
 226                 encoding = 'utf-16'
 227             else:
 228                 encoding = 'utf-8'
 229         if self._downloader.params.get('dump_intermediate_pages', False):
 230             try:
 231                 url = url_or_request.get_full_url()
 232             except AttributeError:
 233                 url = url_or_request
 234             self.to_screen(u'Dumping request to ' + url)
 235             dump = base64.b64encode(webpage_bytes).decode('ascii')
 236             self._downloader.to_screen(dump)
 237         if self._downloader.params.get('write_pages', False):
 238             try:
 239                 url = url_or_request.get_full_url()
 240             except AttributeError:
 241                 url = url_or_request
 242             if len(url) > 200:
 243                 h = u'___' + hashlib.md5(url.encode('utf-8')).hexdigest()
 244                 url = url[:200 - len(h)] + h
 245             raw_filename = ('%s_%s.dump' % (video_id, url))
 246             filename = sanitize_filename(raw_filename, restricted=True)
 247             self.to_screen(u'Saving request to ' + filename)
 248             with open(filename, 'wb') as outf:
 249                 outf.write(webpage_bytes)
 250
 251         content = webpage_bytes.decode(encoding, 'replace')
 252         return (content, urlh)
 253
 254     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 255         """ Returns the data of the page as a string """
 256         res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
 257         if res is False:
 258             return res
 259         else:
 260             content, _ = res
 261             return content
 262
 263     def _download_xml(self, url_or_request, video_id,
 264                       note=u'Downloading XML', errnote=u'Unable to download XML',
 265                       transform_source=None):
 266         """Return the xml as an xml.etree.ElementTree.Element"""
 267         xml_string = self._download_webpage(url_or_request, video_id, note, errnote)
 268         if transform_source:
 269             xml_string = transform_source(xml_string)
 270         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 271
 272     def _download_json(self, url_or_request, video_id,
 273                        note=u'Downloading JSON metadata',
 274                        errnote=u'Unable to download JSON metadata'):
 275         json_string = self._download_webpage(url_or_request, video_id, note, errnote)
 276         try:
 277             return json.loads(json_string)
 278         except ValueError as ve:
 279             raise ExtractorError('Failed to download JSON', cause=ve)
 280
 281     def report_warning(self, msg, video_id=None):
 282         idstr = u'' if video_id is None else u'%s: ' % video_id
 283         self._downloader.report_warning(
 284             u'[%s] %s%s' % (self.IE_NAME, idstr, msg))
 285
 286     def to_screen(self, msg):
 287         """Print msg to screen, prefixing it with '[ie_name]'"""
 288         self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg))
 289
 290     def report_extraction(self, id_or_name):
 291         """Report information extraction."""
 292         self.to_screen(u'%s: Extracting information' % id_or_name)
 293
 294     def report_download_webpage(self, video_id):
 295         """Report webpage download."""
 296         self.to_screen(u'%s: Downloading webpage' % video_id)
 297
 298     def report_age_confirmation(self):
 299         """Report attempt to confirm age."""
 300         self.to_screen(u'Confirming age')
 301
 302     def report_login(self):
 303         """Report attempt to log in."""
 304         self.to_screen(u'Logging in')
 305
 306     #Methods for following #608
 307     @staticmethod
 308     def url_result(url, ie=None, video_id=None):
 309         """Returns a url that points to a page that should be processed"""
 310         #TODO: ie should be the class used for getting the info
 311         video_info = {'_type': 'url',
 312                       'url': url,
 313                       'ie_key': ie}
 314         if video_id is not None:
 315             video_info['id'] = video_id
 316         return video_info
 317     @staticmethod
 318     def playlist_result(entries, playlist_id=None, playlist_title=None):
 319         """Returns a playlist"""
 320         video_info = {'_type': 'playlist',
 321                       'entries': entries}
 322         if playlist_id:
 323             video_info['id'] = playlist_id
 324         if playlist_title:
 325             video_info['title'] = playlist_title
 326         return video_info
 327
 328     def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
 329         """
 330         Perform a regex search on the given string, using a single or a list of
 331         patterns returning the first matching group.
 332         In case of failure return a default value or raise a WARNING or a
 333         RegexNotFoundError, depending on fatal, specifying the field name.
 334         """
 335         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 336             mobj = re.search(pattern, string, flags)
 337         else:
 338             for p in pattern:
 339                 mobj = re.search(p, string, flags)
 340                 if mobj: break
 341
 342         if os.name != 'nt' and sys.stderr.isatty():
 343             _name = u'\033[0;34m%s\033[0m' % name
 344         else:
 345             _name = name
 346
 347         if mobj:
 348             # return the first matching group
 349             return next(g for g in mobj.groups() if g is not None)
 350         elif default is not _NO_DEFAULT:
 351             return default
 352         elif fatal:
 353             raise RegexNotFoundError(u'Unable to extract %s' % _name)
 354         else:
 355             self._downloader.report_warning(u'unable to extract %s; '
 356                 u'please report this issue on http://yt-dl.org/bug' % _name)
 357             return None
 358
 359     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
 360         """
 361         Like _search_regex, but strips HTML tags and unescapes entities.
 362         """
 363         res = self._search_regex(pattern, string, name, default, fatal, flags)
 364         if res:
 365             return clean_html(res).strip()
 366         else:
 367             return res
 368
 369     def _get_login_info(self):
 370         """
 371         Get the the login info as (username, password)
 372         It will look in the netrc file using the _NETRC_MACHINE value
 373         If there's no info available, return (None, None)
 374         """
 375         if self._downloader is None:
 376             return (None, None)
 377
 378         username = None
 379         password = None
 380         downloader_params = self._downloader.params
 381
 382         # Attempt to use provided username and password or .netrc data
 383         if downloader_params.get('username', None) is not None:
 384             username = downloader_params['username']
 385             password = downloader_params['password']
 386         elif downloader_params.get('usenetrc', False):
 387             try:
 388                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 389                 if info is not None:
 390                     username = info[0]
 391                     password = info[2]
 392                 else:
 393                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 394             except (IOError, netrc.NetrcParseError) as err:
 395                 self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
 396
 397         return (username, password)
 398
 399     # Helper functions for extracting OpenGraph info
 400     @staticmethod
 401     def _og_regexes(prop):
 402         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
 403         property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
 404         template = r'<meta[^>]+?%s[^>]+?%s'
 405         return [
 406             template % (property_re, content_re),
 407             template % (content_re, property_re),
 408         ]
 409
 410     def _og_search_property(self, prop, html, name=None, **kargs):
 411         if name is None:
 412             name = 'OpenGraph %s' % prop
 413         escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
 414         if escaped is None:
 415             return None
 416         return unescapeHTML(escaped)
 417
 418     def _og_search_thumbnail(self, html, **kargs):
 419         return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)
 420
 421     def _og_search_description(self, html, **kargs):
 422         return self._og_search_property('description', html, fatal=False, **kargs)
 423
 424     def _og_search_title(self, html, **kargs):
 425         return self._og_search_property('title', html, **kargs)
 426
 427     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
 428         regexes = self._og_regexes('video')
 429         if secure: regexes = self._og_regexes('video:secure_url') + regexes
 430         return self._html_search_regex(regexes, html, name, **kargs)
 431
 432     def _html_search_meta(self, name, html, display_name=None):
 433         if display_name is None:
 434             display_name = name
 435         return self._html_search_regex(
 436             r'''(?ix)<meta
 437                     (?=[^>]+(?:itemprop|name|property)=["\']%s["\'])
 438                     [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
 439             html, display_name, fatal=False)
 440
 441     def _dc_search_uploader(self, html):
 442         return self._html_search_meta('dc.creator', html, 'uploader')
 443
 444     def _rta_search(self, html):
 445         # See http://www.rtalabel.org/index.php?content=howtofaq#single
 446         if re.search(r'(?ix)<meta\s+name="rating"\s+'
 447                      r'     content="RTA-5042-1996-1400-1577-RTA"',
 448                      html):
 449             return 18
 450         return 0
 451
 452     def _media_rating_search(self, html):
 453         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
 454         rating = self._html_search_meta('rating', html)
 455
 456         if not rating:
 457             return None
 458
 459         RATING_TABLE = {
 460             'safe for kids': 0,
 461             'general': 8,
 462             '14 years': 14,
 463             'mature': 17,
 464             'restricted': 19,
 465         }
 466         return RATING_TABLE.get(rating.lower(), None)
 467
 468     def _twitter_search_player(self, html):
 469         return self._html_search_meta('twitter:player', html,
 470             'twitter card player')
 471
 472     def _sort_formats(self, formats):
 473         if not formats:
 474             raise ExtractorError(u'No video formats found')
 475
 476         def _formats_key(f):
 477             # TODO remove the following workaround
 478             from ..utils import determine_ext
 479             if not f.get('ext') and 'url' in f:
 480                 f['ext'] = determine_ext(f['url'])
 481
 482             preference = f.get('preference')
 483             if preference is None:
 484                 proto = f.get('protocol')
 485                 if proto is None:
 486                     proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
 487
 488                 preference = 0 if proto in ['http', 'https'] else -0.1
 489                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
 490                     preference -= 0.5
 491
 492             if f.get('vcodec') == 'none':  # audio only
 493                 if self._downloader.params.get('prefer_free_formats'):
 494                     ORDER = [u'aac', u'mp3', u'm4a', u'webm', u'ogg', u'opus']
 495                 else:
 496                     ORDER = [u'webm', u'opus', u'ogg', u'mp3', u'aac', u'm4a']
 497                 ext_preference = 0
 498                 try:
 499                     audio_ext_preference = ORDER.index(f['ext'])
 500                 except ValueError:
 501                     audio_ext_preference = -1
 502             else:
 503                 if self._downloader.params.get('prefer_free_formats'):
 504                     ORDER = [u'flv', u'mp4', u'webm']
 505                 else:
 506                     ORDER = [u'webm', u'flv', u'mp4']
 507                 try:
 508                     ext_preference = ORDER.index(f['ext'])
 509                 except ValueError:
 510                     ext_preference = -1
 511                 audio_ext_preference = 0
 512
 513             return (
 514                 preference,
 515                 f.get('quality') if f.get('quality') is not None else -1,
 516                 f.get('height') if f.get('height') is not None else -1,
 517                 f.get('width') if f.get('width') is not None else -1,
 518                 ext_preference,
 519                 f.get('tbr') if f.get('tbr') is not None else -1,
 520                 f.get('vbr') if f.get('vbr') is not None else -1,
 521                 f.get('abr') if f.get('abr') is not None else -1,
 522                 audio_ext_preference,
 523                 f.get('filesize') if f.get('filesize') is not None else -1,
 524                 f.get('format_id'),
 525             )
 526         formats.sort(key=_formats_key)
 527
 528
 529 class SearchInfoExtractor(InfoExtractor):
 530     """
 531     Base class for paged search queries extractors.
 532     They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
 533     Instances should define _SEARCH_KEY and _MAX_RESULTS.
 534     """
 535
 536     @classmethod
 537     def _make_valid_url(cls):
 538         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
 539
 540     @classmethod
 541     def suitable(cls, url):
 542         return re.match(cls._make_valid_url(), url) is not None
 543
 544     def _real_extract(self, query):
 545         mobj = re.match(self._make_valid_url(), query)
 546         if mobj is None:
 547             raise ExtractorError(u'Invalid search query "%s"' % query)
 548
 549         prefix = mobj.group('prefix')
 550         query = mobj.group('query')
 551         if prefix == '':
 552             return self._get_n_results(query, 1)
 553         elif prefix == 'all':
 554             return self._get_n_results(query, self._MAX_RESULTS)
 555         else:
 556             n = int(prefix)
 557             if n <= 0:
 558                 raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query))
 559             elif n > self._MAX_RESULTS:
 560                 self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
 561                 n = self._MAX_RESULTS
 562             return self._get_n_results(query, n)
 563
 564     def _get_n_results(self, query, n):
 565         """Get a specified number of results for a query"""
 566         raise NotImplementedError("This method must be implemented by subclasses")
 567
 568     @property
 569     def SEARCH_KEY(self):
 570         return self._SEARCH_KEY