youtube_dl/extractor/common.py

   1 import base64
   2 import hashlib
   3 import json
   4 import os
   5 import re
   6 import socket
   7 import sys
   8 import netrc
   9 import xml.etree.ElementTree
  10
  11 from ..utils import (
  12     compat_http_client,
  13     compat_urllib_error,
  14     compat_urllib_parse_urlparse,
  15     compat_str,
  16
  17     clean_html,
  18     compiled_regex_type,
  19     ExtractorError,
  20     RegexNotFoundError,
  21     sanitize_filename,
  22     unescapeHTML,
  23 )
  24 _NO_DEFAULT = object()
  25
  26
  27 class InfoExtractor(object):
  28     """Information Extractor class.
  29
  30     Information extractors are the classes that, given a URL, extract
  31     information about the video (or videos) the URL refers to. This
  32     information includes the real video URL, the video title, author and
  33     others. The information is stored in a dictionary which is then
  34     passed to the FileDownloader. The FileDownloader processes this
  35     information possibly downloading the video to the file system, among
  36     other possible outcomes.
  37
  38     The dictionaries must include the following fields:
  39
  40     id:             Video identifier.
  41     title:          Video title, unescaped.
  42
  43     Additionally, it must contain either a formats entry or a url one:
  44
  45     formats:        A list of dictionaries for each format available, ordered
  46                     from worst to best quality.
  47
  48                     Potential fields:
  49                     * url        Mandatory. The URL of the video file
  50                     * ext        Will be calculated from url if missing
  51                     * format     A human-readable description of the format
  52                                  ("mp4 container with h264/opus").
  53                                  Calculated from the format_id, width, height.
  54                                  and format_note fields if missing.
  55                     * format_id  A short description of the format
  56                                  ("mp4_h264_opus" or "19").
  57                                 Technically optional, but strongly recommended.
  58                     * format_note Additional info about the format
  59                                  ("3D" or "DASH video")
  60                     * width      Width of the video, if known
  61                     * height     Height of the video, if known
  62                     * resolution Textual description of width and height
  63                     * tbr        Average bitrate of audio and video in KBit/s
  64                     * abr        Average audio bitrate in KBit/s
  65                     * acodec     Name of the audio codec in use
  66                     * asr        Audio sampling rate in Hertz
  67                     * vbr        Average video bitrate in KBit/s
  68                     * vcodec     Name of the video codec in use
  69                     * container  Name of the container format
  70                     * filesize   The number of bytes, if known in advance
  71                     * player_url SWF Player URL (used for rtmpdump).
  72                     * protocol   The protocol that will be used for the actual
  73                                  download, lower-case.
  74                                  "http", "https", "rtsp", "rtmp", "m3u8" or so.
  75                     * preference Order number of this format. If this field is
  76                                  present and not None, the formats get sorted
  77                                  by this field, regardless of all other values.
  78                                  -1 for default (order by other properties),
  79                                  -2 or smaller for less than default.
  80                     * quality    Order number of the video quality of this
  81                                  format, irrespective of the file format.
  82                                  -1 for default (order by other properties),
  83                                  -2 or smaller for less than default.
  84     url:            Final video URL.
  85     ext:            Video filename extension.
  86     format:         The video format, defaults to ext (used for --get-format)
  87     player_url:     SWF Player URL (used for rtmpdump).
  88
  89     The following fields are optional:
  90
  91     display_id      An alternative identifier for the video, not necessarily
  92                     unique, but available before title. Typically, id is
  93                     something like "4234987", title "Dancing naked mole rats",
  94                     and display_id "dancing-naked-mole-rats"
  95     thumbnails:     A list of dictionaries (with the entries "resolution" and
  96                     "url") for the varying thumbnails
  97     thumbnail:      Full URL to a video thumbnail image.
  98     description:    One-line video description.
  99     uploader:       Full name of the video uploader.
 100     timestamp:      UNIX timestamp of the moment the video became available.
 101     upload_date:    Video upload date (YYYYMMDD).
 102                     If not explicitly set, calculated from timestamp.
 103     uploader_id:    Nickname or id of the video uploader.
 104     location:       Physical location of the video.
 105     subtitles:      The subtitle file contents as a dictionary in the format
 106                     {language: subtitles}.
 107     duration:       Length of the video in seconds, as an integer.
 108     view_count:     How many users have watched the video on the platform.
 109     like_count:     Number of positive ratings of the video
 110     dislike_count:  Number of negative ratings of the video
 111     comment_count:  Number of comments on the video
 112     age_limit:      Age restriction for the video, as an integer (years)
 113     webpage_url:    The url to the video webpage, if given to youtube-dl it
 114                     should allow to get the same result again. (It will be set
 115                     by YoutubeDL if it's missing)
 116
 117     Unless mentioned otherwise, the fields should be Unicode strings.
 118
 119     Subclasses of this one should re-define the _real_initialize() and
 120     _real_extract() methods and define a _VALID_URL regexp.
 121     Probably, they should also be added to the list of extractors.
 122
 123     Finally, the _WORKING attribute should be set to False for broken IEs
 124     in order to warn the users and skip the tests.
 125     """
 126
 127     _ready = False
 128     _downloader = None
 129     _WORKING = True
 130
 131     def __init__(self, downloader=None):
 132         """Constructor. Receives an optional downloader."""
 133         self._ready = False
 134         self.set_downloader(downloader)
 135
 136     @classmethod
 137     def suitable(cls, url):
 138         """Receives a URL and returns True if suitable for this IE."""
 139
 140         # This does not use has/getattr intentionally - we want to know whether
 141         # we have cached the regexp for *this* class, whereas getattr would also
 142         # match the superclass
 143         if '_VALID_URL_RE' not in cls.__dict__:
 144             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 145         return cls._VALID_URL_RE.match(url) is not None
 146
 147     @classmethod
 148     def working(cls):
 149         """Getter method for _WORKING."""
 150         return cls._WORKING
 151
 152     def initialize(self):
 153         """Initializes an instance (authentication, etc)."""
 154         if not self._ready:
 155             self._real_initialize()
 156             self._ready = True
 157
 158     def extract(self, url):
 159         """Extracts URL information and returns it in list of dicts."""
 160         self.initialize()
 161         return self._real_extract(url)
 162
 163     def set_downloader(self, downloader):
 164         """Sets the downloader for this IE."""
 165         self._downloader = downloader
 166
 167     def _real_initialize(self):
 168         """Real initialization process. Redefine in subclasses."""
 169         pass
 170
 171     def _real_extract(self, url):
 172         """Real extraction process. Redefine in subclasses."""
 173         pass
 174
 175     @classmethod
 176     def ie_key(cls):
 177         """A string for getting the InfoExtractor with get_info_extractor"""
 178         return cls.__name__[:-2]
 179
 180     @property
 181     def IE_NAME(self):
 182         return type(self).__name__[:-2]
 183
 184     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 185         """ Returns the response handle """
 186         if note is None:
 187             self.report_download_webpage(video_id)
 188         elif note is not False:
 189             if video_id is None:
 190                 self.to_screen(u'%s' % (note,))
 191             else:
 192                 self.to_screen(u'%s: %s' % (video_id, note))
 193         try:
 194             return self._downloader.urlopen(url_or_request)
 195         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 196             if errnote is False:
 197                 return False
 198             if errnote is None:
 199                 errnote = u'Unable to download webpage'
 200             errmsg = u'%s: %s' % (errnote, compat_str(err))
 201             if fatal:
 202                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 203             else:
 204                 self._downloader.report_warning(errmsg)
 205                 return False
 206
 207     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 208         """ Returns a tuple (page content as string, URL handle) """
 209
 210         # Strip hashes from the URL (#1038)
 211         if isinstance(url_or_request, (compat_str, str)):
 212             url_or_request = url_or_request.partition('#')[0]
 213
 214         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
 215         if urlh is False:
 216             assert not fatal
 217             return False
 218         content_type = urlh.headers.get('Content-Type', '')
 219         webpage_bytes = urlh.read()
 220         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 221         if m:
 222             encoding = m.group(1)
 223         else:
 224             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 225                           webpage_bytes[:1024])
 226             if m:
 227                 encoding = m.group(1).decode('ascii')
 228             elif webpage_bytes.startswith(b'\xff\xfe'):
 229                 encoding = 'utf-16'
 230             else:
 231                 encoding = 'utf-8'
 232         if self._downloader.params.get('dump_intermediate_pages', False):
 233             try:
 234                 url = url_or_request.get_full_url()
 235             except AttributeError:
 236                 url = url_or_request
 237             self.to_screen(u'Dumping request to ' + url)
 238             dump = base64.b64encode(webpage_bytes).decode('ascii')
 239             self._downloader.to_screen(dump)
 240         if self._downloader.params.get('write_pages', False):
 241             try:
 242                 url = url_or_request.get_full_url()
 243             except AttributeError:
 244                 url = url_or_request
 245             if len(url) > 200:
 246                 h = u'___' + hashlib.md5(url.encode('utf-8')).hexdigest()
 247                 url = url[:200 - len(h)] + h
 248             raw_filename = ('%s_%s.dump' % (video_id, url))
 249             filename = sanitize_filename(raw_filename, restricted=True)
 250             self.to_screen(u'Saving request to ' + filename)
 251             with open(filename, 'wb') as outf:
 252                 outf.write(webpage_bytes)
 253
 254         try:
 255             content = webpage_bytes.decode(encoding, 'replace')
 256         except LookupError:
 257             content = webpage_bytes.decode('utf-8', 'replace')
 258
 259         if (u'<title>Access to this site is blocked</title>' in content and
 260                 u'Websense' in content[:512]):
 261             msg = u'Access to this webpage has been blocked by Websense filtering software in your network.'
 262             blocked_iframe = self._html_search_regex(
 263                 r'<iframe src="([^"]+)"', content,
 264                 u'Websense information URL', default=None)
 265             if blocked_iframe:
 266                 msg += u' Visit %s for more details' % blocked_iframe
 267             raise ExtractorError(msg, expected=True)
 268
 269         return (content, urlh)
 270
 271     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 272         """ Returns the data of the page as a string """
 273         res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
 274         if res is False:
 275             return res
 276         else:
 277             content, _ = res
 278             return content
 279
 280     def _download_xml(self, url_or_request, video_id,
 281                       note=u'Downloading XML', errnote=u'Unable to download XML',
 282                       transform_source=None):
 283         """Return the xml as an xml.etree.ElementTree.Element"""
 284         xml_string = self._download_webpage(url_or_request, video_id, note, errnote)
 285         if transform_source:
 286             xml_string = transform_source(xml_string)
 287         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 288
 289     def _download_json(self, url_or_request, video_id,
 290                        note=u'Downloading JSON metadata',
 291                        errnote=u'Unable to download JSON metadata',
 292                        transform_source=None):
 293         json_string = self._download_webpage(url_or_request, video_id, note, errnote)
 294         if transform_source:
 295             json_string = transform_source(json_string)
 296         try:
 297             return json.loads(json_string)
 298         except ValueError as ve:
 299             raise ExtractorError('Failed to download JSON', cause=ve)
 300
 301     def report_warning(self, msg, video_id=None):
 302         idstr = u'' if video_id is None else u'%s: ' % video_id
 303         self._downloader.report_warning(
 304             u'[%s] %s%s' % (self.IE_NAME, idstr, msg))
 305
 306     def to_screen(self, msg):
 307         """Print msg to screen, prefixing it with '[ie_name]'"""
 308         self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg))
 309
 310     def report_extraction(self, id_or_name):
 311         """Report information extraction."""
 312         self.to_screen(u'%s: Extracting information' % id_or_name)
 313
 314     def report_download_webpage(self, video_id):
 315         """Report webpage download."""
 316         self.to_screen(u'%s: Downloading webpage' % video_id)
 317
 318     def report_age_confirmation(self):
 319         """Report attempt to confirm age."""
 320         self.to_screen(u'Confirming age')
 321
 322     def report_login(self):
 323         """Report attempt to log in."""
 324         self.to_screen(u'Logging in')
 325
 326     #Methods for following #608
 327     @staticmethod
 328     def url_result(url, ie=None, video_id=None):
 329         """Returns a url that points to a page that should be processed"""
 330         #TODO: ie should be the class used for getting the info
 331         video_info = {'_type': 'url',
 332                       'url': url,
 333                       'ie_key': ie}
 334         if video_id is not None:
 335             video_info['id'] = video_id
 336         return video_info
 337     @staticmethod
 338     def playlist_result(entries, playlist_id=None, playlist_title=None):
 339         """Returns a playlist"""
 340         video_info = {'_type': 'playlist',
 341                       'entries': entries}
 342         if playlist_id:
 343             video_info['id'] = playlist_id
 344         if playlist_title:
 345             video_info['title'] = playlist_title
 346         return video_info
 347
 348     def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
 349         """
 350         Perform a regex search on the given string, using a single or a list of
 351         patterns returning the first matching group.
 352         In case of failure return a default value or raise a WARNING or a
 353         RegexNotFoundError, depending on fatal, specifying the field name.
 354         """
 355         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 356             mobj = re.search(pattern, string, flags)
 357         else:
 358             for p in pattern:
 359                 mobj = re.search(p, string, flags)
 360                 if mobj: break
 361
 362         if os.name != 'nt' and sys.stderr.isatty():
 363             _name = u'\033[0;34m%s\033[0m' % name
 364         else:
 365             _name = name
 366
 367         if mobj:
 368             # return the first matching group
 369             return next(g for g in mobj.groups() if g is not None)
 370         elif default is not _NO_DEFAULT:
 371             return default
 372         elif fatal:
 373             raise RegexNotFoundError(u'Unable to extract %s' % _name)
 374         else:
 375             self._downloader.report_warning(u'unable to extract %s; '
 376                 u'please report this issue on http://yt-dl.org/bug' % _name)
 377             return None
 378
 379     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
 380         """
 381         Like _search_regex, but strips HTML tags and unescapes entities.
 382         """
 383         res = self._search_regex(pattern, string, name, default, fatal, flags)
 384         if res:
 385             return clean_html(res).strip()
 386         else:
 387             return res
 388
 389     def _get_login_info(self):
 390         """
 391         Get the the login info as (username, password)
 392         It will look in the netrc file using the _NETRC_MACHINE value
 393         If there's no info available, return (None, None)
 394         """
 395         if self._downloader is None:
 396             return (None, None)
 397
 398         username = None
 399         password = None
 400         downloader_params = self._downloader.params
 401
 402         # Attempt to use provided username and password or .netrc data
 403         if downloader_params.get('username', None) is not None:
 404             username = downloader_params['username']
 405             password = downloader_params['password']
 406         elif downloader_params.get('usenetrc', False):
 407             try:
 408                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 409                 if info is not None:
 410                     username = info[0]
 411                     password = info[2]
 412                 else:
 413                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 414             except (IOError, netrc.NetrcParseError) as err:
 415                 self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
 416
 417         return (username, password)
 418
 419     # Helper functions for extracting OpenGraph info
 420     @staticmethod
 421     def _og_regexes(prop):
 422         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
 423         property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
 424         template = r'<meta[^>]+?%s[^>]+?%s'
 425         return [
 426             template % (property_re, content_re),
 427             template % (content_re, property_re),
 428         ]
 429
 430     def _og_search_property(self, prop, html, name=None, **kargs):
 431         if name is None:
 432             name = 'OpenGraph %s' % prop
 433         escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
 434         if escaped is None:
 435             return None
 436         return unescapeHTML(escaped)
 437
 438     def _og_search_thumbnail(self, html, **kargs):
 439         return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)
 440
 441     def _og_search_description(self, html, **kargs):
 442         return self._og_search_property('description', html, fatal=False, **kargs)
 443
 444     def _og_search_title(self, html, **kargs):
 445         return self._og_search_property('title', html, **kargs)
 446
 447     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
 448         regexes = self._og_regexes('video')
 449         if secure: regexes = self._og_regexes('video:secure_url') + regexes
 450         return self._html_search_regex(regexes, html, name, **kargs)
 451
 452     def _html_search_meta(self, name, html, display_name=None, fatal=False):
 453         if display_name is None:
 454             display_name = name
 455         return self._html_search_regex(
 456             r'''(?ix)<meta
 457                     (?=[^>]+(?:itemprop|name|property)=["\']%s["\'])
 458                     [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
 459             html, display_name, fatal=fatal)
 460
 461     def _dc_search_uploader(self, html):
 462         return self._html_search_meta('dc.creator', html, 'uploader')
 463
 464     def _rta_search(self, html):
 465         # See http://www.rtalabel.org/index.php?content=howtofaq#single
 466         if re.search(r'(?ix)<meta\s+name="rating"\s+'
 467                      r'     content="RTA-5042-1996-1400-1577-RTA"',
 468                      html):
 469             return 18
 470         return 0
 471
 472     def _media_rating_search(self, html):
 473         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
 474         rating = self._html_search_meta('rating', html)
 475
 476         if not rating:
 477             return None
 478
 479         RATING_TABLE = {
 480             'safe for kids': 0,
 481             'general': 8,
 482             '14 years': 14,
 483             'mature': 17,
 484             'restricted': 19,
 485         }
 486         return RATING_TABLE.get(rating.lower(), None)
 487
 488     def _twitter_search_player(self, html):
 489         return self._html_search_meta('twitter:player', html,
 490             'twitter card player')
 491
 492     def _sort_formats(self, formats):
 493         if not formats:
 494             raise ExtractorError(u'No video formats found')
 495
 496         def _formats_key(f):
 497             # TODO remove the following workaround
 498             from ..utils import determine_ext
 499             if not f.get('ext') and 'url' in f:
 500                 f['ext'] = determine_ext(f['url'])
 501
 502             preference = f.get('preference')
 503             if preference is None:
 504                 proto = f.get('protocol')
 505                 if proto is None:
 506                     proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
 507
 508                 preference = 0 if proto in ['http', 'https'] else -0.1
 509                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
 510                     preference -= 0.5
 511
 512             if f.get('vcodec') == 'none':  # audio only
 513                 if self._downloader.params.get('prefer_free_formats'):
 514                     ORDER = [u'aac', u'mp3', u'm4a', u'webm', u'ogg', u'opus']
 515                 else:
 516                     ORDER = [u'webm', u'opus', u'ogg', u'mp3', u'aac', u'm4a']
 517                 ext_preference = 0
 518                 try:
 519                     audio_ext_preference = ORDER.index(f['ext'])
 520                 except ValueError:
 521                     audio_ext_preference = -1
 522             else:
 523                 if self._downloader.params.get('prefer_free_formats'):
 524                     ORDER = [u'flv', u'mp4', u'webm']
 525                 else:
 526                     ORDER = [u'webm', u'flv', u'mp4']
 527                 try:
 528                     ext_preference = ORDER.index(f['ext'])
 529                 except ValueError:
 530                     ext_preference = -1
 531                 audio_ext_preference = 0
 532
 533             return (
 534                 preference,
 535                 f.get('quality') if f.get('quality') is not None else -1,
 536                 f.get('height') if f.get('height') is not None else -1,
 537                 f.get('width') if f.get('width') is not None else -1,
 538                 ext_preference,
 539                 f.get('tbr') if f.get('tbr') is not None else -1,
 540                 f.get('vbr') if f.get('vbr') is not None else -1,
 541                 f.get('abr') if f.get('abr') is not None else -1,
 542                 audio_ext_preference,
 543                 f.get('filesize') if f.get('filesize') is not None else -1,
 544                 f.get('format_id'),
 545             )
 546         formats.sort(key=_formats_key)
 547
 548
 549 class SearchInfoExtractor(InfoExtractor):
 550     """
 551     Base class for paged search queries extractors.
 552     They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
 553     Instances should define _SEARCH_KEY and _MAX_RESULTS.
 554     """
 555
 556     @classmethod
 557     def _make_valid_url(cls):
 558         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
 559
 560     @classmethod
 561     def suitable(cls, url):
 562         return re.match(cls._make_valid_url(), url) is not None
 563
 564     def _real_extract(self, query):
 565         mobj = re.match(self._make_valid_url(), query)
 566         if mobj is None:
 567             raise ExtractorError(u'Invalid search query "%s"' % query)
 568
 569         prefix = mobj.group('prefix')
 570         query = mobj.group('query')
 571         if prefix == '':
 572             return self._get_n_results(query, 1)
 573         elif prefix == 'all':
 574             return self._get_n_results(query, self._MAX_RESULTS)
 575         else:
 576             n = int(prefix)
 577             if n <= 0:
 578                 raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query))
 579             elif n > self._MAX_RESULTS:
 580                 self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
 581                 n = self._MAX_RESULTS
 582             return self._get_n_results(query, n)
 583
 584     def _get_n_results(self, query, n):
 585         """Get a specified number of results for a query"""
 586         raise NotImplementedError("This method must be implemented by subclasses")
 587
 588     @property
 589     def SEARCH_KEY(self):
 590         return self._SEARCH_KEY