youtube_dl/extractor/common.py

   1 from __future__ import unicode_literals
   2
   3 import base64
   4 import datetime
   5 import hashlib
   6 import json
   7 import netrc
   8 import os
   9 import re
  10 import socket
  11 import sys
  12 import time
  13 import xml.etree.ElementTree
  14
  15 from ..compat import (
  16     compat_cookiejar,
  17     compat_HTTPError,
  18     compat_http_client,
  19     compat_urllib_error,
  20     compat_urllib_parse_urlparse,
  21     compat_urlparse,
  22     compat_str,
  23 )
  24 from ..utils import (
  25     NO_DEFAULT,
  26     age_restricted,
  27     bug_reports_message,
  28     clean_html,
  29     compiled_regex_type,
  30     ExtractorError,
  31     float_or_none,
  32     int_or_none,
  33     RegexNotFoundError,
  34     sanitize_filename,
  35     unescapeHTML,
  36 )
  37
  38
  39 class InfoExtractor(object):
  40     """Information Extractor class.
  41
  42     Information extractors are the classes that, given a URL, extract
  43     information about the video (or videos) the URL refers to. This
  44     information includes the real video URL, the video title, author and
  45     others. The information is stored in a dictionary which is then
  46     passed to the YoutubeDL. The YoutubeDL processes this
  47     information possibly downloading the video to the file system, among
  48     other possible outcomes.
  49
  50     The type field determines the type of the result.
  51     By far the most common value (and the default if _type is missing) is
  52     "video", which indicates a single video.
  53
  54     For a video, the dictionaries must include the following fields:
  55
  56     id:             Video identifier.
  57     title:          Video title, unescaped.
  58
  59     Additionally, it must contain either a formats entry or a url one:
  60
  61     formats:        A list of dictionaries for each format available, ordered
  62                     from worst to best quality.
  63
  64                     Potential fields:
  65                     * url        Mandatory. The URL of the video file
  66                     * ext        Will be calculated from url if missing
  67                     * format     A human-readable description of the format
  68                                  ("mp4 container with h264/opus").
  69                                  Calculated from the format_id, width, height.
  70                                  and format_note fields if missing.
  71                     * format_id  A short description of the format
  72                                  ("mp4_h264_opus" or "19").
  73                                 Technically optional, but strongly recommended.
  74                     * format_note Additional info about the format
  75                                  ("3D" or "DASH video")
  76                     * width      Width of the video, if known
  77                     * height     Height of the video, if known
  78                     * resolution Textual description of width and height
  79                     * tbr        Average bitrate of audio and video in KBit/s
  80                     * abr        Average audio bitrate in KBit/s
  81                     * acodec     Name of the audio codec in use
  82                     * asr        Audio sampling rate in Hertz
  83                     * vbr        Average video bitrate in KBit/s
  84                     * fps        Frame rate
  85                     * vcodec     Name of the video codec in use
  86                     * container  Name of the container format
  87                     * filesize   The number of bytes, if known in advance
  88                     * filesize_approx  An estimate for the number of bytes
  89                     * player_url SWF Player URL (used for rtmpdump).
  90                     * protocol   The protocol that will be used for the actual
  91                                  download, lower-case.
  92                                  "http", "https", "rtsp", "rtmp", "rtmpe",
  93                                  "m3u8", or "m3u8_native".
  94                     * preference Order number of this format. If this field is
  95                                  present and not None, the formats get sorted
  96                                  by this field, regardless of all other values.
  97                                  -1 for default (order by other properties),
  98                                  -2 or smaller for less than default.
  99                                  < -1000 to hide the format (if there is
 100                                     another one which is strictly better)
 101                     * language_preference  Is this in the correct requested
 102                                  language?
 103                                  10 if it's what the URL is about,
 104                                  -1 for default (don't know),
 105                                  -10 otherwise, other values reserved for now.
 106                     * quality    Order number of the video quality of this
 107                                  format, irrespective of the file format.
 108                                  -1 for default (order by other properties),
 109                                  -2 or smaller for less than default.
 110                     * source_preference  Order number for this video source
 111                                   (quality takes higher priority)
 112                                  -1 for default (order by other properties),
 113                                  -2 or smaller for less than default.
 114                     * http_headers  A dictionary of additional HTTP headers
 115                                  to add to the request.
 116                     * stretched_ratio  If given and not 1, indicates that the
 117                                  video's pixels are not square.
 118                                  width : height ratio as float.
 119                     * no_resume  The server does not support resuming the
 120                                  (HTTP or RTMP) download. Boolean.
 121
 122     url:            Final video URL.
 123     ext:            Video filename extension.
 124     format:         The video format, defaults to ext (used for --get-format)
 125     player_url:     SWF Player URL (used for rtmpdump).
 126
 127     The following fields are optional:
 128
 129     alt_title:      A secondary title of the video.
 130     display_id      An alternative identifier for the video, not necessarily
 131                     unique, but available before title. Typically, id is
 132                     something like "4234987", title "Dancing naked mole rats",
 133                     and display_id "dancing-naked-mole-rats"
 134     thumbnails:     A list of dictionaries, with the following entries:
 135                         * "id" (optional, string) - Thumbnail format ID
 136                         * "url"
 137                         * "preference" (optional, int) - quality of the image
 138                         * "width" (optional, int)
 139                         * "height" (optional, int)
 140                         * "resolution" (optional, string "{width}x{height"},
 141                                         deprecated)
 142     thumbnail:      Full URL to a video thumbnail image.
 143     description:    Full video description.
 144     uploader:       Full name of the video uploader.
 145     creator:        The main artist who created the video.
 146     timestamp:      UNIX timestamp of the moment the video became available.
 147     upload_date:    Video upload date (YYYYMMDD).
 148                     If not explicitly set, calculated from timestamp.
 149     uploader_id:    Nickname or id of the video uploader.
 150     location:       Physical location where the video was filmed.
 151     subtitles:      The available subtitles as a dictionary in the format
 152                     {language: subformats}. "subformats" is a list sorted from
 153                     lower to higher preference, each element is a dictionary
 154                     with the "ext" entry and one of:
 155                         * "data": The subtitles file contents
 156                         * "url": A url pointing to the subtitles file
 157     automatic_captions: Like 'subtitles', used by the YoutubeIE for
 158                     automatically generated captions
 159     duration:       Length of the video in seconds, as an integer.
 160     view_count:     How many users have watched the video on the platform.
 161     like_count:     Number of positive ratings of the video
 162     dislike_count:  Number of negative ratings of the video
 163     average_rating: Average rating give by users, the scale used depends on the webpage
 164     comment_count:  Number of comments on the video
 165     comments:       A list of comments, each with one or more of the following
 166                     properties (all but one of text or html optional):
 167                         * "author" - human-readable name of the comment author
 168                         * "author_id" - user ID of the comment author
 169                         * "id" - Comment ID
 170                         * "html" - Comment as HTML
 171                         * "text" - Plain text of the comment
 172                         * "timestamp" - UNIX timestamp of comment
 173                         * "parent" - ID of the comment this one is replying to.
 174                                      Set to "root" to indicate that this is a
 175                                      comment to the original video.
 176     age_limit:      Age restriction for the video, as an integer (years)
 177     webpage_url:    The url to the video webpage, if given to youtube-dl it
 178                     should allow to get the same result again. (It will be set
 179                     by YoutubeDL if it's missing)
 180     categories:     A list of categories that the video falls in, for example
 181                     ["Sports", "Berlin"]
 182     is_live:        True, False, or None (=unknown). Whether this video is a
 183                     live stream that goes on instead of a fixed-length video.
 184
 185     Unless mentioned otherwise, the fields should be Unicode strings.
 186
 187     Unless mentioned otherwise, None is equivalent to absence of information.
 188
 189
 190     _type "playlist" indicates multiple videos.
 191     There must be a key "entries", which is a list, an iterable, or a PagedList
 192     object, each element of which is a valid dictionary by this specification.
 193
 194     Additionally, playlists can have "title" and "id" attributes with the same
 195     semantics as videos (see above).
 196
 197
 198     _type "multi_video" indicates that there are multiple videos that
 199     form a single show, for examples multiple acts of an opera or TV episode.
 200     It must have an entries key like a playlist and contain all the keys
 201     required for a video at the same time.
 202
 203
 204     _type "url" indicates that the video must be extracted from another
 205     location, possibly by a different extractor. Its only required key is:
 206     "url" - the next URL to extract.
 207     The key "ie_key" can be set to the class name (minus the trailing "IE",
 208     e.g. "Youtube") if the extractor class is known in advance.
 209     Additionally, the dictionary may have any properties of the resolved entity
 210     known in advance, for example "title" if the title of the referred video is
 211     known ahead of time.
 212
 213
 214     _type "url_transparent" entities have the same specification as "url", but
 215     indicate that the given additional information is more precise than the one
 216     associated with the resolved URL.
 217     This is useful when a site employs a video service that hosts the video and
 218     its technical metadata, but that video service does not embed a useful
 219     title, description etc.
 220
 221
 222     Subclasses of this one should re-define the _real_initialize() and
 223     _real_extract() methods and define a _VALID_URL regexp.
 224     Probably, they should also be added to the list of extractors.
 225
 226     Finally, the _WORKING attribute should be set to False for broken IEs
 227     in order to warn the users and skip the tests.
 228     """
 229
 230     _ready = False
 231     _downloader = None
 232     _WORKING = True
 233
 234     def __init__(self, downloader=None):
 235         """Constructor. Receives an optional downloader."""
 236         self._ready = False
 237         self.set_downloader(downloader)
 238
 239     @classmethod
 240     def suitable(cls, url):
 241         """Receives a URL and returns True if suitable for this IE."""
 242
 243         # This does not use has/getattr intentionally - we want to know whether
 244         # we have cached the regexp for *this* class, whereas getattr would also
 245         # match the superclass
 246         if '_VALID_URL_RE' not in cls.__dict__:
 247             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 248         return cls._VALID_URL_RE.match(url) is not None
 249
 250     @classmethod
 251     def _match_id(cls, url):
 252         if '_VALID_URL_RE' not in cls.__dict__:
 253             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 254         m = cls._VALID_URL_RE.match(url)
 255         assert m
 256         return m.group('id')
 257
 258     @classmethod
 259     def working(cls):
 260         """Getter method for _WORKING."""
 261         return cls._WORKING
 262
 263     def initialize(self):
 264         """Initializes an instance (authentication, etc)."""
 265         if not self._ready:
 266             self._real_initialize()
 267             self._ready = True
 268
 269     def extract(self, url):
 270         """Extracts URL information and returns it in list of dicts."""
 271         try:
 272             self.initialize()
 273             return self._real_extract(url)
 274         except ExtractorError:
 275             raise
 276         except compat_http_client.IncompleteRead as e:
 277             raise ExtractorError('A network error has occured.', cause=e, expected=True)
 278         except (KeyError, StopIteration) as e:
 279             raise ExtractorError('An extractor error has occured.', cause=e)
 280
 281     def set_downloader(self, downloader):
 282         """Sets the downloader for this IE."""
 283         self._downloader = downloader
 284
 285     def _real_initialize(self):
 286         """Real initialization process. Redefine in subclasses."""
 287         pass
 288
 289     def _real_extract(self, url):
 290         """Real extraction process. Redefine in subclasses."""
 291         pass
 292
 293     @classmethod
 294     def ie_key(cls):
 295         """A string for getting the InfoExtractor with get_info_extractor"""
 296         return cls.__name__[:-2]
 297
 298     @property
 299     def IE_NAME(self):
 300         return type(self).__name__[:-2]
 301
 302     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 303         """ Returns the response handle """
 304         if note is None:
 305             self.report_download_webpage(video_id)
 306         elif note is not False:
 307             if video_id is None:
 308                 self.to_screen('%s' % (note,))
 309             else:
 310                 self.to_screen('%s: %s' % (video_id, note))
 311         try:
 312             return self._downloader.urlopen(url_or_request)
 313         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 314             if errnote is False:
 315                 return False
 316             if errnote is None:
 317                 errnote = 'Unable to download webpage'
 318             errmsg = '%s: %s' % (errnote, compat_str(err))
 319             if fatal:
 320                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 321             else:
 322                 self._downloader.report_warning(errmsg)
 323                 return False
 324
 325     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
 326         """ Returns a tuple (page content as string, URL handle) """
 327         # Strip hashes from the URL (#1038)
 328         if isinstance(url_or_request, (compat_str, str)):
 329             url_or_request = url_or_request.partition('#')[0]
 330
 331         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
 332         if urlh is False:
 333             assert not fatal
 334             return False
 335         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 336         return (content, urlh)
 337
 338     @staticmethod
 339     def _guess_encoding_from_content(content_type, webpage_bytes):
 340         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 341         if m:
 342             encoding = m.group(1)
 343         else:
 344             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 345                           webpage_bytes[:1024])
 346             if m:
 347                 encoding = m.group(1).decode('ascii')
 348             elif webpage_bytes.startswith(b'\xff\xfe'):
 349                 encoding = 'utf-16'
 350             else:
 351                 encoding = 'utf-8'
 352
 353         return encoding
 354
 355     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
 356         content_type = urlh.headers.get('Content-Type', '')
 357         webpage_bytes = urlh.read()
 358         if prefix is not None:
 359             webpage_bytes = prefix + webpage_bytes
 360         if not encoding:
 361             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
 362         if self._downloader.params.get('dump_intermediate_pages', False):
 363             try:
 364                 url = url_or_request.get_full_url()
 365             except AttributeError:
 366                 url = url_or_request
 367             self.to_screen('Dumping request to ' + url)
 368             dump = base64.b64encode(webpage_bytes).decode('ascii')
 369             self._downloader.to_screen(dump)
 370         if self._downloader.params.get('write_pages', False):
 371             try:
 372                 url = url_or_request.get_full_url()
 373             except AttributeError:
 374                 url = url_or_request
 375             basen = '%s_%s' % (video_id, url)
 376             if len(basen) > 240:
 377                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
 378                 basen = basen[:240 - len(h)] + h
 379             raw_filename = basen + '.dump'
 380             filename = sanitize_filename(raw_filename, restricted=True)
 381             self.to_screen('Saving request to ' + filename)
 382             # Working around MAX_PATH limitation on Windows (see
 383             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
 384             if os.name == 'nt':
 385                 absfilepath = os.path.abspath(filename)
 386                 if len(absfilepath) > 259:
 387                     filename = '\\\\?\\' + absfilepath
 388             with open(filename, 'wb') as outf:
 389                 outf.write(webpage_bytes)
 390
 391         try:
 392             content = webpage_bytes.decode(encoding, 'replace')
 393         except LookupError:
 394             content = webpage_bytes.decode('utf-8', 'replace')
 395
 396         if ('<title>Access to this site is blocked</title>' in content and
 397                 'Websense' in content[:512]):
 398             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
 399             blocked_iframe = self._html_search_regex(
 400                 r'<iframe src="([^"]+)"', content,
 401                 'Websense information URL', default=None)
 402             if blocked_iframe:
 403                 msg += ' Visit %s for more details' % blocked_iframe
 404             raise ExtractorError(msg, expected=True)
 405         if '<title>The URL you requested has been blocked</title>' in content[:512]:
 406             msg = (
 407                 'Access to this webpage has been blocked by Indian censorship. '
 408                 'Use a VPN or proxy server (with --proxy) to route around it.')
 409             block_msg = self._html_search_regex(
 410                 r'</h1><p>(.*?)</p>',
 411                 content, 'block message', default=None)
 412             if block_msg:
 413                 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
 414             raise ExtractorError(msg, expected=True)
 415
 416         return content
 417
 418     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
 419         """ Returns the data of the page as a string """
 420         success = False
 421         try_count = 0
 422         while success is False:
 423             try:
 424                 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 425                 success = True
 426             except compat_http_client.IncompleteRead as e:
 427                 try_count += 1
 428                 if try_count >= tries:
 429                     raise e
 430                 self._sleep(timeout, video_id)
 431         if res is False:
 432             return res
 433         else:
 434             content, _ = res
 435             return content
 436
 437     def _download_xml(self, url_or_request, video_id,
 438                       note='Downloading XML', errnote='Unable to download XML',
 439                       transform_source=None, fatal=True, encoding=None):
 440         """Return the xml as an xml.etree.ElementTree.Element"""
 441         xml_string = self._download_webpage(
 442             url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
 443         if xml_string is False:
 444             return xml_string
 445         if transform_source:
 446             xml_string = transform_source(xml_string)
 447         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 448
 449     def _download_json(self, url_or_request, video_id,
 450                        note='Downloading JSON metadata',
 451                        errnote='Unable to download JSON metadata',
 452                        transform_source=None,
 453                        fatal=True, encoding=None):
 454         json_string = self._download_webpage(
 455             url_or_request, video_id, note, errnote, fatal=fatal,
 456             encoding=encoding)
 457         if (not fatal) and json_string is False:
 458             return None
 459         return self._parse_json(
 460             json_string, video_id, transform_source=transform_source, fatal=fatal)
 461
 462     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
 463         if transform_source:
 464             json_string = transform_source(json_string)
 465         try:
 466             return json.loads(json_string)
 467         except ValueError as ve:
 468             errmsg = '%s: Failed to parse JSON ' % video_id
 469             if fatal:
 470                 raise ExtractorError(errmsg, cause=ve)
 471             else:
 472                 self.report_warning(errmsg + str(ve))
 473
 474     def report_warning(self, msg, video_id=None):
 475         idstr = '' if video_id is None else '%s: ' % video_id
 476         self._downloader.report_warning(
 477             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
 478
 479     def to_screen(self, msg):
 480         """Print msg to screen, prefixing it with '[ie_name]'"""
 481         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
 482
 483     def report_extraction(self, id_or_name):
 484         """Report information extraction."""
 485         self.to_screen('%s: Extracting information' % id_or_name)
 486
 487     def report_download_webpage(self, video_id):
 488         """Report webpage download."""
 489         self.to_screen('%s: Downloading webpage' % video_id)
 490
 491     def report_age_confirmation(self):
 492         """Report attempt to confirm age."""
 493         self.to_screen('Confirming age')
 494
 495     def report_login(self):
 496         """Report attempt to log in."""
 497         self.to_screen('Logging in')
 498
 499     # Methods for following #608
 500     @staticmethod
 501     def url_result(url, ie=None, video_id=None, video_title=None):
 502         """Returns a url that points to a page that should be processed"""
 503         # TODO: ie should be the class used for getting the info
 504         video_info = {'_type': 'url',
 505                       'url': url,
 506                       'ie_key': ie}
 507         if video_id is not None:
 508             video_info['id'] = video_id
 509         if video_title is not None:
 510             video_info['title'] = video_title
 511         return video_info
 512
 513     @staticmethod
 514     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
 515         """Returns a playlist"""
 516         video_info = {'_type': 'playlist',
 517                       'entries': entries}
 518         if playlist_id:
 519             video_info['id'] = playlist_id
 520         if playlist_title:
 521             video_info['title'] = playlist_title
 522         if playlist_description:
 523             video_info['description'] = playlist_description
 524         return video_info
 525
 526     def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
 527         """
 528         Perform a regex search on the given string, using a single or a list of
 529         patterns returning the first matching group.
 530         In case of failure return a default value or raise a WARNING or a
 531         RegexNotFoundError, depending on fatal, specifying the field name.
 532         """
 533         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 534             mobj = re.search(pattern, string, flags)
 535         else:
 536             for p in pattern:
 537                 mobj = re.search(p, string, flags)
 538                 if mobj:
 539                     break
 540
 541         if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
 542             _name = '\033[0;34m%s\033[0m' % name
 543         else:
 544             _name = name
 545
 546         if mobj:
 547             if group is None:
 548                 # return the first matching group
 549                 return next(g for g in mobj.groups() if g is not None)
 550             else:
 551                 return mobj.group(group)
 552         elif default is not NO_DEFAULT:
 553             return default
 554         elif fatal:
 555             raise RegexNotFoundError('Unable to extract %s' % _name)
 556         else:
 557             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
 558             return None
 559
 560     def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
 561         """
 562         Like _search_regex, but strips HTML tags and unescapes entities.
 563         """
 564         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
 565         if res:
 566             return clean_html(res).strip()
 567         else:
 568             return res
 569
 570     def _get_login_info(self):
 571         """
 572         Get the login info as (username, password)
 573         It will look in the netrc file using the _NETRC_MACHINE value
 574         If there's no info available, return (None, None)
 575         """
 576         if self._downloader is None:
 577             return (None, None)
 578
 579         username = None
 580         password = None
 581         downloader_params = self._downloader.params
 582
 583         # Attempt to use provided username and password or .netrc data
 584         if downloader_params.get('username', None) is not None:
 585             username = downloader_params['username']
 586             password = downloader_params['password']
 587         elif downloader_params.get('usenetrc', False):
 588             try:
 589                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 590                 if info is not None:
 591                     username = info[0]
 592                     password = info[2]
 593                 else:
 594                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 595             except (IOError, netrc.NetrcParseError) as err:
 596                 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
 597
 598         return (username, password)
 599
 600     def _get_tfa_info(self):
 601         """
 602         Get the two-factor authentication info
 603         TODO - asking the user will be required for sms/phone verify
 604         currently just uses the command line option
 605         If there's no info available, return None
 606         """
 607         if self._downloader is None:
 608             return None
 609         downloader_params = self._downloader.params
 610
 611         if downloader_params.get('twofactor', None) is not None:
 612             return downloader_params['twofactor']
 613
 614         return None
 615
 616     # Helper functions for extracting OpenGraph info
 617     @staticmethod
 618     def _og_regexes(prop):
 619         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
 620         property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
 621         template = r'<meta[^>]+?%s[^>]+?%s'
 622         return [
 623             template % (property_re, content_re),
 624             template % (content_re, property_re),
 625         ]
 626
 627     def _og_search_property(self, prop, html, name=None, **kargs):
 628         if name is None:
 629             name = 'OpenGraph %s' % prop
 630         escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
 631         if escaped is None:
 632             return None
 633         return unescapeHTML(escaped)
 634
 635     def _og_search_thumbnail(self, html, **kargs):
 636         return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
 637
 638     def _og_search_description(self, html, **kargs):
 639         return self._og_search_property('description', html, fatal=False, **kargs)
 640
 641     def _og_search_title(self, html, **kargs):
 642         return self._og_search_property('title', html, **kargs)
 643
 644     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
 645         regexes = self._og_regexes('video') + self._og_regexes('video:url')
 646         if secure:
 647             regexes = self._og_regexes('video:secure_url') + regexes
 648         return self._html_search_regex(regexes, html, name, **kargs)
 649
 650     def _og_search_url(self, html, **kargs):
 651         return self._og_search_property('url', html, **kargs)
 652
 653     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
 654         if display_name is None:
 655             display_name = name
 656         return self._html_search_regex(
 657             r'''(?isx)<meta
 658                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
 659                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
 660             html, display_name, fatal=fatal, group='content', **kwargs)
 661
 662     def _dc_search_uploader(self, html):
 663         return self._html_search_meta('dc.creator', html, 'uploader')
 664
 665     def _rta_search(self, html):
 666         # See http://www.rtalabel.org/index.php?content=howtofaq#single
 667         if re.search(r'(?ix)<meta\s+name="rating"\s+'
 668                      r'     content="RTA-5042-1996-1400-1577-RTA"',
 669                      html):
 670             return 18
 671         return 0
 672
 673     def _media_rating_search(self, html):
 674         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
 675         rating = self._html_search_meta('rating', html)
 676
 677         if not rating:
 678             return None
 679
 680         RATING_TABLE = {
 681             'safe for kids': 0,
 682             'general': 8,
 683             '14 years': 14,
 684             'mature': 17,
 685             'restricted': 19,
 686         }
 687         return RATING_TABLE.get(rating.lower(), None)
 688
 689     def _family_friendly_search(self, html):
 690         # See http://schema.org/VideoObject
 691         family_friendly = self._html_search_meta('isFamilyFriendly', html)
 692
 693         if not family_friendly:
 694             return None
 695
 696         RATING_TABLE = {
 697             '1': 0,
 698             'true': 0,
 699             '0': 18,
 700             'false': 18,
 701         }
 702         return RATING_TABLE.get(family_friendly.lower(), None)
 703
 704     def _twitter_search_player(self, html):
 705         return self._html_search_meta('twitter:player', html,
 706                                       'twitter card player')
 707
 708     @staticmethod
 709     def _form_hidden_inputs(html):
 710         return dict(re.findall(
 711             r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
 712             html))
 713
 714     def _sort_formats(self, formats, field_preference=None):
 715         if not formats:
 716             raise ExtractorError('No video formats found')
 717
 718         def _formats_key(f):
 719             # TODO remove the following workaround
 720             from ..utils import determine_ext
 721             if not f.get('ext') and 'url' in f:
 722                 f['ext'] = determine_ext(f['url'])
 723
 724             if isinstance(field_preference, (list, tuple)):
 725                 return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
 726
 727             preference = f.get('preference')
 728             if preference is None:
 729                 proto = f.get('protocol')
 730                 if proto is None:
 731                     proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
 732
 733                 preference = 0 if proto in ['http', 'https'] else -0.1
 734                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
 735                     preference -= 0.5
 736
 737             if f.get('vcodec') == 'none':  # audio only
 738                 if self._downloader.params.get('prefer_free_formats'):
 739                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
 740                 else:
 741                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
 742                 ext_preference = 0
 743                 try:
 744                     audio_ext_preference = ORDER.index(f['ext'])
 745                 except ValueError:
 746                     audio_ext_preference = -1
 747             else:
 748                 if self._downloader.params.get('prefer_free_formats'):
 749                     ORDER = ['flv', 'mp4', 'webm']
 750                 else:
 751                     ORDER = ['webm', 'flv', 'mp4']
 752                 try:
 753                     ext_preference = ORDER.index(f['ext'])
 754                 except ValueError:
 755                     ext_preference = -1
 756                 audio_ext_preference = 0
 757
 758             return (
 759                 preference,
 760                 f.get('language_preference') if f.get('language_preference') is not None else -1,
 761                 f.get('quality') if f.get('quality') is not None else -1,
 762                 f.get('tbr') if f.get('tbr') is not None else -1,
 763                 f.get('filesize') if f.get('filesize') is not None else -1,
 764                 f.get('vbr') if f.get('vbr') is not None else -1,
 765                 f.get('height') if f.get('height') is not None else -1,
 766                 f.get('width') if f.get('width') is not None else -1,
 767                 ext_preference,
 768                 f.get('abr') if f.get('abr') is not None else -1,
 769                 audio_ext_preference,
 770                 f.get('fps') if f.get('fps') is not None else -1,
 771                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
 772                 f.get('source_preference') if f.get('source_preference') is not None else -1,
 773                 f.get('format_id') if f.get('format_id') is not None else '',
 774             )
 775         formats.sort(key=_formats_key)
 776
 777     def _check_formats(self, formats, video_id):
 778         if formats:
 779             formats[:] = filter(
 780                 lambda f: self._is_valid_url(
 781                     f['url'], video_id,
 782                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
 783                 formats)
 784
 785     def _is_valid_url(self, url, video_id, item='video'):
 786         url = self._proto_relative_url(url, scheme='http:')
 787         # For now assume non HTTP(S) URLs always valid
 788         if not (url.startswith('http://') or url.startswith('https://')):
 789             return True
 790         try:
 791             self._request_webpage(url, video_id, 'Checking %s URL' % item)
 792             return True
 793         except ExtractorError as e:
 794             if isinstance(e.cause, compat_HTTPError):
 795                 self.to_screen(
 796                     '%s: %s URL is invalid, skipping' % (video_id, item))
 797                 return False
 798             raise
 799
 800     def http_scheme(self):
 801         """ Either "http:" or "https:", depending on the user's preferences """
 802         return (
 803             'http:'
 804             if self._downloader.params.get('prefer_insecure', False)
 805             else 'https:')
 806
 807     def _proto_relative_url(self, url, scheme=None):
 808         if url is None:
 809             return url
 810         if url.startswith('//'):
 811             if scheme is None:
 812                 scheme = self.http_scheme()
 813             return scheme + url
 814         else:
 815             return url
 816
 817     def _sleep(self, timeout, video_id, msg_template=None):
 818         if msg_template is None:
 819             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
 820         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
 821         self.to_screen(msg)
 822         time.sleep(timeout)
 823
 824     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
 825         manifest = self._download_xml(
 826             manifest_url, video_id, 'Downloading f4m manifest',
 827             'Unable to download f4m manifest')
 828
 829         formats = []
 830         manifest_version = '1.0'
 831         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
 832         if not media_nodes:
 833             manifest_version = '2.0'
 834             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
 835         for i, media_el in enumerate(media_nodes):
 836             if manifest_version == '2.0':
 837                 manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
 838                                 (media_el.attrib.get('href') or media_el.attrib.get('url')))
 839             tbr = int_or_none(media_el.attrib.get('bitrate'))
 840             formats.append({
 841                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
 842                 'url': manifest_url,
 843                 'ext': 'flv',
 844                 'tbr': tbr,
 845                 'width': int_or_none(media_el.attrib.get('width')),
 846                 'height': int_or_none(media_el.attrib.get('height')),
 847                 'preference': preference,
 848             })
 849         self._sort_formats(formats)
 850
 851         return formats
 852
 853     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
 854                               entry_protocol='m3u8', preference=None,
 855                               m3u8_id=None, note=None, errnote=None,
 856                               fatal=True):
 857
 858         formats = [{
 859             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
 860             'url': m3u8_url,
 861             'ext': ext,
 862             'protocol': 'm3u8',
 863             'preference': preference - 1 if preference else -1,
 864             'resolution': 'multiple',
 865             'format_note': 'Quality selection URL',
 866         }]
 867
 868         format_url = lambda u: (
 869             u
 870             if re.match(r'^https?://', u)
 871             else compat_urlparse.urljoin(m3u8_url, u))
 872
 873         m3u8_doc = self._download_webpage(
 874             m3u8_url, video_id,
 875             note=note or 'Downloading m3u8 information',
 876             errnote=errnote or 'Failed to download m3u8 information',
 877             fatal=fatal)
 878         if m3u8_doc is False:
 879             return m3u8_doc
 880         last_info = None
 881         last_media = None
 882         kv_rex = re.compile(
 883             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
 884         for line in m3u8_doc.splitlines():
 885             if line.startswith('#EXT-X-STREAM-INF:'):
 886                 last_info = {}
 887                 for m in kv_rex.finditer(line):
 888                     v = m.group('val')
 889                     if v.startswith('"'):
 890                         v = v[1:-1]
 891                     last_info[m.group('key')] = v
 892             elif line.startswith('#EXT-X-MEDIA:'):
 893                 last_media = {}
 894                 for m in kv_rex.finditer(line):
 895                     v = m.group('val')
 896                     if v.startswith('"'):
 897                         v = v[1:-1]
 898                     last_media[m.group('key')] = v
 899             elif line.startswith('#') or not line.strip():
 900                 continue
 901             else:
 902                 if last_info is None:
 903                     formats.append({'url': format_url(line)})
 904                     continue
 905                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
 906                 format_id = []
 907                 if m3u8_id:
 908                     format_id.append(m3u8_id)
 909                 last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
 910                 format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
 911                 f = {
 912                     'format_id': '-'.join(format_id),
 913                     'url': format_url(line.strip()),
 914                     'tbr': tbr,
 915                     'ext': ext,
 916                     'protocol': entry_protocol,
 917                     'preference': preference,
 918                 }
 919                 codecs = last_info.get('CODECS')
 920                 if codecs:
 921                     # TODO: looks like video codec is not always necessarily goes first
 922                     va_codecs = codecs.split(',')
 923                     if va_codecs[0]:
 924                         f['vcodec'] = va_codecs[0].partition('.')[0]
 925                     if len(va_codecs) > 1 and va_codecs[1]:
 926                         f['acodec'] = va_codecs[1].partition('.')[0]
 927                 resolution = last_info.get('RESOLUTION')
 928                 if resolution:
 929                     width_str, height_str = resolution.split('x')
 930                     f['width'] = int(width_str)
 931                     f['height'] = int(height_str)
 932                 if last_media is not None:
 933                     f['m3u8_media'] = last_media
 934                     last_media = None
 935                 formats.append(f)
 936                 last_info = {}
 937         self._sort_formats(formats)
 938         return formats
 939
 940     # TODO: improve extraction
 941     def _extract_smil_formats(self, smil_url, video_id, fatal=True):
 942         smil = self._download_xml(
 943             smil_url, video_id, 'Downloading SMIL file',
 944             'Unable to download SMIL file', fatal=fatal)
 945         if smil is False:
 946             assert not fatal
 947             return []
 948
 949         base = smil.find('./head/meta').get('base')
 950
 951         formats = []
 952         rtmp_count = 0
 953         if smil.findall('./body/seq/video'):
 954             video = smil.findall('./body/seq/video')[0]
 955             fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
 956             formats.extend(fmts)
 957         else:
 958             for video in smil.findall('./body/switch/video'):
 959                 fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
 960                 formats.extend(fmts)
 961
 962         self._sort_formats(formats)
 963
 964         return formats
 965
 966     def _parse_smil_video(self, video, video_id, base, rtmp_count):
 967         src = video.get('src')
 968         if not src:
 969             return ([], rtmp_count)
 970         bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
 971         width = int_or_none(video.get('width'))
 972         height = int_or_none(video.get('height'))
 973         proto = video.get('proto')
 974         if not proto:
 975             if base:
 976                 if base.startswith('rtmp'):
 977                     proto = 'rtmp'
 978                 elif base.startswith('http'):
 979                     proto = 'http'
 980         ext = video.get('ext')
 981         if proto == 'm3u8':
 982             return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
 983         elif proto == 'rtmp':
 984             rtmp_count += 1
 985             streamer = video.get('streamer') or base
 986             return ([{
 987                 'url': streamer,
 988                 'play_path': src,
 989                 'ext': 'flv',
 990                 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
 991                 'tbr': bitrate,
 992                 'width': width,
 993                 'height': height,
 994             }], rtmp_count)
 995         elif proto.startswith('http'):
 996             return ([{
 997                 'url': base + src,
 998                 'ext': ext or 'flv',
 999                 'tbr': bitrate,
1000                 'width': width,
1001                 'height': height,
1002             }], rtmp_count)
1003
1004     def _live_title(self, name):
1005         """ Generate the title for a live video """
1006         now = datetime.datetime.now()
1007         now_str = now.strftime("%Y-%m-%d %H:%M")
1008         return name + ' ' + now_str
1009
1010     def _int(self, v, name, fatal=False, **kwargs):
1011         res = int_or_none(v, **kwargs)
1012         if 'get_attr' in kwargs:
1013             print(getattr(v, kwargs['get_attr']))
1014         if res is None:
1015             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1016             if fatal:
1017                 raise ExtractorError(msg)
1018             else:
1019                 self._downloader.report_warning(msg)
1020         return res
1021
1022     def _float(self, v, name, fatal=False, **kwargs):
1023         res = float_or_none(v, **kwargs)
1024         if res is None:
1025             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1026             if fatal:
1027                 raise ExtractorError(msg)
1028             else:
1029                 self._downloader.report_warning(msg)
1030         return res
1031
1032     def _set_cookie(self, domain, name, value, expire_time=None):
1033         cookie = compat_cookiejar.Cookie(
1034             0, name, value, None, None, domain, None,
1035             None, '/', True, False, expire_time, '', None, None, None)
1036         self._downloader.cookiejar.set_cookie(cookie)
1037
1038     def get_testcases(self, include_onlymatching=False):
1039         t = getattr(self, '_TEST', None)
1040         if t:
1041             assert not hasattr(self, '_TESTS'), \
1042                 '%s has _TEST and _TESTS' % type(self).__name__
1043             tests = [t]
1044         else:
1045             tests = getattr(self, '_TESTS', [])
1046         for t in tests:
1047             if not include_onlymatching and t.get('only_matching', False):
1048                 continue
1049             t['name'] = type(self).__name__[:-len('IE')]
1050             yield t
1051
1052     def is_suitable(self, age_limit):
1053         """ Test whether the extractor is generally suitable for the given
1054         age limit (i.e. pornographic sites are not, all others usually are) """
1055
1056         any_restricted = False
1057         for tc in self.get_testcases(include_onlymatching=False):
1058             if 'playlist' in tc:
1059                 tc = tc['playlist'][0]
1060             is_restricted = age_restricted(
1061                 tc.get('info_dict', {}).get('age_limit'), age_limit)
1062             if not is_restricted:
1063                 return True
1064             any_restricted = any_restricted or is_restricted
1065         return not any_restricted
1066
1067     def extract_subtitles(self, *args, **kwargs):
1068         if (self._downloader.params.get('writesubtitles', False) or
1069                 self._downloader.params.get('listsubtitles')):
1070             return self._get_subtitles(*args, **kwargs)
1071         return {}
1072
1073     def _get_subtitles(self, *args, **kwargs):
1074         raise NotImplementedError("This method must be implemented by subclasses")
1075
1076     def extract_automatic_captions(self, *args, **kwargs):
1077         if (self._downloader.params.get('writeautomaticsub', False) or
1078                 self._downloader.params.get('listsubtitles')):
1079             return self._get_automatic_captions(*args, **kwargs)
1080         return {}
1081
1082     def _get_automatic_captions(self, *args, **kwargs):
1083         raise NotImplementedError("This method must be implemented by subclasses")
1084
1085
1086 class SearchInfoExtractor(InfoExtractor):
1087     """
1088     Base class for paged search queries extractors.
1089     They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
1090     Instances should define _SEARCH_KEY and _MAX_RESULTS.
1091     """
1092
1093     @classmethod
1094     def _make_valid_url(cls):
1095         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
1096
1097     @classmethod
1098     def suitable(cls, url):
1099         return re.match(cls._make_valid_url(), url) is not None
1100
1101     def _real_extract(self, query):
1102         mobj = re.match(self._make_valid_url(), query)
1103         if mobj is None:
1104             raise ExtractorError('Invalid search query "%s"' % query)
1105
1106         prefix = mobj.group('prefix')
1107         query = mobj.group('query')
1108         if prefix == '':
1109             return self._get_n_results(query, 1)
1110         elif prefix == 'all':
1111             return self._get_n_results(query, self._MAX_RESULTS)
1112         else:
1113             n = int(prefix)
1114             if n <= 0:
1115                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
1116             elif n > self._MAX_RESULTS:
1117                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
1118                 n = self._MAX_RESULTS
1119             return self._get_n_results(query, n)
1120
1121     def _get_n_results(self, query, n):
1122         """Get a specified number of results for a query"""
1123         raise NotImplementedError("This method must be implemented by subclasses")
1124
1125     @property
1126     def SEARCH_KEY(self):
1127         return self._SEARCH_KEY